diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 02:57:58 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 02:57:58 +0000 |
| commit | be1c7e50e1e8809ea56f2c9d472eccd8ffd73a97 (patch) | |
| tree | 9754ff1ca740f6346cf8483ec915d4054bc5da2d /fluent-bit/include | |
| parent | Initial commit. (diff) | |
| download | netdata-be1c7e50e1e8809ea56f2c9d472eccd8ffd73a97.tar.xz netdata-be1c7e50e1e8809ea56f2c9d472eccd8ffd73a97.zip | |
Adding upstream version 1.44.3.upstream/1.44.3upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'fluent-bit/include')
168 files changed, 16004 insertions, 0 deletions
diff --git a/fluent-bit/include/CMakeLists.txt b/fluent-bit/include/CMakeLists.txt new file mode 100644 index 00000000..38f1b0dc --- /dev/null +++ b/fluent-bit/include/CMakeLists.txt @@ -0,0 +1,47 @@ +# Install fluent-bit headers +install(FILES "fluent-bit.h" + DESTINATION ${FLB_INSTALL_INCLUDEDIR} + COMPONENT headers + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) + +file(GLOB headers "fluent-bit/*.h") +install(FILES ${headers} + DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit + COMPONENT headers + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) + +file(GLOB headers "fluent-bit/config_format/*.h") +install(FILES ${headers} + DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit/config_format/ + COMPONENT headers + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) + +file(GLOB headers "fluent-bit/tls/*.h") +install(FILES ${headers} + DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit/tls/ + COMPONENT headers + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) + +file(GLOB headers "fluent-bit/wasm/*.h") +install(FILES ${headers} + DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit/wasm/ + COMPONENT headers + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) + +install(FILES "../lib/monkey/include/monkey/mk_core.h" + DESTINATION ${FLB_INSTALL_INCLUDEDIR}/monkey/ + COMPONENT headers-extra + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) + +file(GLOB headers "../lib/monkey/include/monkey/mk_core/*.h") + install(FILES ${headers} + COMPONENT headers-extra + DESTINATION ${FLB_INSTALL_INCLUDEDIR}/monkey/mk_core/ + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) + + +file(GLOB headers "../lib/flb_libco/*.h") + install(FILES ${headers} + COMPONENT headers-extra + DESTINATION ${FLB_INSTALL_INCLUDEDIR}/ + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) diff --git a/fluent-bit/include/fluent-bit.h b/fluent-bit/include/fluent-bit.h new file mode 100644 index 00000000..f84cfc17 --- /dev/null +++ b/fluent-bit/include/fluent-bit.h @@ -0,0 +1,49 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLUENT_BIT_H +#define FLUENT_BIT_H + +#include <msgpack.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#include <monkey/mk_core.h> + +#include <cfl/cfl.h> + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_mem.h> +#include <fluent-bit/flb_str.h> +#include <fluent-bit/flb_macros.h> +#include <fluent-bit/flb_utils.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_version.h> +#include <fluent-bit/flb_error.h> +#include <fluent-bit/flb_output.h> +#include <fluent-bit/flb_engine.h> +#include <fluent-bit/flb_lib.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/fluent-bit/include/fluent-bit/aws/flb_aws_compress.h b/fluent-bit/include/fluent-bit/aws/flb_aws_compress.h new file mode 100644 index 00000000..e1cf9222 --- /dev/null +++ b/fluent-bit/include/fluent-bit/aws/flb_aws_compress.h @@ -0,0 +1,63 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_AWS_COMPRESS +#define FLB_AWS_COMPRESS + +#include <sys/types.h> +#define FLB_AWS_COMPRESS_NONE 0 +#define FLB_AWS_COMPRESS_GZIP 1 +#define FLB_AWS_COMPRESS_ARROW 2 + +/* + * Get compression type from compression keyword. The return value is used to identify + * what compression option to utilize. + * + * Returns int compression type id - FLB_AWS_COMPRESS_<compression-type> + */ +int flb_aws_compression_get_type(const char *compression_keyword); + +/* + * Compress in_data and write result to newly allocated out_data buf + * Client is responsable for freeing out_data. + * + * Returns -1 on error + * Returns 0 on success + */ +int flb_aws_compression_compress(int compression_type, void *in_data, size_t in_len, + void **out_data, size_t *out_len); + +/* + * Truncate and compress in_data and convert to b64 + * If b64 output data is larger than max_out_len, the input is truncated with a + * [Truncated...] suffix appended to the end, and recompressed. The result is written to a + * newly allocated out_data buf. + * Client is responsable for freeing out_data. + * + * out_len and max_out_len do not count the null character as a part of out_data's length, + * though the null character may be included at the end of out_data. + * + * Returns -1 on error + * Returns 0 on success + */ +int flb_aws_compression_b64_truncate_compress(int compression_type, size_t max_out_len, + void *in_data, size_t in_len, + void **out_data, size_t *out_len); + +#endif diff --git a/fluent-bit/include/fluent-bit/aws/flb_aws_error_reporter.h b/fluent-bit/include/fluent-bit/aws/flb_aws_error_reporter.h new file mode 100644 index 00000000..7cb161b4 --- /dev/null +++ b/fluent-bit/include/fluent-bit/aws/flb_aws_error_reporter.h @@ -0,0 +1,84 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef FLB_HAVE_AWS_ERROR_REPORTER + +#ifndef FLB_AWS_ERROR_REPORTER_H +#define FLB_AWS_ERROR_REPORTER_H + +#include <monkey/mk_core/mk_list.h> +#include <fluent-bit/flb_sds.h> + +#include <time.h> + +#define STATUS_MESSAGE_FILE_PATH_ENV "STATUS_MESSAGE_FILE_PATH" +#define STATUS_MESSAGE_TTL_ENV "STATUS_MESSAGE_TTL" +#define STATUS_MESSAGE_MAX_BYTE_LENGTH_ENV "STATUS_MESSAGE_MAX_BYTE_LENGTH" +#define STATUS_MESSAGE_TTL_DEFAULT 20 +#define STATUS_MESSAGE_MAX_BYTE_LENGTH_DEFAULT 1024 + +/* +* error reporter +*/ +struct flb_aws_error_reporter { + flb_sds_t file_path; + int ttl; + int file_size; + struct mk_list messages; + int max_size; +}; + +/* +* error message which saved in memory +*/ +struct flb_error_message { + time_t timestamp; + flb_sds_t data; + size_t len; + struct mk_list _head; +}; + +/* +* create aws error reporter +*/ +struct flb_aws_error_reporter *flb_aws_error_reporter_create(); + +/* +* error reporter write error log to file +*/ +int flb_aws_error_reporter_write(struct flb_aws_error_reporter *error_reporter, + char *msg); + +/* +* clean up the expired error message inside error log local file +*/ +void flb_aws_error_reporter_clean(struct flb_aws_error_reporter *error_reporter); + +/* +* clean up error reporter resource when fluent bit shutdown +*/ +void flb_aws_error_reporter_destroy(struct flb_aws_error_reporter *error_reporter); + +/* +* function used to tell if error reporting feature is enabled or not +*/ +int is_error_reporting_enabled(); + +#endif +#endif /* FLB_HAVE_AWS_ERROR_REPORTER */
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/aws/flb_aws_imds.h b/fluent-bit/include/fluent-bit/aws/flb_aws_imds.h new file mode 100644 index 00000000..065603da --- /dev/null +++ b/fluent-bit/include/fluent-bit/aws/flb_aws_imds.h @@ -0,0 +1,115 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_AWS_IMDS +#define FLB_AWS_IMDS + +#define FLB_AWS_IMDS_HOST "169.254.169.254" +#define FLB_AWS_IMDS_HOST_LEN 15 +#define FLB_AWS_IMDS_PORT 80 +#define FLB_AWS_IMDS_TIMEOUT 1 /* 1 second */ + +#define FLB_AWS_IMDS_VERSION_EVALUATE 0 +#define FLB_AWS_IMDS_VERSION_1 1 +#define FLB_AWS_IMDS_VERSION_2 2 + +/* The following metadata paths can be evaluated with flb_aws_imds_request + * to retrieve specific metadata members */ +#define FLB_AWS_IMDS_INSTANCE_ID_PATH "/latest/meta-data/instance-id/" +#define FLB_AWS_IMDS_AZ_PATH "/latest/meta-data/placement/availability-zone/" +#define FLB_AWS_IMDS_INSTANCE_TYPE_PATH "/latest/meta-data/instance-type/" +#define FLB_AWS_IMDS_PRIVATE_IP_PATH "/latest/meta-data/local-ipv4/" +#define FLB_AWS_IMDS_VPC_ID_PATH_PREFIX "/latest/meta-data/network/interfaces/macs/" +#define FLB_AWS_IMDS_AMI_ID_PATH "/latest/meta-data/ami-id/" +#define FLB_AWS_IMDS_ACCOUNT_ID_PATH "/latest/dynamic/instance-identity/document/" +#define FLB_AWS_IMDS_HOSTNAME_PATH "/latest/meta-data/hostname/" +#define FLB_AWS_IMDS_MAC_PATH "/latest/meta-data/mac/" +#define FLB_AWS_IMDS_INSTANCE_TAG "/latest/meta-data/tags/instance" + +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_sds.h> + +/* IMDS config values */ +struct flb_aws_imds_config { + int use_imds_version; // FLB_AWS_IMDS_VERSION_EVALUATE for automatic detection +}; + +/* Default config values */ +extern const struct flb_aws_imds_config flb_aws_imds_config_default; + +/* Metadata service context struct */ +struct flb_aws_imds { + /* AWS Client to perform mockable requests to IMDS */ + struct flb_aws_client *ec2_imds_client; + + /* IMDSv2 requires a token which must be present in metadata requests */ + flb_sds_t imds_v2_token; + size_t imds_v2_token_len; + + /* + * Plugin can use EC2 metadata v1 or v2; default is FLB_AWS_IMDS_VERSION_EVALUATE + * which is evaluated to FLB_AWS_IMDS_VERSION_1 or FLB_AWS_IMDS_VERSION_2 when + * the IMDS is used. + */ + int imds_version; +}; + +/* + * Create IMDS context + * Returns NULL on error + * Note: Setting the FLB_IO_ASYNC flag is the job of the client. + * Flag Set Example: flags &= ~(FLB_IO_ASYNC) + */ +struct flb_aws_imds *flb_aws_imds_create(const struct flb_aws_imds_config *imds_config, + struct flb_aws_client *ec2_imds_client); + +/* + * Destroy IMDS context + * The client is responsable for destroying + * the "ec2_imds_client" struct. + */ +void flb_aws_imds_destroy(struct flb_aws_imds *ctx); + +/* + * Get IMDS metadata. + * Sets flb_sds_t metadata string to the value found at IMDS' metadata_path. + * Returns -1 on error, 0 on success. + */ +int flb_aws_imds_request(struct flb_aws_imds *ctx, const char *metadata_path, + flb_sds_t *metadata, size_t *metadata_len); + +/* + * Get IMDS metadata by key + * Expects metadata to be in a json object format. + * Sets flb_sds_t metadata string to the value associated with provided key. + * Sets flb_sds_t metadata string to "NULL" if key not found. + * Sets flb_sds_t metadata string to the full metadata value if key is NULL. + * Returns -1 on error, 0 on success. + */ +int flb_aws_imds_request_by_key(struct flb_aws_imds *ctx, const char *metadata_path, + flb_sds_t *metadata, size_t *metadata_len, char *key); + +/* + * Get VPC id from EC2 IMDS. Requires multiple IMDS requests. + * Returns sds string encoding vpc_id. + * Note: Modified from AWS filter, not retested + */ +flb_sds_t flb_aws_imds_get_vpc_id(struct flb_aws_imds *ctx); + +#endif diff --git a/fluent-bit/include/fluent-bit/conf/flb_static_conf.h.in b/fluent-bit/include/fluent-bit/conf/flb_static_conf.h.in new file mode 100644 index 00000000..ce8547bf --- /dev/null +++ b/fluent-bit/include/fluent-bit/conf/flb_static_conf.h.in @@ -0,0 +1,30 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_STATIC_CONF_H +#define FLB_STATIC_CONF_H + +@FLB_STATIC_CONF_INC@ + +static unsigned char *flb_config_files[@FLB_STATIC_CONF_ARR_SIZE@][2] = { + @FLB_STATIC_CONF_ARR@ +}; + +static unsigned int flb_config_files_size = @FLB_STATIC_CONF_ARR_SIZE@; +#endif diff --git a/fluent-bit/include/fluent-bit/config_format/flb_cf.h b/fluent-bit/include/fluent-bit/config_format/flb_cf.h new file mode 100644 index 00000000..752e333b --- /dev/null +++ b/fluent-bit/include/fluent-bit/config_format/flb_cf.h @@ -0,0 +1,177 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_MAIN_H +#define FLB_CONFIG_FORMAT_MAIN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <monkey/mk_core.h> + +#include <cfl/cfl.h> +#include <cfl/cfl_sds.h> +#include <cfl/cfl_array.h> +#include <cfl/cfl_list.h> +#include <cfl/cfl_variant.h> + +#define FLB_CF_ERROR_SERVICE_EXISTS "SERVICE definition already exists" +#define FLB_CF_ERROR_META_CHAR "invalid first meta character: '@' expected" +#define FLB_CF_ERROR_KV_INVALID_KEY "invalid key content" +#define FLB_CF_ERROR_KV_INVALID_VAL "invalid value content" + +/* manipulate error state for the context */ +#define flb_cf_error_set(cf, err) cf->error_str = err +#define flb_cf_error_get(cf) cf->error_str +#define flb_cr_error_reset(cf) cf->error_str = "" + +/* meta commands: handled as key value pairs */ +#define flb_cf_meta flb_kv + +enum cf_file_format { + FLB_CF_FLUENTBIT, +#ifdef FLB_HAVE_LIBYAML + FLB_CF_YAML +#endif +}; + +#define FLB_CF_CLASSIC FLB_CF_FLUENTBIT + +enum section_type { + FLB_CF_SERVICE = 0, /* [SERVICE] */ + FLB_CF_PARSER, /* [PARSER] */ + FLB_CF_MULTILINE_PARSER, /* [MULTILINE_PARSER] */ + FLB_CF_CUSTOM, /* [CUSTOM] */ + FLB_CF_INPUT, /* [INPUT] */ + FLB_CF_FILTER, /* [FILTER] */ + FLB_CF_OUTPUT, /* [OUTPUT] */ + FLB_CF_OTHER, /* any other section.. */ +}; + +struct flb_cf_group { + flb_sds_t name; /* group name */ + struct cfl_kvlist *properties; /* key value properties */ + struct mk_list _head; /* link to struct flb_cf_section->groups */ +}; + +struct flb_cf_section { + int type; + flb_sds_t name; /* name (used for FLB_CF_OTHER type) */ + struct cfl_kvlist *properties; /* key value properties */ + + struct mk_list groups; /* list of groups */ + + struct mk_list _head; /* link to struct flb_cf->sections */ + struct mk_list _head_section; /* link to section type, e.g: inputs, filters.. */ +}; + +struct flb_cf { + /* origin format */ + int format; + + /* global service */ + struct flb_cf_section *service; + + /* config environment variables (env section, availble on YAML) */ + struct mk_list env; + + /* meta commands (used by fluentbit classic mode) */ + struct mk_list metas; + + /* parsers */ + struct mk_list parsers; + struct mk_list multiline_parsers; + + /* custom plugins */ + struct mk_list customs; + + /* pipeline */ + struct mk_list inputs; + struct mk_list filters; + struct mk_list outputs; + + /* others */ + struct mk_list others; + + /* list head for all sections */ + struct mk_list sections; + + /* set the last error found */ + char *error_str; + + + /* a list head entry in case the caller want's to link contexts */ + struct mk_list _head; +}; + + +struct flb_cf *flb_cf_create(); +struct flb_cf *flb_cf_create_from_file(struct flb_cf *cf, char *file); +flb_sds_t flb_cf_key_translate(struct flb_cf *cf, char *key, int len); + +void flb_cf_destroy(struct flb_cf *cf); + +int flb_cf_set_origin_format(struct flb_cf *cf, int format); +void flb_cf_dump(struct flb_cf *cf); + +struct flb_kv *flb_cf_env_property_add(struct flb_cf *cf, + char *k_buf, size_t k_len, + char *v_buf, size_t v_len); + +/* metas */ +struct flb_kv *flb_cf_meta_property_add(struct flb_cf *cf, char *meta, int len); + +#define flb_cf_foreach_meta(cf) \ + + +void flb_cf_meta_destroy(struct flb_cf *cf, struct flb_cf_meta *meta); +void flb_cf_meta_destroy_all(struct flb_cf *cf); + +/* groups */ +struct flb_cf_group *flb_cf_group_create(struct flb_cf *cf, struct flb_cf_section *s, + char *name, int len); +struct flb_cf_group *flb_cf_group_get(struct flb_cf *cf, struct flb_cf_section *s, char *name); +void flb_cf_group_print(struct flb_cf_group *g); + +void flb_cf_group_destroy(struct flb_cf_group *g); + +/* sections */ +struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int len); +struct flb_cf_section *flb_cf_section_get_by_name(struct flb_cf *cf, char *name); +void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s); +void flb_cf_section_destroy_all(struct flb_cf *cf); + +/* properties */ +struct cfl_variant *flb_cf_section_property_add(struct flb_cf *cf, + struct cfl_kvlist *kv_list, + char *k_buf, size_t k_len, + char *v_buf, size_t v_len); + +struct cfl_array *flb_cf_section_property_add_list(struct flb_cf *cf, + struct cfl_kvlist *kv_list, + char *k_buf, size_t k_len); + +struct cfl_variant *flb_cf_section_property_get(struct flb_cf *cf, + struct flb_cf_section *s, + char *key); + +char *flb_cf_section_property_get_string(struct flb_cf *cf, + struct flb_cf_section *s, + char *key); + +#endif diff --git a/fluent-bit/include/fluent-bit/config_format/flb_cf_fluentbit.h b/fluent-bit/include/fluent-bit/config_format/flb_cf_fluentbit.h new file mode 100644 index 00000000..1b9691ac --- /dev/null +++ b/fluent-bit/include/fluent-bit/config_format/flb_cf_fluentbit.h @@ -0,0 +1,26 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_FLUENTBIT_H +#define FLB_CONFIG_FORMAT_FLUENTBIT_H + +struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, + char *file_path, char *buf, size_t size); + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/config_format/flb_cf_yaml.h b/fluent-bit/include/fluent-bit/config_format/flb_cf_yaml.h new file mode 100644 index 00000000..3babb138 --- /dev/null +++ b/fluent-bit/include/fluent-bit/config_format/flb_cf_yaml.h @@ -0,0 +1,25 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_YAML_H +#define FLB_CONFIG_FORMAT_YAML_H + +struct flb_cf *flb_cf_yaml_create(struct flb_cf *cf, char *file_path, + char *buf, size_t size); +#endif diff --git a/fluent-bit/include/fluent-bit/flb_api.h b/fluent-bit/include/fluent-bit/flb_api.h new file mode 100644 index 00000000..e1fce20d --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_api.h @@ -0,0 +1,43 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_API_H +#define FLB_API_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_output.h> + +struct flb_api { + const char *(*output_get_property) (const char *, struct flb_output_instance *); + const char *(*input_get_property) (const char *, struct flb_input_instance *); + + void *(*output_get_cmt_instance) (struct flb_output_instance *); + void *(*input_get_cmt_instance) (struct flb_input_instance *); + + void (*log_print) (int, const char*, int, const char*, ...); + int (*input_log_check) (struct flb_input_instance *, int); + int (*output_log_check) (struct flb_output_instance *, int); +}; + +#ifdef FLB_CORE +struct flb_api *flb_api_create(); +void flb_api_destroy(struct flb_api *api); +#endif + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_avro.h b/fluent-bit/include/fluent-bit/flb_avro.h new file mode 100644 index 00000000..db96cf5e --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_avro.h @@ -0,0 +1,44 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_AVRO_H +#define FLB_AVRO_H + +#include <fluent-bit/flb_sds.h> +#include <msgpack.h> +#include <avro.h> + +#include <stdlib.h> + +#define MEMORY_POOL_SUCCESS 1 +#define MEMORY_POOL_ERROR 0 +#define MEMORY_POOL_MINIMUM_SIZE sizeof(void *) + +struct flb_avro_fields { + flb_sds_t schema_id; + flb_sds_t schema_str; +}; + +void *flb_avro_allocator(void *ud, void *ptr, size_t osize, size_t nsize); +avro_value_iface_t *flb_avro_init(avro_value_t *aobject, char *json, size_t json_len, avro_schema_t *aschema); +int flb_msgpack_to_avro(avro_value_t *val, msgpack_object *o); +bool flb_msgpack_raw_to_avro_sds(const void *in_buf, size_t in_size, struct flb_avro_fields *ctx, char *out_buff, size_t *out_size); +int msgpack2avro(avro_value_t *val, msgpack_object *o); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_aws_credentials.h b/fluent-bit/include/fluent-bit/flb_aws_credentials.h new file mode 100644 index 00000000..cba1d680 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_aws_credentials.h @@ -0,0 +1,354 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef FLB_HAVE_AWS + +#ifndef FLB_AWS_CREDENTIALS_H +#define FLB_AWS_CREDENTIALS_H + +#include <fluent-bit/flb_aws_util.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_io.h> +#include <fluent-bit/flb_sds.h> +#include <monkey/mk_core.h> + +/* Refresh creds if they will expire in 1 min or less */ +#define FLB_AWS_REFRESH_WINDOW 60 + +/* 5 second timeout for credential related http requests */ +#define FLB_AWS_CREDENTIAL_NET_TIMEOUT 5 + +/* + * A structure that wraps the sensitive data needed to sign an AWS request + */ +struct flb_aws_credentials { + flb_sds_t access_key_id; + flb_sds_t secret_access_key; + flb_sds_t session_token; +}; + +/* defined below but declared here for the function declarations */ +struct flb_aws_provider; + +/* + * Get credentials using the provider. + * Client is in charge of freeing the returned credentials struct. + * Returns NULL if credentials could not be obtained. + */ +typedef struct flb_aws_credentials*(flb_aws_provider_get_credentials_fn) + (struct flb_aws_provider *provider); + + +/* + * "Initializes the provider". Used in the standard chain to determine which + * provider is valid in the current environment. Init is similar to refresh, + * except all log messages are printed as debug (so that the user does not get + * confusing error messages when 'testing' a provider to see if its available). + */ +typedef int(flb_aws_provider_init_fn)(struct flb_aws_provider *provider); + +/* + * Force a refesh of cached credentials. If client code receives a response + * from AWS indicating that the credentials are expired or invalid, + * it can call this method and retry. + * Returns 0 if the refresh was successful. + */ +typedef int(flb_aws_provider_refresh_fn)(struct flb_aws_provider *provider); + +/* + * Clean up the underlying provider implementation. + * Called by flb_aws_provider_destroy. + */ +typedef void(flb_aws_provider_destroy_fn)(struct flb_aws_provider *provider); + +/* + * Set provider to 'sync' mode; all network IO operations will be performed + * synchronously. This must be set if the provider is called when co-routines + * are not available (ex: during plugin initialization). + */ +typedef void(flb_aws_provider_sync_fn)(struct flb_aws_provider *provider); + +/* + * Set provider to 'async' mode; all network IO operations will be performed + * asynchronously. + * + * All providers are created in 'async' mode by default. + */ +typedef void(flb_aws_provider_async_fn)(struct flb_aws_provider *provider); + +/* + * Call flb_output_upstream_set() on all upstreams created + * by this provider and all sub-providers. + */ +typedef void(flb_aws_provider_upstream_set_fn)(struct flb_aws_provider *provider, + struct flb_output_instance *ins); + +/* + * This structure is a virtual table for the functions implemented by each + * provider + */ +struct flb_aws_provider_vtable { + flb_aws_provider_get_credentials_fn *get_credentials; + flb_aws_provider_init_fn *init; + flb_aws_provider_refresh_fn *refresh; + flb_aws_provider_destroy_fn *destroy; + flb_aws_provider_sync_fn *sync; + flb_aws_provider_async_fn *async; + flb_aws_provider_upstream_set_fn *upstream_set; +}; + +/* + * A generic structure to represent all providers. + */ +struct flb_aws_provider { + /* + * Fluent Bit now has multi-threads/workers, need to a mutex to protect cred provider. + * When a refresh is needed, only one co-routine should refresh. + * When one thread refreshes, the cached creds are freed and reset, there could be a double + * free without a lock. + * We use trylock to prevent deadlock. + */ + pthread_mutex_t lock; + + struct flb_aws_provider_vtable *provider_vtable; + + void *implementation; + + /* Standard credentials chain is a list of providers */ + struct mk_list _head; + + /* Provider managed dependencies; to delete on destroy */ + struct flb_aws_provider *base_aws_provider; + struct flb_tls *cred_tls; /* tls instances can't be re-used; aws provider requires + a separate one */ + struct flb_tls *sts_tls; /* one for the standard chain provider, one for sts + assume role */ +}; + +/* + * Function to free memory used by an aws_credentials structure + */ +void flb_aws_credentials_destroy(struct flb_aws_credentials *creds); + +/* + * Function to free memory used by an flb_aws_provider structure + */ +void flb_aws_provider_destroy(struct flb_aws_provider *provider); + +/* + * The standard chain provider; client code should use this provider by default + */ +struct flb_aws_provider *flb_standard_chain_provider_create(struct flb_config + *config, + struct flb_tls *tls, + char *region, + char *sts_endpoint, + char *proxy, + struct + flb_aws_client_generator + *generator, + char *profile); + +/* Provide base configuration options for managed chain */ +#define FLB_AWS_CREDENTIAL_BASE_CONFIG_MAP(prefix) \ + { \ + FLB_CONFIG_MAP_STR, prefix "region", NULL, \ + 0, FLB_FALSE, 0, \ + "AWS region of your service" \ + }, \ + { \ + FLB_CONFIG_MAP_STR, prefix "sts_endpoint", NULL, \ + 0, FLB_FALSE, 0, \ + "Custom endpoint for the AWS STS API, used with the `" prefix "role_arn` option" \ + }, \ + { \ + FLB_CONFIG_MAP_STR, prefix "role_arn", NULL, \ + 0, FLB_FALSE, 0, \ + "ARN of an IAM role to assume (ex. for cross account access)" \ + }, \ + { \ + FLB_CONFIG_MAP_STR, prefix "external_id", NULL, \ + 0, FLB_FALSE, 0, \ + "Specify an external ID for the STS API, can be used with the `" prefix \ + "role_arn` parameter if your role requires an external ID." \ + }, \ + { \ + FLB_CONFIG_MAP_STR, prefix "profile", NULL, \ + 0, FLB_FALSE, 0, \ + "AWS Profile name. AWS Profiles can be configured with AWS CLI and are usually" \ + "stored in $HOME/.aws/ directory." \ + } +/* + * Managed chain provider; Creates and manages removal of dependancies for an instance + */ +struct flb_aws_provider *flb_managed_chain_provider_create(struct flb_output_instance + *ins, + struct flb_config + *config, + char *config_key_prefix, + char *proxy, + struct + flb_aws_client_generator + *generator); + +/* + * A provider that uses OIDC tokens provided by kubernetes to obtain + * AWS credentials. + * + * The AWS SDKs have defined a spec for an OIDC provider that obtains tokens + * from environment variables or the shared config file. + * This provider only contains the functionality needed for EKS- obtaining the + * location of the OIDC token from an environment variable. + */ +struct flb_aws_provider *flb_eks_provider_create(struct flb_config *config, + struct flb_tls *tls, + char *region, + char *sts_endpoint, + char *proxy, + struct + flb_aws_client_generator + *generator); + + +/* + * STS Assume Role Provider. + */ +struct flb_aws_provider *flb_sts_provider_create(struct flb_config *config, + struct flb_tls *tls, + struct flb_aws_provider + *base_provider, + char *external_id, + char *role_arn, + char *session_name, + char *region, + char *sts_endpoint, + char *proxy, + struct + flb_aws_client_generator + *generator); + +/* + * Standard environment variables + */ +struct flb_aws_provider *flb_aws_env_provider_create(); + +/* + * New http provider - retrieve credentials from a local http server. + * Equivalent to: + * https://github.com/aws/aws-sdk-go/tree/master/aws/credentials/endpointcreds + * + * Calling flb_aws_provider_destroy on this provider frees the memory + * used by host and path. + */ +struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, + flb_sds_t host, + flb_sds_t path, + struct + flb_aws_client_generator + *generator); + +/* + * ECS Provider + * The ECS Provider is just a wrapper around the HTTP Provider + * with the ECS credentials endpoint. + */ +struct flb_aws_provider *flb_ecs_provider_create(struct flb_config *config, + struct + flb_aws_client_generator + *generator); + +/* + * EC2 IMDS Provider + */ +struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config, + struct + flb_aws_client_generator + *generator); + +/* + * New AWS Profile provider, reads from the shared credentials file + */ +struct flb_aws_provider *flb_profile_provider_create(char* profile); + +/* + * Helper functions + */ + +time_t flb_aws_cred_expiration(const char* timestamp); + +struct flb_aws_credentials *flb_parse_sts_resp(char *response, + time_t *expiration); +flb_sds_t flb_sts_uri(char *action, char *role_arn, char *session_name, + char *external_id, char *identity_token); +char *flb_sts_session_name(); + +struct flb_aws_credentials *flb_parse_http_credentials(char *response, + size_t response_len, + time_t *expiration); + +struct flb_aws_credentials *flb_parse_json_credentials(char *response, + size_t response_len, + char *session_token_field, + time_t *expiration); + +#ifdef FLB_HAVE_AWS_CREDENTIAL_PROCESS + +/* + * Parses the input string, which is assumed to be the credential_process + * from the config file, into a sequence of tokens. + * Returns the array of tokens on success, and NULL on failure. + * The array of tokens will be terminated by NULL for use with `execvp`. + * The caller is responsible for calling `flb_free` on the return value. + * Note that this function modifies the input string. + */ +char** parse_credential_process(char* input); + +/* + * Executes the given credential_process, which is assumed to have come + * from the config file, and parses its result into *creds and *expiration. + * Returns 0 on success and < 0 on failure. + * + * If it succeeds, *creds and *expiration will be set appropriately, and the + * caller is responsible for calling `flb_aws_credentials_destroy(*creds)`. + * If the credentials do not expire, then *expiration will be 0. + * + * If it fails, then *creds will be NULL. + */ +int exec_credential_process(char* process, struct flb_aws_credentials** creds, + time_t* expiration); + +#endif /* FLB_HAVE_AWS_CREDENTIAL_PROCESS */ + +/* + * Fluent Bit is single-threaded but asynchonous. Only one co-routine will + * be running at a time, and they only pause/resume for IO. + * + * Thus, while synchronization is needed (to prevent multiple co-routines + * from duplicating effort and performing the same work), it can be obtained + * using a simple integer flag on the provider. + */ + +/* Like a traditional try lock- it does not block if the lock is not obtained */ +int try_lock_provider(struct flb_aws_provider *provider); + +void unlock_provider(struct flb_aws_provider *provider); + + +#endif +#endif /* FLB_HAVE_AWS */ diff --git a/fluent-bit/include/fluent-bit/flb_aws_util.h b/fluent-bit/include/fluent-bit/flb_aws_util.h new file mode 100644 index 00000000..08507dcc --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_aws_util.h @@ -0,0 +1,190 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef FLB_HAVE_AWS + +#ifndef FLB_AWS_UTIL_H + +#include <fluent-bit/flb_output.h> +#include <fluent-bit/flb_time.h> + +#define FLB_AWS_UTIL_H + +#define FLB_AWS_CREDENTIAL_REFRESH_LIMIT 60 + +/* + * The AWS HTTP Client is a wrapper around the Fluent Bit's http library. + * It handles tasks which are common to all AWS API requests (retries, + * error processing, etc). + * It is also easily mockable in unit tests. + */ + + struct flb_aws_client; + + struct flb_aws_header { + char *key; + size_t key_len; + char *val; + size_t val_len; + }; + +typedef struct flb_http_client *(flb_aws_client_request_fn) + (struct flb_aws_client *aws_client, + int method, const char *uri, + const char *body, size_t body_len, + struct flb_aws_header *dynamic_headers, + size_t dynamic_headers_len); + +/* TODO: Eventually will need to add a way to call flb_http_buffer_size */ + +/* + * Virtual table for aws http client behavior. + * This makes the client's functionality mockable in unit tests. + */ +struct flb_aws_client_vtable { + flb_aws_client_request_fn *request; +}; + +struct flb_aws_client { + struct flb_aws_client_vtable *client_vtable; + + /* Name to identify this client: used in log messages and tests */ + char *name; + + /* Sigv4 */ + int has_auth; + int s3_mode; + struct flb_aws_provider *provider; + char *region; + char *service; + + struct flb_upstream *upstream; + + char *host; + int port; + char *proxy; + int flags; + flb_sds_t extra_user_agent; + + /* + * Additional headers which will be added to all requests. + * The AWS client will add auth headers, content length, + * and user agent. + */ + struct flb_aws_header *static_headers; + size_t static_headers_len; + + /* Are requests to AWS services retried? */ + int retry_requests; + + /* + * If an API responds with auth error, we refresh creds and retry. + * For safety, credential refresh can only happen once per + * FLB_AWS_CREDENTIAL_REFRESH_LIMIT. + */ + time_t refresh_limit; + + /* Send all log messages as debug; used in AWS Cred Providers on init */ + int debug_only; +}; + +/* + * Frees the aws_client, the internal flb_http_client, error_code, + * and flb_upstream. + * Caller code must free any other memory. + * (Why? - Because all other memory may be static.) + */ +void flb_aws_client_destroy(struct flb_aws_client *aws_client); + +typedef struct flb_aws_client*(flb_aws_client_create_fn)(); + +/* + * HTTP Client Generator creates a new client structure and sets the vtable. + * Unit tests can implement a custom flb_aws_client_generator which returns a mock client. + * This structure is a virtual table. + * Client code should not free it. + */ +struct flb_aws_client_generator { + flb_aws_client_create_fn *create; +}; + +/* Remove protocol from endpoint */ +char *removeProtocol (char *endpoint, char *protocol); + +/* Get the flb_aws_client_generator */ +struct flb_aws_client_generator *flb_aws_client_generator(); + +/* + * Format an AWS regional API endpoint + */ +char *flb_aws_endpoint(char* service, char* region); + +/* Parses AWS XML API Error responses and returns the value of the <code> tag */ +flb_sds_t flb_aws_xml_error(char *response, size_t response_len); + +/* + * Parses an AWS JSON API error type returned by a request. + */ +flb_sds_t flb_aws_error(char *response, size_t response_len); + +/* + * Similar to 'flb_aws_error', except it prints the JSON error type and message + * to the user in a error log. + * 'api' is the name of the API that was called; this is used in the error log. + */ +void flb_aws_print_error(char *response, size_t response_len, + char *api, struct flb_output_instance *ins); + +/* Similar to 'flb_aws_print_error', but for APIs that return XML */ +void flb_aws_print_xml_error(char *response, size_t response_len, + char *api, struct flb_output_instance *ins); + +/* + * Parses the JSON and gets the value for 'key' + */ +flb_sds_t flb_json_get_val(char *response, size_t response_len, char *key); + +/* + * Parses an XML document and returns the value of the given tag + * Param `tag` should include angle brackets; ex "<code>" + * And param `end` should include end brackets: "</code>" + */ +flb_sds_t flb_aws_xml_get_val(char *response, size_t response_len, char *tag, char *tag_end); + +/* + * Checks if a response contains an AWS Auth error + */ +int flb_aws_is_auth_error(char *payload, size_t payload_size); + +int flb_read_file(const char *path, char **out_buf, size_t *out_size); + +//* Constructs S3 object key as per the format. */ +flb_sds_t flb_get_s3_key(const char *format, time_t time, const char *tag, + char *tag_delimiter, uint64_t seq_index); + +/* + * This function is an extension to strftime which can support milliseconds with %3N, + * support nanoseconds with %9N or %L. The return value is the length of formatted + * time string. + */ +size_t flb_aws_strftime_precision(char **out_buf, const char *time_format, + struct flb_time *tms); + +#endif +#endif /* FLB_HAVE_AWS */ diff --git a/fluent-bit/include/fluent-bit/flb_base64.h b/fluent-bit/include/fluent-bit/flb_base64.h new file mode 100644 index 00000000..2235307d --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_base64.h @@ -0,0 +1,106 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * \file base64.h + * + * \brief RFC 1521 base64 encoding/decoding + */ + +/* This code is based on base64.h from the mbedtls-2.25.0 Library distribution, + * as originally written by Paul Bakker, et al., and forked by the Fluent Bit + * project to provide performant base64 encoding and decoding routines. + * The 2.25.0 implementation is included rather than 2.26.0+ implementation due + * to performance degradation introduced in 2.26.0. + * + * Method and variable names are changed by the Fluent Bit authors to maintain + * consistency with the Fluent Bit project. + * The self test section of the code was removed by the Fluent Bit authors. + * Other minor changes are made by the Fluent Bit authors. + * + * The original source file base64.h is copyright and licensed as follows; + * + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_BASE64_H +#define FLB_BASE64_H + +#include <stddef.h> + +#define FLB_BASE64_ERR_BUFFER_TOO_SMALL -0x002A /**< Output buffer too small. */ +#define FLB_BASE64_ERR_INVALID_CHARACTER -0x002C /**< Invalid character in input. */ + +/** + * \brief Encode a buffer into base64 format + * + * \param dst destination buffer + * \param dlen size of the destination buffer + * \param olen number of bytes written + * \param src source buffer + * \param slen amount of data to be encoded + * + * \return 0 if successful, or FLB_BASE64_ERR_BUFFER_TOO_SMALL. + * *olen is always updated to reflect the amount + * of data that has (or would have) been written. + * If that length cannot be represented, then no data is + * written to the buffer and *olen is set to the maximum + * length representable as a size_t. + * + * \note Call this function with dlen = 0 to obtain the + * required buffer size in *olen + */ +int flb_base64_encode( unsigned char *dst, size_t dlen, size_t *olen, + const unsigned char *src, size_t slen ); + +/** + * \brief Decode a base64-formatted buffer + * + * \param dst destination buffer (can be NULL for checking size) + * \param dlen size of the destination buffer + * \param olen number of bytes written + * \param src source buffer + * \param slen amount of data to be decoded + * + * \return 0 if successful, FLB_BASE64_ERR_BUFFER_TOO_SMALL, or + * FLB_BASE64_ERR_INVALID_CHARACTER if the input data is + * not correct. *olen is always updated to reflect the amount + * of data that has (or would have) been written. + * + * \note Call this function with *dst = NULL or dlen = 0 to obtain + * the required buffer size in *olen + */ +int flb_base64_decode( unsigned char *dst, size_t dlen, size_t *olen, + const unsigned char *src, size_t slen ); + +#endif /* base64.h */ diff --git a/fluent-bit/include/fluent-bit/flb_bits.h b/fluent-bit/include/fluent-bit/flb_bits.h new file mode 100644 index 00000000..0dfd70a5 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_bits.h @@ -0,0 +1,31 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_BITS_H +#define FLB_BITS_H + +#define FLB_BITS_U64_SET(a, b) ((uint64_t) a << 32 | b) +#define FLB_BITS_U64_HIGH(val) ((uint64_t) val >> 32) +#define FLB_BITS_U64_LOW(val) ((uint64_t) val & 0xffffffff) +#define FLB_BITS_CLEAR(val, n) (val & ~(1 << n)) +#define FLB_BIT_MASK(__TYPE__, __ONE_COUNT__) \ + ((__TYPE__) (-((__ONE_COUNT__) != 0))) \ + & (((__TYPE__) -1) >> ((sizeof(__TYPE__) * CHAR_BIT) - (__ONE_COUNT__))) + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_bucket_queue.h b/fluent-bit/include/fluent-bit/flb_bucket_queue.h new file mode 100644 index 00000000..3016454a --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_bucket_queue.h @@ -0,0 +1,140 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Note: This implementation can handle priority item removal via + * flb_bucket_queue_delete_min(bucket_queue) and direct item removal + * via mk_list_del(&item) + */ + +#ifndef FLB_BUCKET_QUEUE_H_ +#define FLB_BUCKET_QUEUE_H_ + +#include <stddef.h> +#include <fluent-bit/flb_mem.h> +#include <monkey/mk_core/mk_list.h> + + +struct flb_bucket_queue +{ + struct mk_list *buckets; + size_t n_buckets; + struct mk_list *top; + size_t n_items; +}; + +static inline struct flb_bucket_queue *flb_bucket_queue_create(size_t priorities) +{ + size_t i; + struct flb_bucket_queue *bucket_queue; + + bucket_queue = (struct flb_bucket_queue *) + flb_malloc(sizeof(struct flb_bucket_queue)); + if (!bucket_queue) { + return NULL; + } + bucket_queue->buckets = (struct mk_list *) + flb_malloc(sizeof(struct mk_list) *priorities); + if (!bucket_queue->buckets) { + flb_free(bucket_queue); + return NULL; + } + for (i = 0; i < priorities; ++i) { + mk_list_init(&bucket_queue->buckets[i]); + } + bucket_queue->n_buckets = priorities; + bucket_queue->top = (bucket_queue->buckets + bucket_queue->n_buckets); /* one past the last element */ + bucket_queue->n_items = 0; + return bucket_queue; +} + +static inline int flb_bucket_queue_is_empty(struct flb_bucket_queue *bucket_queue) +{ + return bucket_queue->top == (bucket_queue->buckets + bucket_queue->n_buckets); +} + +static inline void flb_bucket_queue_seek(struct flb_bucket_queue *bucket_queue) { + while (!flb_bucket_queue_is_empty(bucket_queue) + && (mk_list_is_empty(bucket_queue->top) == 0)) { + ++bucket_queue->top; + } +} + +static inline int flb_bucket_queue_add(struct flb_bucket_queue *bucket_queue, + struct mk_list *item, size_t priority) +{ + if (priority >= bucket_queue->n_buckets) { + /* flb_error("Error: attempting to add item of priority %zu to bucket_queue out " + "of priority range", priority); */ + return -1; + } + flb_bucket_queue_seek(bucket_queue); + mk_list_add(item, &bucket_queue->buckets[priority]); + if (&bucket_queue->buckets[priority] < bucket_queue->top) { + bucket_queue->top = &bucket_queue->buckets[priority]; + } + ++bucket_queue->n_items; + return 0; +} + +/* fifo based on priority */ +static inline struct mk_list *flb_bucket_queue_find_min(struct flb_bucket_queue *bucket_queue) +{ + flb_bucket_queue_seek(bucket_queue); + if (flb_bucket_queue_is_empty(bucket_queue)) { + return NULL; + } + return bucket_queue->top->next; +} + +static inline void flb_bucket_queue_delete_min(struct flb_bucket_queue *bucket_queue) +{ + flb_bucket_queue_seek(bucket_queue); + if (flb_bucket_queue_is_empty(bucket_queue)) { + return; + } + mk_list_del(bucket_queue->top->next); + flb_bucket_queue_seek(bucket_queue); /* this line can be removed. Debugging is harder */ + --bucket_queue->n_items; +} + +static inline struct mk_list *flb_bucket_queue_pop_min(struct flb_bucket_queue *bucket_queue) +{ + struct mk_list *item; + item = flb_bucket_queue_find_min(bucket_queue); + flb_bucket_queue_delete_min(bucket_queue); + return item; +} + +static inline int flb_bucket_queue_destroy( + struct flb_bucket_queue *bucket_queue) +{ + flb_bucket_queue_seek(bucket_queue); + if (!flb_bucket_queue_is_empty(bucket_queue)) { + /* flb_error("Error: attempting to destroy non empty bucket_queue. Remove all " + "items first."); */ + return -1; + } + flb_free(bucket_queue->buckets); + flb_free(bucket_queue); + return 0; +} + +#endif /* !FLB_BUCKET_QUEUE_H_ */ diff --git a/fluent-bit/include/fluent-bit/flb_byteswap.h b/fluent-bit/include/fluent-bit/flb_byteswap.h new file mode 100644 index 00000000..16b4b5cd --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_byteswap.h @@ -0,0 +1,105 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_BYTESWAP_H +#define FLB_BYTESWAP_H + +#include <stdint.h> + +#if defined(FLB_HAVE_WIN32_BYTESWAP) +#include <stdlib.h> +#elif defined(FLB_HAVE_CLANG_BYTESWAP) +#include <intrin.h> +#elif defined(FLB_HAVE_POSIX_BYTESWAP) +#include <byteswap.h> +#endif + +#if defined(FLB_HAVE_WIN32_BYTESWAP) || \ + defined(FLB_HAVE_CLANG_BYTESWAP) +#define FLB_BSWAP_16(value) _byteswap_ushort(value) +#define FLB_BSWAP_32(value) _byteswap_ulong(value) +#define FLB_BSWAP_64(value) _byteswap_uint64(value) + +#elif defined(FLB_HAVE_POSIX_BYTESWAP) +#define FLB_BSWAP_16(value) bswap_16(value) +#define FLB_BSWAP_32(value) bswap_32(value) +#define FLB_BSWAP_64(value) bswap_64(value) + +#else + +union flb_bswap_value_internal { + char raw[8]; + uint16_t word; + uint32_t dword; + uint64_t qword; +}; + +static inline uint16_t FLB_BSWAP_16(uint16_t value) +{ + union flb_bswap_value_internal output; + union flb_bswap_value_internal input; + + output.word = value; + input.word = value; + + output.raw[0] = input.raw[1]; + output.raw[1] = input.raw[0]; + + return output.word; +} + +static inline uint32_t FLB_BSWAP_32(uint32_t value) +{ + union flb_bswap_value_internal output; + union flb_bswap_value_internal input; + + output.dword = value; + input.dword = value; + + output.raw[0] = input.raw[3]; + output.raw[1] = input.raw[2]; + output.raw[2] = input.raw[1]; + output.raw[3] = input.raw[0]; + + return output.dword; +} + +static inline uint64_t FLB_BSWAP_64(uint64_t value) +{ + union flb_bswap_value_internal output; + union flb_bswap_value_internal input; + + output.qword = value; + input.qword = value; + + output.raw[0] = input.raw[7]; + output.raw[1] = input.raw[6]; + output.raw[2] = input.raw[5]; + output.raw[3] = input.raw[4]; + output.raw[4] = input.raw[3]; + output.raw[5] = input.raw[2]; + output.raw[6] = input.raw[1]; + output.raw[7] = input.raw[0]; + + return output.qword; +} + +#endif + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_callback.h b/fluent-bit/include/fluent-bit/flb_callback.h new file mode 100644 index 00000000..e76b8fff --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_callback.h @@ -0,0 +1,48 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CALLBACK_H +#define FLB_CALLBACK_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_hash_table.h> +#include <fluent-bit/flb_sds.h> + +struct flb_callback_entry { + flb_sds_t name; + void (*cb)(char *, void *, void *); + struct mk_list _head; +}; + +struct flb_callback { + flb_sds_t name; /* Context name */ + struct flb_hash_table *ht; /* Hash table */ + struct mk_list entries; /* List for callback entries */ + struct flb_config *config; /* Fluent Bit context */ +}; + +struct flb_callback *flb_callback_create(char *name); +void flb_callback_destroy(struct flb_callback *ctx); +int flb_callback_set(struct flb_callback *ctx, char *name, + void (*cb)(char *, void *, void *)); + +int flb_callback_do(struct flb_callback *ctx, char *name, void *p1, void *p2); +int flb_callback_exists(struct flb_callback *ctx, char *name); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_chunk_trace.h b/fluent-bit/include/fluent-bit/flb_chunk_trace.h new file mode 100644 index 00000000..bd2a9501 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_chunk_trace.h @@ -0,0 +1,101 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CHUNK_TRACE_H +#define FLB_CHUNK_TRACE_H + +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_lib.h> + +#include <chunkio/cio_chunk.h> + +/* A record has been received from input */ +#define FLB_CHUNK_TRACE_TYPE_INPUT 1 +/* A record has been filtered */ +#define FLB_CHUNK_TRACE_TYPE_FILTER 2 +/* A trace with the final record before output */ +#define FLB_CHUNK_TRACE_TYPE_PRE_OUTPUT 3 +/* A record has been output */ +#define FLB_CHUNK_TRACE_TYPE_OUTPUT 4 + +#define FLB_CHUNK_TRACE_LIMIT_TIME 1 +#define FLB_CHUNK_TRACE_LIMIT_COUNT 2 + +struct flb_chunk_trace_input_record { + struct flb_time t; + void *input; + char *buf; + size_t buf_size; +}; + +struct flb_chunk_trace_filter_record { + struct flb_time t; + int trace_version; + void *filter; + char *buf; + size_t buf_size; +}; + +struct flb_chunk_trace_limit { + /* set to one of: */ + /* FLB_CHUNK_TRACE_LIMIT_TIME */ + /* FLB_CHUNK_TRACE_LIMIT_COUNT */ + int type; + + /* limit is in seconds */ + int seconds; + /* unix timestamp when time limit started */ + int seconds_started; + + /* limit is a count */ + int count; +}; + +struct flb_chunk_trace_context { + void *input; + void *output; + int trace_count; + struct flb_chunk_trace_limit limit; + flb_sds_t trace_prefix; + int to_destroy; + int chunks; + flb_ctx_t *flb; + struct cio_ctx *cio; +}; + +struct flb_chunk_trace { + struct flb_input_chunk *ic; + struct flb_chunk_trace_context *ctxt; + flb_sds_t trace_id; + int tracer_versions; +}; + +struct flb_chunk_trace_context *flb_chunk_trace_context_new(void *input, const char *output_name, const char *trace_prefix, void *data, struct mk_list *props); +void flb_chunk_trace_context_destroy(void *input); +struct flb_chunk_trace *flb_chunk_trace_new(struct flb_input_chunk *chunk); +void flb_chunk_trace_destroy(struct flb_chunk_trace *); +int flb_chunk_trace_input(struct flb_chunk_trace *trace); +void flb_chunk_trace_do_input(struct flb_input_chunk *trace); +int flb_chunk_trace_pre_output(struct flb_chunk_trace *trace); +int flb_chunk_trace_filter(struct flb_chunk_trace *trace, void *pfilter, struct flb_time *, struct flb_time *, char *buf, size_t buf_size); +void flb_chunk_trace_free(struct flb_chunk_trace *trace); +int flb_chunk_trace_context_set_limit(void *input, int, int); +int flb_chunk_trace_context_hit_limit(void *input); + +#endif // FLB_CHUNK_TRACE_H diff --git a/fluent-bit/include/fluent-bit/flb_compat.h b/fluent-bit/include/fluent-bit/flb_compat.h new file mode 100644 index 00000000..858600d8 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_compat.h @@ -0,0 +1,146 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This file contains compatibility functions and macros for various platforms. + * + * Including this header file should make platforms behave more consistently; + * Add more macros if you find any missing features. + */ + +#ifndef FLB_COMPAT_H +#define FLB_COMPAT_H + +/* + * libmonkey exposes compat macros for <unistd.h>, which some platforms lack, + * so include the header here. + */ +#include <monkey/mk_core.h> + +#ifdef FLB_SYSTEM_WINDOWS +#define WIN32_LEAN_AND_MEAN +#include <winsock2.h> +#include <windows.h> +#include <Wincrypt.h> /* flb_io_tls.c */ + +#include <monkey/mk_core/mk_sleep.h> +#include <fluent-bit/flb_dlfcn_win32.h> + +#define FLB_DIRCHAR '\\' +#define PATH_MAX MAX_PATH +#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +#define S_ISLNK(m) (0) /* Windows doesn't support S_IFLNK */ +#define SHUT_RD SD_RECEIVE +#define SHUT_WR SD_SEND +#define SHUT_RDWR SD_BOTH + +/* monkey exposes a broken vsnprintf macro. Undo it */ +#undef vsnprintf + +/* + * Windows prefer to add an underscore to each POSIX function. + * To suppress compiler warnings, we need these trivial macros. + */ +#define timezone _timezone +#define tzname _tzname +#define strncasecmp _strnicmp +#define timegm _mkgmtime + +static inline int getpagesize(void) +{ + SYSTEM_INFO info; + GetSystemInfo(&info); + return info.dwPageSize; +} + +static inline struct tm *gmtime_r(const time_t *timep, struct tm *result) +{ + if (gmtime_s(result, timep)) { + return NULL; + } + return result; +} + +static inline char *ctime_r(const time_t *timep, char *result) +{ + char *tmp = ctime(timep); + if (tmp == NULL) { + return NULL; + } + return strcpy(result, tmp); +} + +/* + * We can't just define localtime_r here, since mk_core/mk_utils.c is + * exposing a symbol with the same name inadvertently. + */ +static struct tm *flb_localtime_r(time_t *timep, struct tm *result) +{ + if (localtime_s(result, timep)) { + return NULL; + } + return result; +} +#define localtime_r flb_localtime_r + +static inline char* basename(const char *path) +{ + char drive[_MAX_DRIVE]; + char dir[_MAX_DIR]; + char fname[_MAX_FNAME]; + char ext[_MAX_EXT]; + static char buf[_MAX_PATH]; + + _splitpath_s(path, drive, _MAX_DRIVE, dir, _MAX_DIR, + fname, _MAX_FNAME, ext, _MAX_EXT); + + _makepath_s(buf, _MAX_PATH, "", "", fname, ext); + return buf; +} + +static inline char* realpath(char *path, char *buf) +{ + if (buf != NULL) { + return NULL; /* Read BUGS in realpath(3) */ + } + return _fullpath(NULL, path, 0); +} + +static inline int usleep(LONGLONG usec) +{ + // Convert into 100ns unit. + return nanosleep(usec * 10); +} +#else +#include <netdb.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <sys/socket.h> +#include <arpa/inet.h> +#include <libgen.h> +#include <dlfcn.h> + +#define FLB_DIRCHAR '/' +#endif + +#ifdef FLB_HAVE_UNIX_SOCKET +#include <sys/un.h> +#endif + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_compression.h b/fluent-bit/include/fluent-bit/flb_compression.h new file mode 100644 index 00000000..cdf0360a --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_compression.h @@ -0,0 +1,76 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_COMPRESSION_H +#define FLB_COMPRESSION_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_gzip.h> +#include <stdio.h> + +#define FLB_COMPRESSION_ALGORITHM_NONE 0 +#define FLB_COMPRESSION_ALGORITHM_GZIP 1 + +#define FLB_DECOMPRESSOR_STATE_FAILED -1 +#define FLB_DECOMPRESSOR_STATE_EXPECTING_HEADER 0 +#define FLB_DECOMPRESSOR_STATE_EXPECTING_OPTIONAL_HEADERS 1 +#define FLB_DECOMPRESSOR_STATE_EXPECTING_BODY 2 +#define FLB_DECOMPRESSOR_STATE_EXPECTING_FOOTER 3 + +#define FLB_DECOMPRESSOR_FAILURE -1 +#define FLB_DECOMPRESSOR_CORRUPTED_HEADER -2 +#define FLB_DECOMPRESSOR_INVALID_STATE -3 +#define FLB_DECOMPRESSOR_SUCCESS 0 +#define FLB_DECOMPRESSOR_INSUFFICIENT_DATA 0 + +#define FLB_DECOMPRESSION_BUFFER_SIZE (1024 * 1000) + +struct flb_decompression_context { + size_t input_buffer_length; + size_t input_buffer_size; + uint8_t *input_buffer; + uint8_t *read_buffer; + int algorithm; + int state; + + /* Compression backend specific context (opaque) */ + void *inner_context; +}; + +uint8_t *flb_decompression_context_get_append_buffer( + struct flb_decompression_context *context); + +size_t flb_decompression_context_get_available_space( + struct flb_decompression_context *context); + +int flb_decompression_context_resize_buffer( + struct flb_decompression_context *context, size_t new_size); + +struct flb_decompression_context *flb_decompression_context_create( + int algorithm, + size_t input_buffer_size); + +void flb_decompression_context_destroy( + struct flb_decompression_context *context); + +int flb_decompress(struct flb_decompression_context *context, + void *output_buffer, + size_t *output_length); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_config.h b/fluent-bit/include/fluent-bit/flb_config.h new file mode 100644 index 00000000..a5955908 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_config.h @@ -0,0 +1,370 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_H +#define FLB_CONFIG_H + +#include <time.h> + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_pipe.h> +#include <fluent-bit/flb_log.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_task_map.h> + +#include <monkey/mk_core.h> + +#define FLB_CONFIG_FLUSH_SECS 1 +#define FLB_CONFIG_HTTP_LISTEN "0.0.0.0" +#define FLB_CONFIG_HTTP_PORT "2020" +#define HC_ERRORS_COUNT_DEFAULT 5 +#define HC_RETRY_FAILURE_COUNTS_DEFAULT 5 +#define HEALTH_CHECK_PERIOD 60 +#define FLB_CONFIG_DEFAULT_TAG "fluent_bit" + +/* Main struct to hold the configuration of the runtime service */ +struct flb_config { + struct mk_event ch_event; + + int support_mode; /* enterprise support mode ? */ + int is_ingestion_active; /* date ingestion active/allowed */ + int is_shutting_down; /* is the service shutting down ? */ + int is_running; /* service running ? */ + double flush; /* Flush timeout */ + + /* + * Maximum grace time on shutdown. If set to -1, the engine will + * shutdown when all remaining tasks are flushed + */ + int grace; + int grace_count; /* Count of grace shutdown tries */ + flb_pipefd_t flush_fd; /* Timer FD associated to flush */ + int convert_nan_to_null; /* convert null to nan ? */ + + int daemon; /* Run as a daemon ? */ + flb_pipefd_t shutdown_fd; /* Shutdown FD, 5 seconds */ + + int verbose; /* Verbose mode (default OFF) */ + time_t init_time; /* Time when Fluent Bit started */ + + /* Used in library mode */ + pthread_t worker; /* worker tid */ + flb_pipefd_t ch_data[2]; /* pipe to communicate caller with worker */ + flb_pipefd_t ch_manager[2]; /* channel to administrate fluent bit */ + flb_pipefd_t ch_notif[2]; /* channel to receive notifications */ + + flb_pipefd_t ch_self_events[2]; /* channel to recieve thread tasks */ + + /* Channel event loop (just for ch_notif) */ + struct mk_event_loop *ch_evl; + + struct mk_rconf *file; + + /* main configuration */ + struct flb_cf *cf_main; + /* command line configuration (handled by fluent-bit bin) */ + struct flb_cf *cf_opts; + struct mk_list cf_parsers_list; + + flb_sds_t program_name; /* argv[0] */ + + /* + * If a configuration file was used, this variable will contain the + * absolute path for the directory that contains the file. + */ + char *conf_path; + + /* if the configuration come from the file system, store the given path */ + flb_sds_t conf_path_file; + + /* if the external plugins come from the file system, store the given paths from command line */ + struct mk_list external_plugins; + + /* Event */ + struct mk_event event_flush; + struct mk_event event_shutdown; + struct mk_event event_thread_init; /* event to initiate thread in engine */ + + /* Collectors */ + pthread_mutex_t collectors_mutex; + + /* Dynamic (dso) plugins context */ + void *dso_plugins; + + /* Plugins references */ + struct mk_list processor_plugins; + struct mk_list custom_plugins; + struct mk_list in_plugins; + struct mk_list parser_plugins; /* not yet implemented */ + struct mk_list filter_plugins; + struct mk_list out_plugins; + + /* Custom instances */ + struct mk_list customs; + + /* Inputs instances */ + struct mk_list inputs; + + /* Parsers instances */ + struct mk_list parsers; + + /* Multiline core parser definitions */ + struct mk_list multiline_parsers; + + /* Outputs instances */ + struct mk_list outputs; /* list of output plugins */ + + /* Filter instances */ + struct mk_list filters; + + struct mk_event_loop *evl; /* the event loop (mk_core) */ + + struct flb_bucket_queue *evl_bktq; /* bucket queue for evl track event priority */ + + /* Proxies */ + struct mk_list proxies; + + /* Kernel info */ + struct flb_kernel *kernel; + + /* Logging */ + char *log_file; + struct flb_log *log; + + /* Parser Conf */ + char *parsers_file; + + /* Plugins config file */ + char *plugins_file; + + /* Environment */ + void *env; + + /* Working Directory */ + char *workdir; + + /* Exit status code */ + int exit_status_code; + + /* Workers: threads spawn using flb_worker_create() */ + struct mk_list workers; + + /* Metrics exporter */ +#ifdef FLB_HAVE_METRICS + void *metrics; +#endif + + /* + * CMetric lists: a linked list to keep a reference of every + * cmetric context created. + */ + struct mk_list cmetrics; + + /* HTTP Server */ +#ifdef FLB_HAVE_HTTP_SERVER + int http_server; /* HTTP Server running */ + char *http_port; /* HTTP Port / TCP number */ + char *http_listen; /* Interface Address */ + void *http_ctx; /* Monkey HTTP context */ + int health_check; /* health check enable */ + int hc_errors_count; /* health check error counts as unhealthy*/ + int hc_retry_failure_count; /* health check retry failures count as unhealthy*/ + int health_check_period; /* period by second for health status check */ +#endif + + /* + * There are two ways to use proxy in fluent-bit: + * 1. Similar with http and datadog plugin, passing proxy directly to + * flb_http_client and use proxy host and port when creating upstream. + * HTTPS traffic is not supported this way. + * 2. Similar with stackdriver plugin, passing http_proxy in flb_config + * (or by setting HTTP_PROXY env variable). HTTPS is supported this way. But + * proxy shouldn't be passed when calling flb_http_client(). + */ + char *http_proxy; + + /* + * A comma-separated list of host names that shouldn't go through + * any proxy is set in (only an asterisk, * matches all hosts). + * As a convention (https://curl.se/docs/manual.html), this value can be set + * and respected by `NO_PROXY` environment variable when `HTTP_PROXY` is used. + * Example: NO_PROXY="127.0.0.1,localhost,kubernetes.default.svc" + * Note: only `,` is allowed as seperator between URLs. + */ + char *no_proxy; + + /* DNS */ + char *dns_mode; + char *dns_resolver; + int dns_prefer_ipv4; + + /* Chunk I/O Buffering */ + void *cio; + char *storage_path; + void *storage_input_plugin; + char *storage_sync; /* sync mode */ + int storage_metrics; /* enable/disable storage metrics */ + int storage_checksum; /* checksum enabled */ + int storage_max_chunks_up; /* max number of chunks 'up' in memory */ + int storage_del_bad_chunks; /* delete irrecoverable chunks */ + char *storage_bl_mem_limit; /* storage backlog memory limit */ + struct flb_storage_metrics *storage_metrics_ctx; /* storage metrics context */ + int storage_trim_files; /* enable/disable file trimming */ + + /* Embedded SQL Database support (SQLite3) */ +#ifdef FLB_HAVE_SQLDB + struct mk_list sqldb_list; +#endif + + /* LuaJIT environment's context */ +#ifdef FLB_HAVE_LUAJIT + struct mk_list luajit_list; +#endif + + /* WASM environment's context */ +#ifdef FLB_HAVE_WASM + struct mk_list wasm_list; +#endif + +#ifdef FLB_HAVE_STREAM_PROCESSOR + char *stream_processor_file; /* SP configuration file */ + void *stream_processor_ctx; /* SP context */ + int stream_processor_str_conv; /* SP enable converting from string to number */ + + /* + * Temporal list to hold tasks defined before the SP context is created + * by the engine. The list is passed upon start and destroyed. + */ + struct mk_list stream_processor_tasks; +#endif + +#ifdef FLB_HAVE_CHUNK_TRACE + int enable_chunk_trace; +#endif /* FLB_HAVE_CHUNK_TRACE */ + + int enable_hot_reload; + int ensure_thread_safety_on_hot_reloading; + unsigned int hot_reloaded_count; + + /* Co-routines */ + unsigned int coro_stack_size; + + /* Upstream contexts created by plugins */ + struct mk_list upstreams; + + /* Downstream contexts created by plugins */ + struct mk_list downstreams; + + /* + * Input table-id: table to keep a reference of thread-IDs used by the + * input plugins. + */ + uint16_t in_table_id[512]; + + void *sched; + unsigned int sched_cap; + unsigned int sched_base; + + struct flb_task_map tasks_map[2048]; + + int dry_run; +}; + +#define FLB_CONFIG_LOG_LEVEL(c) (c->log->level) + +struct flb_config *flb_config_init(); +void flb_config_exit(struct flb_config *config); +const char *flb_config_prop_get(const char *key, struct mk_list *list); +int flb_config_set_property(struct flb_config *config, + const char *k, const char *v); +int flb_config_set_program_name(struct flb_config *config, char *name); +int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf); + +int set_log_level_from_env(struct flb_config *config); +#ifdef FLB_HAVE_STATIC_CONF +struct flb_cf *flb_config_static_open(const char *file); +#endif + +struct flb_service_config { + char *key; + int type; + size_t offset; +}; + +enum conf_type { + FLB_CONF_TYPE_INT, + FLB_CONF_TYPE_DOUBLE, + FLB_CONF_TYPE_BOOL, + FLB_CONF_TYPE_STR, + FLB_CONF_TYPE_OTHER, +}; + +#define FLB_CONF_STR_FLUSH "Flush" +#define FLB_CONF_STR_GRACE "Grace" +#define FLB_CONF_STR_DAEMON "Daemon" +#define FLB_CONF_STR_LOGFILE "Log_File" +#define FLB_CONF_STR_LOGLEVEL "Log_Level" +#define FLB_CONF_STR_PARSERS_FILE "Parsers_File" +#define FLB_CONF_STR_PLUGINS_FILE "Plugins_File" +#define FLB_CONF_STR_STREAMS_FILE "Streams_File" +#define FLB_CONF_STR_STREAMS_STR_CONV "sp.convert_from_str_to_num" +#define FLB_CONF_STR_CONV_NAN "json.convert_nan_to_null" + +/* FLB_HAVE_HTTP_SERVER */ +#ifdef FLB_HAVE_HTTP_SERVER +#define FLB_CONF_STR_HTTP_SERVER "HTTP_Server" +#define FLB_CONF_STR_HTTP_LISTEN "HTTP_Listen" +#define FLB_CONF_STR_HTTP_PORT "HTTP_Port" +#define FLB_CONF_STR_HEALTH_CHECK "Health_Check" +#define FLB_CONF_STR_HC_ERRORS_COUNT "HC_Errors_Count" +#define FLB_CONF_STR_HC_RETRIES_FAILURE_COUNT "HC_Retry_Failure_Count" +#define FLB_CONF_STR_HC_PERIOD "HC_Period" +#endif /* !FLB_HAVE_HTTP_SERVER */ + +#ifdef FLB_HAVE_CHUNK_TRACE +#define FLB_CONF_STR_ENABLE_CHUNK_TRACE "Enable_Chunk_Trace" +#endif /* FLB_HAVE_CHUNK_TRACE */ + +#define FLB_CONF_STR_HOT_RELOAD "Hot_Reload" +#define FLB_CONF_STR_HOT_RELOAD_ENSURE_THREAD_SAFETY "Hot_Reload.Ensure_Thread_Safety" + +/* DNS */ +#define FLB_CONF_DNS_MODE "dns.mode" +#define FLB_CONF_DNS_RESOLVER "dns.resolver" +#define FLB_CONF_DNS_PREFER_IPV4 "dns.prefer_ipv4" + +/* Storage / Chunk I/O */ +#define FLB_CONF_STORAGE_PATH "storage.path" +#define FLB_CONF_STORAGE_SYNC "storage.sync" +#define FLB_CONF_STORAGE_METRICS "storage.metrics" +#define FLB_CONF_STORAGE_CHECKSUM "storage.checksum" +#define FLB_CONF_STORAGE_BL_MEM_LIMIT "storage.backlog.mem_limit" +#define FLB_CONF_STORAGE_MAX_CHUNKS_UP "storage.max_chunks_up" +#define FLB_CONF_STORAGE_DELETE_IRRECOVERABLE_CHUNKS \ + "storage.delete_irrecoverable_chunks" +#define FLB_CONF_STORAGE_TRIM_FILES "storage.trim_files" + +/* Coroutines */ +#define FLB_CONF_STR_CORO_STACK_SIZE "Coro_Stack_Size" + +/* Scheduler */ +#define FLB_CONF_STR_SCHED_CAP "scheduler.cap" +#define FLB_CONF_STR_SCHED_BASE "scheduler.base" + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_config_format.h b/fluent-bit/include/fluent-bit/flb_config_format.h new file mode 100644 index 00000000..8205ebcc --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_config_format.h @@ -0,0 +1,34 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_H +#define FLB_CONFIG_FORMAT_H + +#include <fluent-bit/flb_info.h> + +#include "config_format/flb_cf.h" +#include "config_format/flb_cf_fluentbit.h" + +#ifdef FLB_HAVE_LIBYAML +#include "config_format/flb_cf_yaml.h" +#endif + +#define FLB_DEFAULT_CF_BUF_SIZE 4096 + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_config_map.h b/fluent-bit/include/fluent-bit/flb_config_map.h new file mode 100644 index 00000000..3fe94abe --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_config_map.h @@ -0,0 +1,109 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_MAP_H +#define FLB_CONFIG_MAP_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_slist.h> +#include <fluent-bit/flb_sds.h> +#include <monkey/mk_core.h> + +/* Configuration types */ +#define FLB_CONFIG_MAP_STR 0 /* string */ +#define FLB_CONFIG_MAP_STR_PREFIX 1 /* string that starts with */ +#define FLB_CONFIG_MAP_INT 2 /* integer */ +#define FLB_CONFIG_MAP_BOOL 3 /* boolean */ +#define FLB_CONFIG_MAP_DOUBLE 4 /* double */ +#define FLB_CONFIG_MAP_SIZE 5 /* string size to integer (e.g: 2M) */ +#define FLB_CONFIG_MAP_TIME 6 /* string time to integer seconds (e.g: 2H) */ +#define FLB_CONFIG_MAP_DEPRECATED 7 /* for deprecated parameter */ + +#define FLB_CONFIG_MAP_CLIST 30 /* comma separated list of strings */ +#define FLB_CONFIG_MAP_CLIST_1 31 /* split up to 1 node + remaining data */ +#define FLB_CONFIG_MAP_CLIST_2 32 /* split up to 2 nodes + remaining data */ +#define FLB_CONFIG_MAP_CLIST_3 33 /* split up to 3 nodes + remaining data */ +#define FLB_CONFIG_MAP_CLIST_4 34 /* split up to 4 nodes + remaining data */ + +#define FLB_CONFIG_MAP_SLIST 40 /* empty space separated list of strings */ +#define FLB_CONFIG_MAP_SLIST_1 41 /* split up to 1 node + remaining data */ +#define FLB_CONFIG_MAP_SLIST_2 42 /* split up to 2 nodes + remaining data */ +#define FLB_CONFIG_MAP_SLIST_3 43 /* split up to 3 nodes + remaining data */ +#define FLB_CONFIG_MAP_SLIST_4 44 /* split up to 4 nodes + remaining data */ + +#define FLB_CONFIG_MAP_MULT 1 + +struct flb_config_map_val { + union { + int i_num; /* FLB_CONFIG_MAP_INT */ + char boolean; /* FLB_CONFIG_MAP_BOOL */ + double d_num; /* FLB_CONFIG_MAP_DOUBLE */ + size_t s_num; /* FLB_CONFIG_MAP_SIZE */ + flb_sds_t str; /* FLB_CONFIG_MAP_STR */ + struct mk_list *list; /* FLB_CONFIG_MAP_CLIST and FLB_CONFIG_MAP_SLIST */ + } val; + struct mk_list *mult; + struct mk_list _head; /* Link to list if this entry is a 'multiple' entry */ +}; + +struct flb_config_map { + /* Public fields used by plugins at registration */ + int type; /* type */ + flb_sds_t name; /* property name */ + flb_sds_t def_value; /* default value */ + int flags; /* option flags (e.g: multiple entries allowed) */ + int set_property; /* set context property ? (use offset ?) */ + uintptr_t offset; /* member offset */ + flb_sds_t desc; /* description */ + + struct flb_config_map_val value; /* lookup value */ + struct mk_list _head; +}; + +#define flb_config_map_foreach(curr, v, head) \ + for (curr = (head)->next, v = mk_list_entry(curr, \ + struct flb_config_map_val, \ + _head); \ + curr != (head); curr = curr->next, v = mk_list_entry(curr, \ + struct flb_config_map_val, \ + _head)) + +static inline int flb_config_map_mult_type(int type) +{ + if (type >= FLB_CONFIG_MAP_CLIST && type <= FLB_CONFIG_MAP_CLIST_4) { + return FLB_CONFIG_MAP_CLIST; + } + + if (type >= FLB_CONFIG_MAP_SLIST && type <= FLB_CONFIG_MAP_SLIST_4) { + return FLB_CONFIG_MAP_SLIST; + } + + return -1; +} + +int flb_config_map_properties_check(char *context_name, + struct mk_list *in_properties, + struct mk_list *map); +struct mk_list *flb_config_map_create(struct flb_config *config, + struct flb_config_map *map); +void flb_config_map_destroy(struct mk_list *list); +int flb_config_map_expected_values(int type); +int flb_config_map_set(struct mk_list *properties, struct mk_list *map, void *context); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_connection.h b/fluent-bit/include/fluent-bit/flb_connection.h new file mode 100644 index 00000000..46fd80c4 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_connection.h @@ -0,0 +1,184 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONNECTION_H +#define FLB_CONNECTION_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_socket.h> +#include <fluent-bit/flb_config.h> + +#define FLB_UNKNOWN_CONNECTION 0 +#define FLB_UPSTREAM_CONNECTION 1 +#define FLB_DOWNSTREAM_CONNECTION 2 + +/* FLB_CONNECTION_MAX_PORT_LENGTH is the maximum length of + * an unsigned 16 bit integer + */ + +#define FLB_CONNECTION_MAX_PORT_LENGTH 5 + +/* FLB_CONNECTION_MAX_LABEL_LENGTH is the maximum length of + * any of the following variants plus an optional colon if + * the spec includes a port number : + * + * udp:// + * tcp:// + * unix:// + */ +#define FLB_CONNECTION_MAX_LABEL_LENGTH 7 + +#define FLB_CONNECTION_MAX_IPV4_ADDRESS_LENGTH 15 +#define FLB_CONNECTION_MAX_IPV6_ADDRESS_LENGTH 39 +#define FLB_CONNECTION_MAX_UNIX_ADDRESS_LENGTH (11 + 4 + 1) +#define FLB_CONNECTION_MAX_ADDRESS_LENGTH FLB_CONNECTION_MAX_IPV6_ADDRESS_LENGTH + +#define FLB_CONNECTION_MAX_USER_FRIENDLY_ADDRESS_LENGTH \ + (FLB_CONNECTION_MAX_ADDRESS_LENGTH + \ + FLB_CONNECTION_MAX_PORT_LENGTH + \ + FLB_CONNECTION_MAX_LABEL_LENGTH) + +struct flb_net_setup; +struct flb_upstream; +struct flb_downstream; +struct flb_tls_session; + +/* Base network connection */ +struct flb_connection { + struct mk_event event; + + void *user_data; + + /* Socket */ + flb_sockfd_t fd; + + struct sockaddr_storage raw_remote_host; + + char remote_host[FLB_CONNECTION_MAX_ADDRESS_LENGTH + 1]; + unsigned short int remote_port; + + char user_friendly_remote_host[FLB_CONNECTION_MAX_USER_FRIENDLY_ADDRESS_LENGTH + 1]; + + /* Net setup shortcut */ + struct flb_net_setup *net; + + /* + * Custom 'error' for the connection file descriptor. Commonly used to + * specify a reason for an exception that was generated locally: consider + * a connect timeout, we shutdown(2) the connection but in reallity we + * might want to express an 'ETIMEDOUT' + */ + int net_error; + + /* If this flag is set, then destroy_conn will ignore this connection, this + * helps mitigate issues caused by flb_upstream_conn_timeouts marking a connection + * to be dropped and the event loop manager function destroying that connection + * at the end of the cycle while the connection coroutine is still suspended which + * causes the outer functions to access invalid memory when handling the error amongst + * other things. + */ + int busy_flag; + + /* This flag is used to determine if the connection was shut down to ensure we + * don't do it twice when a timeout is detected. + * + * This is required in order to overcome a limitation in the async read / write + * functions that will be addressed as soon as possible. + */ + int shutdown_flag; + + /* + * Recycle: if the connection is keepalive, this flag is always on, but if + * the caller wants to drop the connection once is released, it can set + * recycle to false. + */ + int recycle; + + /* Keepalive */ + int ka_count; /* how many times this connection has been used */ + + /* Timestamps */ + time_t ts_assigned; + time_t ts_created; + time_t ts_available; /* sets the 'start' available time */ + + /* IO activity tracker used to enforce net.io_timeout */ + time_t ts_io_timeout; + + /* Connect */ + time_t ts_connect_start; + time_t ts_connect_timeout; + + /* Event loop */ + struct mk_event_loop *evl; + + /* Parent stream */ + union { + struct flb_stream *stream; + struct flb_upstream *upstream; + struct flb_downstream *downstream; + }; + + /* Coroutine in charge of this connection */ + struct flb_coro *coroutine; + + /* Connection type : FLB_UPSTREAM_CONNECTION or FLB_DOWNSTREAM_CONNECTION */ + int type; + + int dynamically_allocated; + + /* + * Link to list head on the stream, if the connection is busy, + * it's linked to 'busy_queue', otherwise it resides in 'av_queue' + * for upstream connections so it can be used by a plugin or + * 'destroy_queue' awaiting release. + */ + struct mk_list _head; + + /* Each TCP connections using TLS needs a session */ + struct flb_tls_session *tls_session; +}; + +int flb_connection_setup(struct flb_connection *connection, + flb_sockfd_t socket, + int type, + void *stream, + struct mk_event_loop *event_loop, + struct flb_coro *coroutine); + +struct flb_connection *flb_connection_create(flb_sockfd_t socket, + int type, + void *stream, + struct mk_event_loop *event_loop, + struct flb_coro *coroutine); + +void flb_connection_destroy(struct flb_connection *connection); + +void flb_connection_set_remote_host(struct flb_connection *connection, + struct sockaddr *remote_host); +char *flb_connection_get_remote_address(struct flb_connection *connection); + +int flb_connection_get_flags(struct flb_connection *connection); +void flb_connection_reset_connection_timeout(struct flb_connection *connection); +void flb_connection_unset_connection_timeout(struct flb_connection *connection); + +void flb_connection_reset_io_timeout(struct flb_connection *connection); +void flb_connection_unset_io_timeout(struct flb_connection *connection); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_coro.h b/fluent-bit/include/fluent-bit/flb_coro.h new file mode 100644 index 00000000..8e14137b --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_coro.h @@ -0,0 +1,136 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CORO_H +#define FLB_CORO_H + +/* Required by OSX */ +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE +#endif + +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE +#endif + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_macros.h> +#include <fluent-bit/flb_log.h> +#include <fluent-bit/flb_mem.h> + +#include <monkey/mk_core.h> + +#include <stdlib.h> +#include <limits.h> +#include <libco.h> + +#ifdef FLB_HAVE_VALGRIND +#include <valgrind/valgrind.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +struct flb_coro { + +#ifdef FLB_HAVE_VALGRIND + unsigned int valgrind_stack_id; +#endif + + /* libco 'contexts' */ + cothread_t caller; + cothread_t callee; + + void *data; +}; + +#ifdef FLB_SYSTEM_MACOS +#ifdef __aarch64__ +#define STACK_FACTOR 1.5 /* Use 36KiB for coro stacks */ +#else +#define STACK_FACTOR 2 /* Use 24KiB for coro stacks */ +#endif +#else +#define STACK_FACTOR 1 +#endif + +#ifdef FLB_CORO_STACK_SIZE +#define FLB_CORO_STACK_SIZE_BYTE FLB_CORO_STACK_SIZE +#else +#define FLB_CORO_STACK_SIZE_BYTE ((3 * STACK_FACTOR * PTHREAD_STACK_MIN) / 2) +#endif + +#define FLB_CORO_DATA(coro) (((char *) coro) + sizeof(struct flb_coro)) + +static FLB_INLINE void flb_coro_yield(struct flb_coro *coro, int ended) +{ + co_switch(coro->caller); +} + +static FLB_INLINE void flb_coro_destroy(struct flb_coro *coro) +{ + flb_trace("[coro] destroy coroutine=%p data=%p", coro, + FLB_CORO_DATA(coro)); + +#ifdef FLB_HAVE_VALGRIND + VALGRIND_STACK_DEREGISTER(coro->valgrind_stack_id); +#endif + + if (coro->callee != NULL) { + co_delete(coro->callee); + } + + flb_free(coro); +} + +#define flb_coro_return(th) co_switch(th->caller) + +void flb_coro_init(); +void flb_coro_thread_init(); + +struct flb_coro *flb_coro_get(); +void flb_coro_set(struct flb_coro *coro); + +static FLB_INLINE void flb_coro_resume(struct flb_coro *coro) +{ + flb_coro_set(coro); + coro->caller = co_active(); + co_switch(coro->callee); +} + +static FLB_INLINE struct flb_coro *flb_coro_create(void *data) +{ + struct flb_coro *coro; + + /* Create a thread context and initialize */ + coro = (struct flb_coro *) flb_calloc(1, sizeof(struct flb_coro)); + if (!coro) { + flb_errno(); + return NULL; + } + coro->data = data; + return coro; +} + +#ifdef __cplusplus +} +#endif + +#endif /* !FLB_CORO_H */ diff --git a/fluent-bit/include/fluent-bit/flb_crypto.h b/fluent-bit/include/fluent-bit/flb_crypto.h new file mode 100644 index 00000000..e406388e --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_crypto.h @@ -0,0 +1,101 @@ +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CRYPTO_H +#define FLB_CRYPTO_H + +#include <stdint.h> +#include <openssl/err.h> +#include <openssl/pem.h> +#include <openssl/sha.h> +#include <openssl/evp.h> +#include <openssl/hmac.h> +#include <openssl/engine.h> +#include <fluent-bit/flb_crypto_constants.h> + + +struct flb_crypto { + const EVP_MD *digest_algorithm; + EVP_PKEY_CTX *backend_context; + int last_operation; + int padding_type; + size_t block_size; + unsigned long last_error; + EVP_PKEY *key; +}; + +int flb_crypto_init(struct flb_crypto *context, + int padding_type, + int digest_algorithm, + int key_type, + unsigned char *key, + size_t key_length); + +int flb_crypto_cleanup(struct flb_crypto *context); + +int flb_crypto_transform(struct flb_crypto *context, + int operation, + unsigned char *input_buffer, + size_t input_length, + unsigned char *output_buffer, + size_t *output_length); + +int flb_crypto_sign(struct flb_crypto *context, + unsigned char *input_buffer, + size_t input_length, + unsigned char *output_buffer, + size_t *output_length); + +int flb_crypto_encrypt(struct flb_crypto *context, + unsigned char *input_buffer, + size_t input_length, + unsigned char *output_buffer, + size_t *output_length); + +int flb_crypto_decrypt(struct flb_crypto *context, + unsigned char *input_buffer, + size_t input_length, + unsigned char *output_buffer, + size_t *output_length); + +int flb_crypto_sign_simple(int key_type, + int padding_type, + int digest_algorithm, + unsigned char *key, + size_t key_length, + unsigned char *input_buffer, + size_t input_length, + unsigned char *output_buffer, + size_t *output_length); + +int flb_crypto_encrypt_simple(int padding_type, + unsigned char *key, + size_t key_length, + unsigned char *input_buffer, + size_t input_length, + unsigned char *output_buffer, + size_t *output_length); + +int flb_crypto_decrypt_simple(int padding_type, + unsigned char *key, + size_t key_length, + unsigned char *input_buffer, + size_t input_length, + unsigned char *output_buffer, + size_t *output_length); + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_crypto_constants.h b/fluent-bit/include/fluent-bit/flb_crypto_constants.h new file mode 100644 index 00000000..5728e0b4 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_crypto_constants.h @@ -0,0 +1,56 @@ +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CRYPTO_CONSTANTS_H +#define FLB_CRYPTO_CONSTANTS_H + +#include <openssl/opensslv.h> + +#if OPENSSL_VERSION_NUMBER < 0x10100000 +#define FLB_CRYPTO_OPENSSL_COMPAT_MODE 0 +#elif OPENSSL_VERSION_NUMBER < 0x30000000 +#define FLB_CRYPTO_OPENSSL_COMPAT_MODE 1 +#else +#define FLB_CRYPTO_OPENSSL_COMPAT_MODE 3 +#endif + +#define FLB_CRYPTO_SUCCESS 0 +#define FLB_CRYPTO_BACKEND_ERROR 1 +#define FLB_CRYPTO_INVALID_STATE 2 +#define FLB_CRYPTO_INVALID_ARGUMENT 3 +#define FLB_CRYPTO_ALLOCATION_ERROR 4 + +#define FLB_CRYPTO_PUBLIC_KEY 1 +#define FLB_CRYPTO_PRIVATE_KEY 2 + +#define FLB_CRYPTO_PADDING_NONE 0 +#define FLB_CRYPTO_PADDING_PKCS1 1 +#define FLB_CRYPTO_PADDING_PKCS1_OEAP 2 +#define FLB_CRYPTO_PADDING_PKCS1_X931 3 +#define FLB_CRYPTO_PADDING_PKCS1_PSS 4 + +#define FLB_HASH_NONE 0 +#define FLB_HASH_MD5 1 +#define FLB_HASH_SHA256 2 +#define FLB_HASH_SHA512 3 + +#define FLB_CRYPTO_OPERATION_NONE 0 +#define FLB_CRYPTO_OPERATION_SIGN 1 +#define FLB_CRYPTO_OPERATION_ENCRYPT 2 +#define FLB_CRYPTO_OPERATION_DECRYPT 3 + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_csv.h b/fluent-bit/include/fluent-bit/flb_csv.h new file mode 100644 index 00000000..b6b29711 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_csv.h @@ -0,0 +1,64 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CSV_H +#define FLB_CSV_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_time.h> + +enum { + FLB_CSV_SUCCESS = 0, + FLB_CSV_ALLOC_FAILED = -1, + FLB_CSV_INVALID_STATE = -2, + FLB_CSV_EOF = -3, +}; + +typedef void (*flb_csv_field_parsed_callback)(void *data, + const char *field, + size_t field_len); + +struct flb_csv_state { + flb_csv_field_parsed_callback field_callback; + flb_sds_t buffered_data; + flb_sds_t escape_buffer; + size_t offset; + size_t start; + size_t length; + size_t field_count; + int state; + bool field_parsed; + bool has_dquote; + void *data; +}; + +void flb_csv_init(struct flb_csv_state *state, + flb_csv_field_parsed_callback field_callback, + void *data); + +int flb_csv_parse_record(struct flb_csv_state *state, + char **bufptr, + size_t *buflen, + size_t *field_count); + +void flb_csv_destroy(struct flb_csv_state *state); + + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_custom.h b/fluent-bit/include/fluent-bit/flb_custom.h new file mode 100644 index 00000000..f75e8de1 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_custom.h @@ -0,0 +1,94 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CUSTOM_H +#define FLB_CUSTOM_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_config_map.h> +#ifdef FLB_HAVE_METRICS +#include <fluent-bit/flb_metrics.h> +#endif + +struct flb_custom_instance; + +struct flb_custom_plugin { + int flags; /* Flags (not available at the moment */ + char *name; /* Custom plugin short name */ + char *description; /* Description */ + + /* Config map */ + struct flb_config_map *config_map; + + /* Callbacks */ + int (*cb_init) (struct flb_custom_instance *, struct flb_config *, void *); + int (*cb_run) (const void *, size_t, const char *, int, + void **, size_t *, + struct flb_custom_instance *, + void *, struct flb_config *); + int (*cb_exit) (void *, struct flb_config *); + + struct mk_list _head; /* Link to parent list (config->custom) */ +}; + +struct flb_custom_instance { + int id; /* instance id */ + int log_level; /* instance log level */ + char name[32]; /* numbered name */ + char *alias; /* alias name */ + void *context; /* instance local context */ + void *data; + struct flb_custom_plugin *p; /* original plugin */ + struct mk_list properties; /* config properties */ + struct mk_list *config_map; /* configuration map */ + struct mk_list _head; /* link to config->customs */ + + /* + * CMetrics + * -------- + */ + struct cmt *cmt; /* parent context */ + + /* Keep a reference to the original context this instance belongs to */ + struct flb_config *config; +}; + +static inline int flb_custom_config_map_set(struct flb_custom_instance *ins, + void *context) +{ + return flb_config_map_set(&ins->properties, ins->config_map, context); +} + +int flb_custom_set_property(struct flb_custom_instance *ins, + const char *k, const char *v); +const char *flb_custom_get_property(const char *key, + struct flb_custom_instance *ins); + +struct flb_custom_instance *flb_custom_new(struct flb_config *config, + const char *custom, void *data); +void flb_custom_exit(struct flb_config *config); +const char *flb_custom_name(struct flb_custom_instance *ins); +int flb_custom_plugin_property_check(struct flb_custom_instance *ins, + struct flb_config *config); +int flb_custom_init_all(struct flb_config *config); +void flb_custom_set_context(struct flb_custom_instance *ins, void *context); +void flb_custom_instance_destroy(struct flb_custom_instance *ins); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_custom_plugin.h b/fluent-bit/include/fluent-bit/flb_custom_plugin.h new file mode 100644 index 00000000..eca4f559 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_custom_plugin.h @@ -0,0 +1,53 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CUSTOM_PLUGIN_H +#define FLB_CUSTOM_PLUGIN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_custom.h> +#include <fluent-bit/flb_log.h> + +#define flb_plg_error(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_ERROR)) \ + flb_log_print(FLB_LOG_ERROR, NULL, 0, "[custom:%s:%s] " fmt, \ + ctx->p->name, flb_custom_name(ctx), ##__VA_ARGS__) + +#define flb_plg_warn(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_WARN)) \ + flb_log_print(FLB_LOG_WARN, NULL, 0, "[custom:%s:%s] " fmt, \ + ctx->p->name, flb_custom_name(ctx), ##__VA_ARGS__) + +#define flb_plg_info(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_INFO)) \ + flb_log_print(FLB_LOG_INFO, NULL, 0, "[custom:%s:%s] " fmt, \ + ctx->p->name, flb_custom_name(ctx), ##__VA_ARGS__) + +#define flb_plg_debug(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_DEBUG)) \ + flb_log_print(FLB_LOG_DEBUG, NULL, 0, "[custom:%s:%s] " fmt, \ + ctx->p->name, flb_custom_name(ctx), ##__VA_ARGS__) + +#define flb_plg_trace(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_TRACE)) \ + flb_log_print(FLB_LOG_TRACE, NULL, 0, \ + "[custom:%s:%s at %s:%i] " fmt, \ + ctx->p->name, flb_custom_name(ctx), __FLB_FILENAME__, \ + __LINE__, ##__VA_ARGS__) +#endif diff --git a/fluent-bit/include/fluent-bit/flb_dlfcn_win32.h b/fluent-bit/include/fluent-bit/flb_dlfcn_win32.h new file mode 100644 index 00000000..61345648 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_dlfcn_win32.h @@ -0,0 +1,25 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define RTLD_LAZY 0 + +void *dlopen(const char *filename, int _flag); +char *dlerror(void); +void *dlsym(void *handle, const char *symbol); +int dlclose(void *handle); diff --git a/fluent-bit/include/fluent-bit/flb_downstream.h b/fluent-bit/include/fluent-bit/flb_downstream.h new file mode 100644 index 00000000..117fdc2c --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_downstream.h @@ -0,0 +1,88 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_DOWNSTREAM_H +#define FLB_DOWNSTREAM_H + +#include <monkey/mk_core.h> + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_socket.h> +#include <fluent-bit/flb_network.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_io.h> +#include <fluent-bit/flb_stream.h> + +struct flb_connection; + +/* Downstream handler */ +struct flb_downstream { + struct flb_stream base; + + char *host; + unsigned short int port; + flb_sockfd_t server_fd; + struct flb_connection *dgram_connection; + + struct mk_list busy_queue; + struct mk_list destroy_queue; +}; + +static inline int flb_downstream_is_shutting_down(struct flb_downstream *downstream) +{ + return flb_stream_is_shutting_down(&downstream->base); +} + +void flb_downstream_init(); + +int flb_downstream_setup(struct flb_downstream *stream, + int transport, int flags, + const char *host, + unsigned short int port, + struct flb_tls *tls, + struct flb_config *config, + struct flb_net_setup *net_setup); + +struct flb_downstream *flb_downstream_create(int transport, int flags, + const char *host, + unsigned short int port, + struct flb_tls *tls, + struct flb_config *config, + struct flb_net_setup *net_setup); + +void flb_downstream_destroy(struct flb_downstream *downstream); + +int flb_downstream_set_property(struct flb_config *config, + struct flb_net_setup *net, char *k, char *v); + +struct flb_connection *flb_downstream_conn_get(struct flb_downstream *stream); + +int flb_downstream_conn_release(struct flb_connection *connection); + +int flb_downstream_conn_pending_destroy_list(struct mk_list *list); + +int flb_downstream_conn_timeouts(struct mk_list *list); + +int flb_downstream_is_async(struct flb_downstream *downstream); + +void flb_downstream_thread_safe(struct flb_downstream *stream); + +struct mk_list *flb_downstream_get_config_map(struct flb_config *config); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_downstream_conn.h b/fluent-bit/include/fluent-bit/flb_downstream_conn.h new file mode 100644 index 00000000..b6e28d0b --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_downstream_conn.h @@ -0,0 +1,33 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_DOWNSTREAM_CONN_H +#define FLB_DOWNSTREAM_CONN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_socket.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_connection.h> + +int flb_downstream_conn_release(struct flb_connection *connection); +int flb_downstream_conn_timeouts(struct mk_list *list); +int flb_downstream_conn_pending_destroy(struct flb_downstream *downstream); +struct flb_connection *flb_downstream_conn_get(struct flb_downstream *stream); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_dump.h b/fluent-bit/include/fluent-bit/flb_dump.h new file mode 100644 index 00000000..455c8bb3 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_dump.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_DUMP_H +#define FLB_DUMP_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> + +void flb_dump(struct flb_config *ctx); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_endian.h b/fluent-bit/include/fluent-bit/flb_endian.h new file mode 100644 index 00000000..982c6358 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_endian.h @@ -0,0 +1,54 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* This file provides a portability layer for endian functions like + * {be,le}{16,32,64}toh(). + * + * You can include <fluent-bit/flb_endian.h> instead of <endian.h> and + * ditto <sys/endian.h>. + */ + +#ifndef FLB_ENDIAN_H +#define FLB_ENDIAN_H + +/* endian.h is non-standard and platforms disagree with each other on it. + * + * * GLIBC (and its family) supports <endian.h>. + * + * * BSD libc provides <sys/endian.h> instead (Note that OpenBSD since + * 5.6 supports conventional function names). + * + * * macOS does not support <endian.h> but exposes another set of macros + * in libkern. + */ +#if defined(__GLIBC__) +#include <endian.h> +#elif defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) +#include <sys/endian.h> +#elif defined(__APPLE__) +#include <libkern/OSByteOrder.h> +#define le16toh(x) OSSwapLittleToHostInt16(x) +#define le32toh(x) OSSwapLittleToHostInt32(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#endif + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_engine.h b/fluent-bit/include/fluent-bit/flb_engine.h new file mode 100644 index 00000000..4c1cb076 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_engine.h @@ -0,0 +1,46 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ENGINE_H +#define FLB_ENGINE_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_input.h> +#include <fluent-bit/flb_output.h> +#include <fluent-bit/flb_thread_storage.h> + +#define FLB_ENGINE_OUTPUT_EVENT_BATCH_SIZE 1 + +int flb_engine_start(struct flb_config *config); +int flb_engine_failed(struct flb_config *config); +int flb_engine_flush(struct flb_config *config, + struct flb_input_plugin *in_force); +int flb_engine_exit(struct flb_config *config); +int flb_engine_exit_status(struct flb_config *config, int status); +int flb_engine_shutdown(struct flb_config *config); +int flb_engine_destroy_tasks(struct mk_list *tasks); +void flb_engine_reschedule_retries(struct flb_config *config); + +/* Engine event loop */ +void flb_engine_evl_init(); +struct mk_event_loop *flb_engine_evl_get(); +void flb_engine_evl_set(struct mk_event_loop *evl); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_engine_dispatch.h b/fluent-bit/include/fluent-bit/flb_engine_dispatch.h new file mode 100644 index 00000000..b61d5304 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_engine_dispatch.h @@ -0,0 +1,32 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ENGINE_DISPATCH_H +#define FLB_ENGINE_DISPATCH_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_input.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_task.h> + +int flb_engine_dispatch(uint64_t id, struct flb_input_instance *in, + struct flb_config *config); +int flb_engine_dispatch_retry(struct flb_task_retry *retry, + struct flb_config *config); +#endif diff --git a/fluent-bit/include/fluent-bit/flb_engine_macros.h b/fluent-bit/include/fluent-bit/flb_engine_macros.h new file mode 100644 index 00000000..825febc0 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_engine_macros.h @@ -0,0 +1,77 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ENGINE_MACROS_H +#define FLB_ENGINE_MACROS_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_bits.h> + +/* Types of events handled by the Server engine */ +#define FLB_ENGINE_EV_CORE MK_EVENT_NOTIFICATION +#define FLB_ENGINE_EV_CUSTOM MK_EVENT_CUSTOM +#define FLB_ENGINE_EV_THREAD 1024 +#define FLB_ENGINE_EV_SCHED 2048 +#define FLB_ENGINE_EV_SCHED_FRAME (FLB_ENGINE_EV_SCHED + 4096) + +#define FLB_ENGINE_EV_INPUT 8192 +#define FLB_ENGINE_EV_THREAD_INPUT 16384 /* reserved, not used yet */ + +#define FLB_ENGINE_EV_OUTPUT 32768 +#define FLB_ENGINE_EV_THREAD_OUTPUT 65536 +#define FLB_ENGINE_EV_THREAD_ENGINE 131072 + +/* Engine events: all engine events set the left 32 bits to '1' */ +#define FLB_ENGINE_EV_STARTED FLB_BITS_U64_SET(1, 1) /* Engine started */ +#define FLB_ENGINE_EV_FAILED FLB_BITS_U64_SET(1, 2) /* Engine started */ +#define FLB_ENGINE_EV_STOP FLB_BITS_U64_SET(1, 3) /* Requested to stop */ +#define FLB_ENGINE_EV_SHUTDOWN FLB_BITS_U64_SET(1, 4) /* Engine shutdown */ + +/* Similar to engine events, but used as return values */ +#define FLB_ENGINE_STARTED FLB_BITS_U64_LOW(FLB_ENGINE_EV_STARTED) +#define FLB_ENGINE_FAILED FLB_BITS_U64_LOW(FLB_ENGINE_EV_FAILED) +#define FLB_ENGINE_STOP FLB_BITS_U64_LOW(FLB_ENGINE_EV_STOP) +#define FLB_ENGINE_SHUTDOWN FLB_BITS_U64_LOW(FLB_ENGINE_EV_SHUTDOWN) + +/* Engine signals: Task, it only refer to the type */ +#define FLB_ENGINE_TASK 2 +#define FLB_ENGINE_IN_CORO 3 + +/* Engine priority queue configuration */ +#define FLB_ENGINE_LOOP_MAX_ITER 10 /* Max events processed per round */ + +/* Engine event priorities: min value prioritized */ +#define FLB_ENGINE_PRIORITY_DEFAULT MK_EVENT_PRIORITY_DEFAULT +#define FLB_ENGINE_PRIORITY_COUNT 8 +#define FLB_ENGINE_PRIORITY_TOP 0 +#define FLB_ENGINE_PRIORITY_BOTTOM (FLB_ENGINE_PRIORITY_COUNT - 1) +#define FLB_ENGINE_PRIORITY_NETWORK 1 + +#define FLB_ENGINE_PRIORITY_CB_SCHED FLB_ENGINE_PRIORITY_TOP +#define FLB_ENGINE_PRIORITY_CB_TIMER FLB_ENGINE_PRIORITY_TOP +#define FLB_ENGINE_PRIORITY_SHUTDOWN FLB_ENGINE_PRIORITY_TOP +#define FLB_ENGINE_PRIORITY_FLUSH (FLB_ENGINE_PRIORITY_NETWORK + 1) + +#define FLB_ENGINE_PRIORITY_DNS FLB_ENGINE_PRIORITY_NETWORK +#define FLB_ENGINE_PRIORITY_CONNECT FLB_ENGINE_PRIORITY_NETWORK +#define FLB_ENGINE_PRIORITY_SEND_RECV FLB_ENGINE_PRIORITY_NETWORK + +#define FLB_ENGINE_PRIORITY_THREAD FLB_ENGINE_PRIORITY_DEFAULT + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_env.h b/fluent-bit/include/fluent-bit/flb_env.h new file mode 100644 index 00000000..fe7dd13e --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_env.h @@ -0,0 +1,44 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ENV_H +#define FLB_ENV_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> + +#define FLB_ENV_SIZE 64 + +struct flb_env { + int warn_unused; /* warn about unused environment variable */ + struct flb_hash_table *ht; +}; + +static inline void flb_env_warn_unused(struct flb_env *env, int warn) +{ + env->warn_unused = warn; +} + +struct flb_env *flb_env_create(); +void flb_env_destroy(struct flb_env *env); +int flb_env_set(struct flb_env *env, const char *key, const char *val); +const char *flb_env_get(struct flb_env *env, const char *key); +flb_sds_t flb_env_var_translate(struct flb_env *env, const char *value); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_error.h b/fluent-bit/include/fluent-bit/flb_error.h new file mode 100644 index 00000000..9cc6f2d4 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_error.h @@ -0,0 +1,54 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ERROR_H +#define FLB_ERROR_H + +#define FLB_ERR_CFG_FILE 10 +#define FLB_ERR_CFG_FILE_FORMAT 11 +#define FLB_ERR_CFG_FILE_STOP 12 +#define FLB_ERR_CFG_FLUSH 20 +#define FLB_ERR_CFG_FLUSH_CREATE 21 +#define FLB_ERR_CFG_FLUSH_REGISTER 22 +#define FLB_ERR_CUSTOM_INVALID 49 +#define FLB_ERR_INPUT_INVALID 50 +#define FLB_ERR_INPUT_UNDEF 51 +#define FLB_ERR_INPUT_UNSUP 52 +#define FLB_ERR_OUTPUT_UNDEF 100 +#define FLB_ERR_OUTPUT_INVALID 101 +#define FLB_ERR_OUTPUT_UNIQ 102 +#define FLB_ERR_FILTER_INVALID 201 + +/* Parser */ +#define FLB_ERR_CFG_PARSER_FILE 300 + +/* Plugin */ +#define FLB_ERR_CFG_PLUGIN_FILE 400 + +/* JSON errors */ +#define FLB_ERR_JSON_INVAL -501 +#define FLB_ERR_JSON_PART -502 + +/* Coroutine errors */ +#define FLB_ERR_CORO_STACK_SIZE -600 + +/* Reloading */ +#define FLB_ERR_RELOADING_IN_PROGRESS 700 + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_event.h b/fluent-bit/include/fluent-bit/flb_event.h new file mode 100644 index 00000000..6217f077 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_event.h @@ -0,0 +1,57 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_EVENT_H +#define FLB_EVENT_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_input_chunk.h> + +/* Event types */ +#define FLB_EVENT_TYPE_LOGS FLB_INPUT_CHUNK_TYPE_LOGS +#define FLB_EVENT_TYPE_METRICS FLB_INPUT_CHUNK_TYPE_METRICS +#define FLB_EVENT_TYPE_TRACES FLB_INPUT_CHUNK_TYPE_TRACES + +#define FLB_EVENT_TYPE_HAS_TRACE FLB_INPUT_CHUNK_HAS_TRACE + +/* + * The flb_event_chunk structure is a full context used in the output plugins + * flush callback. It contains the type of records (logs, metrics), the tag, + * msgpack buffer, it size and a hint of the serialized msgpack events. + */ +struct flb_event_chunk { + int type; /* event type */ + flb_sds_t tag; /* tag associated */ + void *data; /* event content */ + size_t size; /* size of event */ + size_t total_events; /* total number of serialized events */ +}; + +struct flb_event_chunk *flb_event_chunk_create(int type, + int total_events, + char *tag_buf, int tag_len, + char *buf_data, size_t buf_size); + +int flb_event_chunk_update(struct flb_event_chunk *evc, + char *buf_data, size_t buf_size); + +void flb_event_chunk_destroy(struct flb_event_chunk *evc); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_event_loop.h b/fluent-bit/include/fluent-bit/flb_event_loop.h new file mode 100644 index 00000000..e8c75360 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_event_loop.h @@ -0,0 +1,102 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_EVENT_LOOP_H +#define FLB_EVENT_LOOP_H + +#include <monkey/mk_core/mk_event.h> +#include <fluent-bit/flb_bucket_queue.h> +#include <fluent-bit/flb_log.h> + +static inline void flb_event_load_bucket_queue_event(struct flb_bucket_queue *bktq, + struct mk_event *event) +{ + if (event->_priority_head.prev == NULL) { /* not in bktq */ + flb_bucket_queue_add(bktq, &event->_priority_head, event->priority); + } +} + +/* Priority queue utility */ +static inline void flb_event_load_bucket_queue(struct flb_bucket_queue *bktq, + struct mk_event_loop *evl) +{ + struct mk_event *event; + mk_event_foreach(event, evl) { + if (event->status != MK_EVENT_NONE) { /* not deleted event */ + flb_event_load_bucket_queue_event(bktq, event); + } + } +} + +/* Accommadate inject */ +static inline void flb_event_load_injected_events(struct flb_bucket_queue *bktq, + struct mk_event_loop *evl, + int n_events_initial) +{ + struct mk_event *event; + int i; + + if ( evl->n_events < n_events_initial) { + flb_error("[flb_event_loop] event(s) removed from ready list. " + "This should never happen"); + return; + } + + /* Some events have been added through mk_event_inject */ + if (evl->n_events > n_events_initial) { + i = 0; + mk_event_foreach(event, evl) { + if (i >= n_events_initial) { + flb_event_load_bucket_queue_event(bktq, event); + } + ++i; + } + } +} + +#define flb_event_priority_live_foreach(event, bktq, evl, max_iter) \ + int __flb_event_priority_live_foreach_iter = 0; \ + int __flb_event_priority_live_foreach_n_events = evl->n_events; \ + for ( \ + /* init */ \ + flb_event_load_bucket_queue(bktq, evl), \ + event = flb_bucket_queue_find_min(bktq) ? \ + mk_list_entry( \ + flb_bucket_queue_pop_min(bktq), struct mk_event, _priority_head) : \ + NULL; \ + \ + /* condition */ \ + event != NULL && \ + (__flb_event_priority_live_foreach_iter < max_iter || max_iter == -1); \ + \ + /* update */ \ + ++__flb_event_priority_live_foreach_iter, \ + flb_event_load_injected_events(bktq, evl, \ + __flb_event_priority_live_foreach_n_events), \ + mk_event_wait_2(evl, 0), \ + __flb_event_priority_live_foreach_n_events = evl->n_events, \ + flb_event_load_bucket_queue(bktq, evl), \ + event = flb_bucket_queue_find_min(bktq) ? \ + mk_list_entry( \ + flb_bucket_queue_pop_min(bktq), struct mk_event, _priority_head) : \ + NULL \ + ) + +#endif /* !FLB_EVENT_LOOP_H */ diff --git a/fluent-bit/include/fluent-bit/flb_file.h b/fluent-bit/include/fluent-bit/flb_file.h new file mode 100644 index 00000000..0fbb357b --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_file.h @@ -0,0 +1,29 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FILE_H +#define FLB_FILE_H + +#include <fluent-bit/flb_sds.h> + +flb_sds_t flb_file_read(const char *path); +// TODO int flb_file_write(const char *path, flb_sds_t contents); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_filter.h b/fluent-bit/include/fluent-bit/flb_filter.h new file mode 100644 index 00000000..12649073 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_filter.h @@ -0,0 +1,148 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FILTER_H +#define FLB_FILTER_H + +#include <fluent-bit/flb_info.h> + +#ifdef FLB_HAVE_REGEX +#include <fluent-bit/flb_regex.h> +#endif + +#ifdef FLB_HAVE_METRICS +#include <fluent-bit/flb_metrics.h> +#endif + +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_config_map.h> +#include <fluent-bit/flb_input_chunk.h> +#include <msgpack.h> + +#include <cmetrics/cmetrics.h> +#include <cmetrics/cmt_counter.h> + +#define FLB_FILTER_MODIFIED 1 +#define FLB_FILTER_NOTOUCH 2 + +/* + * Types are defined by creating a mask using numbers. However, it's important + * to note that the masks used in this process are different from the ones used + * in flb_event.h. The original chunk values are not actually masks, but rather set + * numbers starting from 0; this is for compatibility reasons. + */ +#define FLB_FILTER_LOGS 1 +#define FLB_FILTER_METRICS 2 +#define FLB_FILTER_TRACES 4 + +struct flb_input_instance; +struct flb_filter_instance; + +struct flb_filter_plugin { + int event_type; /* Event type: logs, metrics, traces */ + int flags; /* Flags (not available at the moment */ + char *name; /* Filter short name */ + char *description; /* Description */ + + /* Config map */ + struct flb_config_map *config_map; + + /* Callbacks */ + int (*cb_init) (struct flb_filter_instance *, struct flb_config *, void *); + int (*cb_filter) (const void *, size_t, + const char *, int, + void **, size_t *, + struct flb_filter_instance *, + struct flb_input_instance *, + void *, struct flb_config *); + int (*cb_exit) (void *, struct flb_config *); + + struct mk_list _head; /* Link to parent list (config->filters) */ +}; + +struct flb_filter_instance { + int event_type; /* Event type: logs, metrics, traces */ + int id; /* instance id */ + int log_level; /* instance log level */ + int log_suppress_interval; /* log suppression interval */ + char name[32]; /* numbered name */ + char *alias; /* alias name */ + char *match; /* match rule based on Tags */ +#ifdef FLB_HAVE_REGEX + struct flb_regex *match_regex; /* match rule (regex) based on Tags */ +#endif + void *parent_processor; /* Parent processor */ + void *context; /* Instance local context */ + void *data; + struct flb_filter_plugin *p; /* original plugin */ + struct mk_list properties; /* config properties */ + struct mk_list *config_map; /* configuration map */ + + struct mk_list _head; /* link to config->filters */ + + /* + * CMetrics + * -------- + */ + struct cmt *cmt; /* parent context */ + struct cmt_counter *cmt_records; /* m: filter_records_total */ + struct cmt_counter *cmt_bytes; /* m: filter_bytes_total */ + struct cmt_counter *cmt_add_records; /* m: filter_add_records_total */ + struct cmt_counter *cmt_drop_records; /* m: filter_drop_records_total */ + +#ifdef FLB_HAVE_METRICS + struct flb_metrics *metrics; /* metrics */ +#endif + + /* Keep a reference to the original context this instance belongs to */ + struct flb_config *config; +}; + +static inline int flb_filter_config_map_set(struct flb_filter_instance *ins, + void *context) +{ + return flb_config_map_set(&ins->properties, ins->config_map, context); +} + +int flb_filter_set_property(struct flb_filter_instance *ins, + const char *k, const char *v); +const char *flb_filter_get_property(const char *key, + struct flb_filter_instance *ins); + +struct flb_filter_instance *flb_filter_new(struct flb_config *config, + const char *filter, void *data); +void flb_filter_instance_exit(struct flb_filter_instance *ins, + struct flb_config *config); + +void flb_filter_exit(struct flb_config *config); +void flb_filter_do(struct flb_input_chunk *ic, + const void *data, size_t bytes, + const char *tag, int tag_len, + struct flb_config *config); +const char *flb_filter_name(struct flb_filter_instance *ins); + +int flb_filter_match_property_existence(struct flb_filter_instance *ins); +int flb_filter_plugin_property_check(struct flb_filter_instance *ins, + struct flb_config *config); +int flb_filter_init(struct flb_config *config, struct flb_filter_instance *ins); +int flb_filter_init_all(struct flb_config *config); +void flb_filter_set_context(struct flb_filter_instance *ins, void *context); +void flb_filter_instance_destroy(struct flb_filter_instance *ins); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_filter_plugin.h b/fluent-bit/include/fluent-bit/flb_filter_plugin.h new file mode 100644 index 00000000..5cc994e2 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_filter_plugin.h @@ -0,0 +1,61 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FILTER_PLUGIN_H +#define FLB_FILTER_PLUGIN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_filter.h> +#include <fluent-bit/flb_log.h> + +#define flb_filter_plugin_log_suppress_check(ins, fmt, ...) \ + flb_log_suppress_check(ins->log_suppress_interval, fmt, ##__VA_ARGS__) + +#define flb_plg_error(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_ERROR)) \ + if (flb_filter_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_ERROR, NULL, 0, "[filter:%s:%s] " fmt, \ + ctx->p->name, flb_filter_name(ctx), ##__VA_ARGS__) + +#define flb_plg_warn(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_WARN)) \ + if (flb_filter_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_WARN, NULL, 0, "[filter:%s:%s] " fmt, \ + ctx->p->name, flb_filter_name(ctx), ##__VA_ARGS__) + +#define flb_plg_info(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_INFO)) \ + if (flb_filter_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_INFO, NULL, 0, "[filter:%s:%s] " fmt, \ + ctx->p->name, flb_filter_name(ctx), ##__VA_ARGS__) + +#define flb_plg_debug(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_DEBUG)) \ + if (flb_filter_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_DEBUG, NULL, 0, "[filter:%s:%s] " fmt, \ + ctx->p->name, flb_filter_name(ctx), ##__VA_ARGS__) + +#define flb_plg_trace(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_TRACE)) \ + if (flb_filter_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_TRACE, NULL, 0, \ + "[filter:%s:%s at %s:%i] " fmt, \ + ctx->p->name, flb_filter_name(ctx), __FLB_FILENAME__, \ + __LINE__, ##__VA_ARGS__) +#endif diff --git a/fluent-bit/include/fluent-bit/flb_fstore.h b/fluent-bit/include/fluent-bit/flb_fstore.h new file mode 100644 index 00000000..8829e6a3 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_fstore.h @@ -0,0 +1,93 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FSTORE_H +#define FLB_FSTORE_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <chunkio/chunkio.h> +#include <chunkio/cio_stats.h> + +#define FLB_FSTORE_LOCK 222 +#define FLB_FSTORE_UNLOCK 333 + +#define FLB_FSTORE_MEM CIO_STORE_MEM +#define FLB_FSTORE_FS CIO_STORE_FS + +struct flb_fstore_file { + flb_sds_t name; /* file name */ + + void *meta_buf; /* copy of metadata content */ + size_t meta_size; /* metadata size */ + void *data; /* opaque data type for user/caller context */ + struct cio_chunk *chunk; /* chunk context */ + struct cio_stream *stream; /* parent stream that owns this file */ + struct mk_list _head; /* link to parent flb_fstore->files */ +}; + +struct flb_fstore_stream { + char *name; /* reference to stream->name */ + flb_sds_t path; /* stream filesystem path */ + struct cio_stream *stream; /* Chunk I/O stream context */ + struct mk_list files; + struct mk_list _head; +}; + +struct flb_fstore { + int store_type; + char *root_path; + struct cio_ctx *cio; /* Chunk I/O context */ + struct mk_list streams; +}; + +struct flb_fstore *flb_fstore_create(char *path, int store_type); + +int flb_fstore_destroy(struct flb_fstore *fs); + +struct flb_fstore_stream *flb_fstore_stream_create(struct flb_fstore *fs, + char *stream_name); +void flb_fstore_stream_destroy(struct flb_fstore_stream *stream, int delete); + +int flb_fstore_file_meta_set(struct flb_fstore *fs, + struct flb_fstore_file *fsf, + void *meta, size_t size); + +int flb_fstore_file_meta_get(struct flb_fstore *fs, + struct flb_fstore_file *fsf); + +struct flb_fstore_file *flb_fstore_file_create(struct flb_fstore *fs, + struct flb_fstore_stream *fs_stream, + char *name, size_t size); + +int flb_fstore_file_content_copy(struct flb_fstore *fs, + struct flb_fstore_file *fsf, + void **out_buf, size_t *out_size); + +int flb_fstore_file_append(struct flb_fstore_file *fsf, void *data, size_t size); +struct flb_fstore_file *flb_fstore_file_get(struct flb_fstore *fs, + struct flb_fstore_stream *fs_stream, + char *name, size_t size); +int flb_fstore_file_inactive(struct flb_fstore *fs, + struct flb_fstore_file *fsf); +int flb_fstore_file_delete(struct flb_fstore *fs, + struct flb_fstore_file *fsf); +void flb_fstore_dump(struct flb_fstore *fs); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_gzip.h b/fluent-bit/include/fluent-bit/flb_gzip.h new file mode 100644 index 00000000..f6b942d4 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_gzip.h @@ -0,0 +1,39 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_GZIP_H +#define FLB_GZIP_H + +#include <fluent-bit/flb_info.h> +#include <stdio.h> + +struct flb_decompression_context; + +int flb_gzip_compress(void *in_data, size_t in_len, + void **out_data, size_t *out_len); +int flb_gzip_uncompress(void *in_data, size_t in_len, + void **out_data, size_t *out_size); + +void *flb_gzip_decompression_context_create(); +void flb_gzip_decompression_context_destroy(void *context); + +int flb_gzip_decompressor_dispatch(struct flb_decompression_context *context, + void *out_data, size_t *out_size); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_hash.h b/fluent-bit/include/fluent-bit/flb_hash.h new file mode 100644 index 00000000..8c1c7bdf --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_hash.h @@ -0,0 +1,89 @@ +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * A wrapper for the OpenSSL SHA512 functions if OpenSSL is available. + * Otherwise, the functions in this header file provide + * the following public domain sha512 hash implementation. + * + * This is based on the musl libc SHA512 implementation. Follow the + * link for the original source code. + * https://git.musl-libc.org/cgit/musl/tree/src/crypt/crypt_sha512.c?h=v1.1.22 + * + * Here is how to use it: + * + * #include <fluent-bit/flb_crypto.h> + * + * void main(void) + * { + * char buf[64]; + * + * result = flb_hash_simple(FLB_CRYPTO_SHA256, + * (unsigned char *) "aiueo", 5, + * buf, sizeof(buf)); + * + * if (result != FLB_CRYPTO_SUCCESS) { + * printf("error\n"); + * return NULL; + * } + * } + */ + +#ifndef FLB_HASH_H +#define FLB_HASH_H + +#include <stdint.h> +#include <openssl/err.h> +#include <openssl/sha.h> +#include <openssl/evp.h> +#include <openssl/hmac.h> +#include <openssl/engine.h> +#include <fluent-bit/flb_crypto_constants.h> + + +struct flb_hash { + EVP_MD_CTX *backend_context; + size_t digest_size; + unsigned long last_error; +}; + +int flb_hash_init(struct flb_hash *context, int digest_type); + +int flb_hash_cleanup(struct flb_hash *context); + +int flb_hash_finalize(struct flb_hash *context, + unsigned char *digest_buffer, + size_t digest_buffer_size); + +int flb_hash_update(struct flb_hash *context, + unsigned char *data, + size_t data_length); + +int flb_hash_simple_batch(int hash_type, + size_t entry_count, + unsigned char **data_entries, + size_t *length_entries, + unsigned char *digest_buffer, + size_t digest_buffer_size); + +int flb_hash_simple(int hash_type, + unsigned char *data, + size_t data_length, + unsigned char *digest_buffer, + size_t digest_buffer_size); + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_hash_table.h b/fluent-bit/include/fluent-bit/flb_hash_table.h new file mode 100644 index 00000000..115088d4 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_hash_table.h @@ -0,0 +1,89 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HASH_TABLE_H +#define FLB_HASH_TABLE_H + +#include <fluent-bit/flb_info.h> +#include <cfl/cfl.h> + +#include <monkey/mk_core.h> + +#include <stdio.h> +#include <stdlib.h> + +/* Eviction modes when the table reach full capacity (if any) */ +#define FLB_HASH_TABLE_EVICT_NONE 0 +#define FLB_HASH_TABLE_EVICT_OLDER 1 +#define FLB_HASH_TABLE_EVICT_LESS_USED 2 +#define FLB_HASH_TABLE_EVICT_RANDOM 3 + +struct flb_hash_table_entry { + time_t created; + uint64_t hits; + uint64_t hash; + char *key; + size_t key_len; + void *val; + ssize_t val_size; + struct flb_hash_table_chain *table; /* link to parent flb_hash_table */ + struct mk_list _head; /* link to flb_hash_table->chains */ + struct mk_list _head_parent; /* link to flb_hash->entries */ +}; + +struct flb_hash_table_chain { + int count; + struct mk_list chains; +}; + +struct flb_hash_table { + int evict_mode; + int max_entries; + int total_count; + int cache_ttl; + size_t size; + struct mk_list entries; + struct flb_hash_table_chain *table; +}; + +struct flb_hash_table *flb_hash_table_create(int evict_mode, size_t size, int max_entries); +struct flb_hash_table *flb_hash_table_create_with_ttl(int cache_ttl, int evict_mode, + size_t size, int max_entries); +void flb_hash_table_destroy(struct flb_hash_table *ht); + +int flb_hash_table_add(struct flb_hash_table *ht, + const char *key, int key_len, + void *val, ssize_t val_size); +int flb_hash_table_get(struct flb_hash_table *ht, + const char *key, int key_len, + void **out_buf, size_t *out_size); + +int flb_hash_table_exists(struct flb_hash_table *ht, uint64_t hash); +int flb_hash_table_get_by_id(struct flb_hash_table *ht, int id, + const char *key, + const char **out_buf, size_t *out_size); + +void *flb_hash_table_get_ptr(struct flb_hash_table *ht, const char *key, int key_len); + +int flb_hash_table_del(struct flb_hash_table *ht, const char *key); + +int flb_hash_table_del_ptr(struct flb_hash_table *ht, const char *key, int key_len, + void *ptr); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_help.h b/fluent-bit/include/fluent-bit/flb_help.h new file mode 100644 index 00000000..06652f53 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_help.h @@ -0,0 +1,45 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HELP_H +#define FLB_HELP_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_custom.h> +#include <fluent-bit/flb_input.h> +#include <fluent-bit/flb_filter.h> +#include <fluent-bit/flb_output.h> + +/* JSON Helper version: current '1' */ +#define FLB_HELP_SCHEMA_VERSION "1" + +#define FLB_HELP_PLUGIN_CUSTOM 0 +#define FLB_HELP_PLUGIN_INPUT 1 +#define FLB_HELP_PLUGIN_OUTPUT 2 +#define FLB_HELP_PLUGIN_FILTER 3 + +int flb_help_custom(struct flb_custom_instance *ins, void **out_buf, size_t *out_size); +int flb_help_input(struct flb_input_instance *ins, void **out_buf, size_t *out_size); +int flb_help_filter(struct flb_filter_instance *ins, void **out_buf, size_t *out_size); +int flb_help_output(struct flb_output_instance *ins, void **out_buf, size_t *out_size); + +flb_sds_t flb_help_build_json_schema(struct flb_config *config); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_hmac.h b/fluent-bit/include/fluent-bit/flb_hmac.h new file mode 100644 index 00000000..73ad2439 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_hmac.h @@ -0,0 +1,69 @@ +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HMAC_H +#define FLB_HMAC_H + +#include <stdint.h> +#include <openssl/err.h> +#include <openssl/sha.h> +#include <openssl/evp.h> +#include <openssl/hmac.h> +#include <openssl/engine.h> +#include <fluent-bit/flb_crypto_constants.h> + +struct flb_hmac { +#if FLB_CRYPTO_OPENSSL_COMPAT_MODE >= 3 + EVP_MAC_CTX *backend_context; + EVP_MAC *mac_algorithm; +#else + HMAC_CTX *backend_context; +#endif + + size_t digest_size; + unsigned long last_error; +}; + +int flb_hmac_init(struct flb_hmac *context, + int algorithm_id, + unsigned char *key, + size_t key_length); + +int flb_hmac_cleanup(struct flb_hmac *context); + +int flb_hmac_finalize(struct flb_hmac *context, + unsigned char *signature_buffer, + size_t signature_buffer_size); + +int flb_hmac_update(struct flb_hmac *context, + unsigned char *data, + size_t data_length); + +int flb_hmac_simple_batch(int hash_type, + unsigned char *key, size_t key_length, + size_t entry_count, + unsigned char **data_entries, + size_t *length_entries, + unsigned char *signature_buffer, + size_t signature_buffer_size); + +int flb_hmac_simple(int hash_type, + unsigned char *key, size_t key_length, + unsigned char *data, size_t data_length, + unsigned char *signature_buffer, + size_t signature_buffer_size); +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_http_client.h b/fluent-bit/include/fluent-bit/flb_http_client.h new file mode 100644 index 00000000..56e7448e --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_http_client.h @@ -0,0 +1,176 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HTTP_CLIENT_H +#define FLB_HTTP_CLIENT_H + +#include <fluent-bit/flb_io.h> +#include <fluent-bit/flb_upstream.h> +#include <fluent-bit/flb_callback.h> + +/* Buffer size */ +#define FLB_HTTP_BUF_SIZE 2048 +#define FLB_HTTP_DATA_SIZE_MAX 4096 +#define FLB_HTTP_DATA_CHUNK 32768 + +/* HTTP Methods */ +#define FLB_HTTP_GET 0 +#define FLB_HTTP_POST 1 +#define FLB_HTTP_PUT 2 +#define FLB_HTTP_HEAD 3 +#define FLB_HTTP_CONNECT 4 +#define FLB_HTTP_PATCH 5 + +/* HTTP Flags */ +#define FLB_HTTP_10 1 +#define FLB_HTTP_11 2 +#define FLB_HTTP_KA 16 + +/* Proxy */ +#define FLB_HTTP_PROXY_NONE 0 +#define FLB_HTTP_PROXY_HTTP 1 +#define FLB_HTTP_PROXY_HTTPS 2 + +/* Internal codes */ +#define FLB_HTTP_ERROR -1 +#define FLB_HTTP_MORE 0 +#define FLB_HTTP_OK 1 +#define FLB_HTTP_NOT_FOUND 2 /* header not found */ + +/* Useful headers */ +#define FLB_HTTP_HEADER_AUTH "Authorization" +#define FLB_HTTP_HEADER_PROXY_AUTH "Proxy-Authorization" +#define FLB_HTTP_HEADER_CONTENT_TYPE "Content-Type" +#define FLB_HTTP_HEADER_CONTENT_ENCODING "Content-Encoding" +#define FLB_HTTP_HEADER_CONNECTION "Connection" +#define FLB_HTTP_HEADER_KA "keep-alive" + +struct flb_http_response { + int status; /* HTTP response status */ + int content_length; /* Content length set by headers */ + int chunked_encoding; /* Chunked transfer encoding ? */ + int connection_close; /* connection: close ? */ + long chunked_cur_size; + long chunked_exp_size; /* expected chunked size */ + char *chunk_processed_end; /* Position to mark last chunk */ + char *headers_end; /* Headers end (\r\n\r\n) */ + + /* Payload: body response: reference to 'data' */ + char *payload; + size_t payload_size; + + /* Buffer to store server response */ + char *data; + size_t data_len; + size_t data_size; + size_t data_size_max; +}; + +/* It hold information about a possible HTTP proxy set by the caller */ +struct flb_http_proxy { + int type; /* One of FLB_HTTP_PROXY_ macros */ + int port; /* TCP Port */ + const char *host; /* Proxy Host */ +}; + +/* HTTP Debug context */ +struct flb_http_debug { + /* HTTP request headers */ + int debug_request_headers; /* debug HTTP request headers */ + void (*cb_debug_request_headers); /* callback to pass raw headers */ + + /* HTTP request payload */ + int debug_request_payload; /* debug HTTP request payload */ + int (*cb_debug_request_payload); +}; + +/* Set a request type */ +struct flb_http_client { + /* Upstream connection */ + struct flb_connection *u_conn; + + /* Request data */ + int method; + int flags; + int header_len; + int header_size; + char *header_buf; + + /* Config */ + int allow_dup_headers; /* allow duplicated headers */ + + /* incoming parameters */ + const char *uri; + const char *query_string; + const char *host; + int port; + + /* payload */ + int body_len; + const char *body_buf; + + struct mk_list headers; + + /* Proxy */ + struct flb_http_proxy proxy; + + /* Response */ + struct flb_http_response resp; + + /* Reference to Callback context */ + void *cb_ctx; +}; + +void flb_http_client_debug(struct flb_http_client *c, + struct flb_callback *cb_ctx); + +struct flb_http_client *flb_http_client(struct flb_connection *u_conn, + int method, const char *uri, + const char *body, size_t body_len, + const char *host, int port, + const char *proxy, int flags); + +int flb_http_add_header(struct flb_http_client *c, + const char *key, size_t key_len, + const char *val, size_t val_len); +flb_sds_t flb_http_get_header(struct flb_http_client *c, + const char *key, size_t key_len); +int flb_http_basic_auth(struct flb_http_client *c, + const char *user, const char *passwd); +int flb_http_proxy_auth(struct flb_http_client *c, + const char *user, const char *passwd); +int flb_http_bearer_auth(struct flb_http_client *c, + const char *token); +int flb_http_set_keepalive(struct flb_http_client *c); +int flb_http_set_content_encoding_gzip(struct flb_http_client *c); +int flb_http_set_callback_context(struct flb_http_client *c, + struct flb_callback *cb_ctx); + +int flb_http_do(struct flb_http_client *c, size_t *bytes); +int flb_http_client_proxy_connect(struct flb_connection *u_conn); +void flb_http_client_destroy(struct flb_http_client *c); +int flb_http_buffer_size(struct flb_http_client *c, size_t size); +size_t flb_http_buffer_available(struct flb_http_client *c); +int flb_http_buffer_increase(struct flb_http_client *c, size_t size, + size_t *out_size); +int flb_http_strip_port_from_host(struct flb_http_client *c); +int flb_http_allow_duplicated_headers(struct flb_http_client *c, int allow); +int flb_http_client_debug_property_is_valid(char *key, char *val); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_http_client_debug.h b/fluent-bit/include/fluent-bit/flb_http_client_debug.h new file mode 100644 index 00000000..e4de7c1b --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_http_client_debug.h @@ -0,0 +1,36 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HTTP_CLIENT_DEBUG_H +#define FLB_HTTP_CLIENT_DEBUG_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_callback.h> + +static inline void flb_http_client_debug_enable(struct flb_http_client *c, + struct flb_callback *cb_ctx) +{ + c->cb_ctx = cb_ctx; +} + +int flb_http_client_debug_setup(struct flb_callback *cb_ctx, + struct mk_list *props); +int flb_http_client_debug_cb(struct flb_http_client *c, char *name); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_http_server.h b/fluent-bit/include/fluent-bit/flb_http_server.h new file mode 100644 index 00000000..2223ba84 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_http_server.h @@ -0,0 +1,32 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit/flb_info.h> +#ifdef FLB_HAVE_HTTP_SERVER + +#ifndef FLB_HTTP_SERVER_H +#define FLB_HTTP_SERVER_H + +#include "http_server/flb_hs.h" +#include "http_server/flb_hs_utils.h" +#include "http_server/flb_hs_endpoints.h" + +#endif + +#endif /* !FLB_HAVE_HTTP_SERVER */ diff --git a/fluent-bit/include/fluent-bit/flb_info.h.in b/fluent-bit/include/fluent-bit/flb_info.h.in new file mode 100644 index 00000000..a89485c6 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_info.h.in @@ -0,0 +1,32 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_INFO_H +#define FLB_INFO_H + +#define STR_HELPER(s) #s +#define STR(s) STR_HELPER(s) + +#define FLB_SOURCE_DIR "@CMAKE_SOURCE_DIR@" + +/* General flags set by CMakeLists.txt */ +@FLB_BUILD_FLAGS@ + +#define FLB_INFO_FLAGS "@FLB_INFO_FLAGS@" +#endif diff --git a/fluent-bit/include/fluent-bit/flb_input.h b/fluent-bit/include/fluent-bit/flb_input.h new file mode 100644 index 00000000..1b4dff82 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_input.h @@ -0,0 +1,739 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_INPUT_H +#define FLB_INPUT_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_engine_macros.h> +#include <fluent-bit/flb_coro.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_network.h> +#include <fluent-bit/flb_mem.h> +#include <fluent-bit/flb_str.h> +#include <fluent-bit/flb_bits.h> +#include <fluent-bit/flb_pipe.h> +#include <fluent-bit/flb_filter.h> +#include <fluent-bit/flb_coro.h> +#include <fluent-bit/flb_mp.h> +#include <fluent-bit/flb_hash_table.h> + +#include <fluent-bit/flb_input_chunk.h> +#include <fluent-bit/flb_input_log.h> +#include <fluent-bit/flb_input_metric.h> +#include <fluent-bit/flb_input_trace.h> +#include <fluent-bit/flb_config_format.h> +#include <fluent-bit/flb_processor.h> + +#ifdef FLB_HAVE_METRICS +#include <fluent-bit/flb_metrics.h> +#endif +#include <fluent-bit/flb_pthread.h> + +#include <cmetrics/cmetrics.h> +#include <monkey/mk_core.h> +#include <msgpack.h> +#include <inttypes.h> + +#define FLB_COLLECT_TIME 1 +#define FLB_COLLECT_FD_EVENT 2 +#define FLB_COLLECT_FD_SERVER 4 + +/* Input plugin flag masks */ +#define FLB_INPUT_NET 4 /* input address may set host and port */ +#define FLB_INPUT_PLUGIN_CORE 0 +#define FLB_INPUT_PLUGIN_PROXY 1 +#define FLB_INPUT_CORO 128 /* plugin requires a thread on callbacks */ +#define FLB_INPUT_PRIVATE 256 /* plugin is not published/exposed */ +#define FLB_INPUT_NOTAG 512 /* plugin might don't have tags */ +#define FLB_INPUT_THREADED 1024 /* plugin must run in a separate thread */ +#define FLB_INPUT_NET_SERVER 2048 /* Input address may set host and port. + * In addition, if TLS is enabled then a + * private key and certificate are required. + */ + +/* Input status */ +#define FLB_INPUT_RUNNING 1 +#define FLB_INPUT_PAUSED 0 + +/* Input plugin event type */ +#define FLB_INPUT_LOGS 0 +#define FLB_INPUT_METRICS 1 +#define FLB_INPUT_TRACES 2 + +struct flb_input_instance; + +struct flb_input_plugin { + /* + * The type defines if this is a core-based plugin or it's handled by + * some specific proxy. + */ + int type; + void *proxy; + + int flags; /* plugin flags */ + + /* The Input name */ + char *name; + + /* Plugin Description */ + char *description; + + struct flb_config_map *config_map; + + /* Initialization */ + int (*cb_init) (struct flb_input_instance *, struct flb_config *, void *); + + /* Pre run */ + int (*cb_pre_run) (struct flb_input_instance *, struct flb_config *, void *); + + /* Collect: every certain amount of time, Fluent Bit trigger this callback */ + int (*cb_collect) (struct flb_input_instance *, struct flb_config *, void *); + + /* + * Flush: each plugin during a collection, it does some buffering, + * when the Flush timer takes place on the Engine, it will trigger + * the cb_flush(...) to obtain the plugin buffer data. This data is + * a MsgPack buffer which will be processed by the Engine and delivered + * to the target output. + */ + + /* Flush a buffer type (raw data) */ + void *(*cb_flush_buf) (void *, size_t *); + + /* Notify that a flush have completed on the collector (buf + iov) */ + void (*cb_flush_end) (void *); + + /* + * Callbacks to notify the plugin when it becomes paused (cannot longer append + * data) and when it can resume operations. + */ + void (*cb_pause) (void *, struct flb_config *); + void (*cb_resume) (void *, struct flb_config *); + + /* + * Optional callback that can be used from a parent caller to ingest + * data into the engine. + */ + int (*cb_ingest) (void *in_context, void *, size_t); + + /* Exit */ + int (*cb_exit) (void *, struct flb_config *); + + /* Destroy */ + void (*cb_destroy) (struct flb_input_plugin *); + + void *instance; + + struct mk_list _head; +}; + +/* + * Each initialized plugin must have an instance, same plugin may be + * loaded more than one time. + * + * An instance try to contain plugin data separating what is fixed data + * and the variable one that is generated when the plugin is invoked. + */ +struct flb_input_instance { + struct mk_event event; /* events handler */ + + struct flb_processor *processor; + + /* + * The instance flags are derived from the fixed plugin flags. This + * is done to offer some flexibility where a plugin instance per + * configuration would like to change some specific behavior. + * + * e.g By default in_tail plugin supports fixed tag, but if a wildcard + * is added to the 'tag', it will instruct to perform dyntag operations + * as the tags will be composed used the file name being watched. + */ + int flags; + + int id; /* instance id */ +#ifdef FLB_HAVE_CHUNK_TRACE + struct flb_chunk_trace_context *chunk_trace_ctxt; + pthread_mutex_t chunk_trace_lock; +#endif /* FLB_HAVE_CHUNK_TRACE */ + int log_level; /* log level for this plugin */ + int log_suppress_interval; /* log suppression interval */ + flb_pipefd_t channel[2]; /* pipe(2) channel */ + int runs_in_coroutine; /* instance runs in coroutine ? */ + char name[32]; /* numbered name (cpu -> cpu.0) */ + char *alias; /* alias name for the instance */ + void *context; /* plugin configuration context */ + flb_pipefd_t ch_events[2]; /* channel for events */ + struct flb_input_plugin *p; /* original plugin */ + + /* Plugin properties */ + char *tag; /* Input tag for routing */ + int tag_len; + int tag_default; /* is it using the default tag? */ + + /* By default all input instances are 'routable' */ + int routable; + + /* flag to pause input when storage is full */ + int storage_pause_on_chunks_overlimit; + + /* + * Input network info: + * + * An input plugin can be specified just using it shortname or using the + * complete network address format, e.g: + * + * $ fluent-bit -i cpu -o plugin://hostname:port/uri + * + * where: + * + * plugin = the output plugin shortname + * name = IP address or hostname of the target + * port = target TCP port + * uri = extra information that may be used by the plugin + */ + struct flb_net_host host; + + /* Reference to struct flb_storage_input context */ + void *storage; + + /* Type of storage: CIO_STORE_FS (filesystem) or CIO_STORE_MEM (memory) */ + int storage_type; + + /* + * Buffers counter: it counts the total of memory used by fixed and dynamic + * message pack buffers used by the input plugin instance. + */ + size_t mem_chunks_size; + size_t mp_total_buf_size; /* FIXME: to be deprecated */ + + /* + * Buffer limit: optional limit set by configuration so this input instance + * cannot exceed more than mp_buf_limit (bytes unit). + * + * As a reference, if an input plugin exceeds the limit, the pause() callback + * will be triggered to notify the input instance it cannot longer append + * more data, on that moment Fluent Bit will avoid to add more records. + * + * When the buffer size goes down (because data was flushed), a resume() + * callback will be triggered, from that moment the plugin can append more + * data. + */ + size_t mem_buf_limit; + + /* + * Define the buffer status: + * + * - FLB_INPUT_RUNNING -> can append more data + * - FLB_INPUT_PAUSED -> cannot append data + */ + int mem_buf_status; + + /* + * Define the buffer status: + * + * - FLB_INPUT_RUNNING -> can append more data + * - FLB_INPUT_PAUSED -> cannot append data + */ + int storage_buf_status; + + /* + * Optional data passed to the plugin, this info is useful when + * running Fluent Bit in library mode and the target plugin needs + * some specific data from it caller. + */ + void *data; + + struct mk_list *config_map; /* configuration map */ + + struct mk_list _head; /* link to config->inputs */ + + struct mk_list routes_direct; /* direct routes set by API */ + struct mk_list routes; /* flb_router_path's list */ + struct mk_list properties; /* properties / configuration */ + struct mk_list collectors; /* collectors */ + + /* Storage Chunks */ + struct mk_list chunks; /* linked list of all chunks */ + + /* + * The following list helps to separate the chunks per its + * status, it can be 'up' or 'down'. + */ + struct mk_list chunks_up; /* linked list of all chunks up */ + struct mk_list chunks_down; /* linked list of all chunks down */ + + /* + * Every co-routine created by the engine when flushing data, it's + * linked into this list header. + */ + struct mk_list tasks; + + /* co-routines for input plugins with FLB_INPUT_CORO flag */ + int input_coro_id; + struct mk_list input_coro_list; + struct mk_list input_coro_list_destroy; + +#ifdef FLB_HAVE_METRICS + /* old metrics API */ + struct flb_metrics *metrics; /* metrics */ +#endif + + + /* is the plugin running in a separate thread ? */ + int is_threaded; + struct flb_input_thread_instance *thi; + + /* + * ring buffer: the ring buffer is used by the instance if is running + * in threaded mode; so when registering a msgpack buffer this happens + * in the ring buffer. + */ + struct flb_ring_buffer *rb; + + /* List of upstreams */ + struct mk_list upstreams; + + /* List of downstreams */ + struct mk_list downstreams; + + /* + * CMetrics + * -------- + * + * All metrics available for an input plugin instance. + */ + struct cmt *cmt; /* parent context */ + struct cmt_counter *cmt_bytes; /* metric: input_bytes_total */ + struct cmt_counter *cmt_records; /* metric: input_records_total */ + + /* is the input instance overlimit ?: 1 or 0 */ + struct cmt_gauge *cmt_storage_overlimit; + + /* memory bytes used by chunks */ + struct cmt_gauge *cmt_storage_memory_bytes; + + /* total number of chunks */ + struct cmt_gauge *cmt_storage_chunks; + + /* total number of chunks up in memory */ + struct cmt_gauge *cmt_storage_chunks_up; + + /* total number of chunks down */ + struct cmt_gauge *cmt_storage_chunks_down; + + /* number of chunks in a busy state */ + struct cmt_gauge *cmt_storage_chunks_busy; + + /* total bytes used by chunks in a busy state */ + struct cmt_gauge *cmt_storage_chunks_busy_bytes; + + /* memory ring buffer (memrb) metrics */ + struct cmt_counter *cmt_memrb_dropped_chunks; + struct cmt_counter *cmt_memrb_dropped_bytes; + + /* + * Indexes for generated chunks: simple hash tables that keeps the latest + * available chunks for writing data operations. This optimizes the + * lookup for candidates chunks to write data. + */ + struct flb_hash_table *ht_log_chunks; + struct flb_hash_table *ht_metric_chunks; + struct flb_hash_table *ht_trace_chunks; + + /* TLS settings */ + int use_tls; /* bool, try to use TLS for I/O */ + int tls_verify; /* Verify certs (default: true) */ + int tls_debug; /* mbedtls debug level */ + char *tls_vhost; /* Virtual hostname for SNI */ + char *tls_ca_path; /* Path to certificates */ + char *tls_ca_file; /* CA root cert */ + char *tls_crt_file; /* Certificate */ + char *tls_key_file; /* Cert Key */ + char *tls_key_passwd; /* Cert Key Password */ + + struct mk_list *tls_config_map; + +#ifdef FLB_HAVE_TLS + struct flb_tls *tls; +#else + void *tls; +#endif + + /* General network options like timeouts and keepalive */ + struct flb_net_setup net_setup; + struct mk_list *net_config_map; + struct mk_list net_properties; + + /* Keep a reference to the original context this instance belongs to */ + struct flb_config *config; +}; + +struct flb_input_collector { + struct mk_event event; + struct mk_event_loop *evl; /* event loop */ + + int id; /* collector id */ + int type; /* collector type */ + int running; /* is running ? (True/False) */ + + /* FLB_COLLECT_FD_EVENT */ + flb_pipefd_t fd_event; /* fd being watched */ + + /* FLB_COLLECT_TIME */ + flb_pipefd_t fd_timer; /* timer fd */ + time_t seconds; /* expire time in seconds */ + long nanoseconds; /* expire nanoseconds */ + + /* Callback */ + int (*cb_collect) (struct flb_input_instance *, + struct flb_config *, void *); + + + /* General references */ + struct flb_input_instance *instance; /* plugin instance */ + struct mk_list _head; /* link to instance collectors */ +}; + +struct flb_input_coro { + int id; /* id returned from flb_input_coro_id_get() */ + time_t start_time; /* start time */ + time_t end_time; /* end time */ + struct flb_input_instance *ins; /* parent input instance */ + struct flb_coro *coro; /* coroutine context */ + struct flb_config *config; /* FLB context */ + struct mk_list _head; /* link to list on input_instance->coros */ +}; + +int flb_input_coro_id_get(struct flb_input_instance *ins); + +static FLB_INLINE +struct flb_input_coro *flb_input_coro_create(struct flb_input_instance *ins, + struct flb_config *config) +{ + struct flb_coro *coro; + struct flb_input_coro *input_coro; + + /* input_coro context */ + input_coro = (struct flb_input_coro *) flb_calloc(1, + sizeof(struct flb_input_coro)); + if (!input_coro) { + flb_errno(); + return NULL; + } + + /* coroutine context */ + coro = flb_coro_create(input_coro); + if (!coro) { + flb_free(input_coro); + return NULL; + } + + input_coro->id = flb_input_coro_id_get(ins); + input_coro->ins = ins; + input_coro->start_time = time(NULL); + input_coro->coro = coro; + input_coro->config = config; + + mk_list_add(&input_coro->_head, &ins->input_coro_list); + + return input_coro; +} + +struct flb_libco_in_params { + struct flb_config *config; + struct flb_input_collector *coll; + struct flb_coro *coro; +}; + +extern pthread_key_t libco_in_param_key; +extern struct flb_libco_in_params libco_in_param; +void flb_input_coro_prepare_destroy(struct flb_input_coro *input_coro); + +static FLB_INLINE void input_params_set(struct flb_coro *coro, + struct flb_input_collector *coll, + struct flb_config *config, + void *context) +{ + struct flb_libco_in_params *params; + + params = pthread_getspecific(libco_in_param_key); + if (params == NULL) { + params = flb_calloc(1, sizeof(struct flb_libco_in_params)); + if (params == NULL) { + flb_errno(); + return; + } + pthread_setspecific(libco_in_param_key, params); + } + + /* Set callback parameters */ + params->coll = coll; + params->config = config; + params->coro = coro; + co_switch(coro->callee); +} + +static FLB_INLINE void input_pre_cb_collect(void) +{ + struct flb_input_collector *coll; + struct flb_config *config; + struct flb_coro *coro; + struct flb_libco_in_params *params; + + params = pthread_getspecific(libco_in_param_key); + if (params == NULL) { + params = flb_calloc(1, sizeof(struct flb_libco_in_params)); + if (params == NULL) { + flb_errno(); + return; + } + pthread_setspecific(libco_in_param_key, params); + } + coll = params->coll; + config = params->config; + coro = params->coro; + + co_switch(coro->caller); + coll->cb_collect(coll->instance, config, coll->instance->context); +} + +static FLB_INLINE void flb_input_coro_resume(struct flb_input_coro *input_coro) +{ + flb_coro_resume(input_coro->coro); +} + +static void libco_in_param_key_destroy(void *data) +{ + struct flb_libco_inparams *params = (struct flb_libco_inparams*)data; + + flb_free(params); +} + +static FLB_INLINE +struct flb_input_coro *flb_input_coro_collect(struct flb_input_collector *coll, + struct flb_config *config) +{ + size_t stack_size; + struct flb_coro *coro; + struct flb_input_coro *input_coro; + + input_coro = flb_input_coro_create(coll->instance, config); + if (!input_coro) { + return NULL; + } + + pthread_key_create(&libco_in_param_key, libco_in_param_key_destroy); + + coro = input_coro->coro; + if (!coro) { + return NULL; + } + + coro->caller = co_active(); + coro->callee = co_create(config->coro_stack_size, + input_pre_cb_collect, &stack_size); + +#ifdef FLB_HAVE_VALGRIND + coro->valgrind_stack_id = VALGRIND_STACK_REGISTER(coro->callee, + ((char *)coro->callee) + stack_size); +#endif + + /* Set parameters */ + input_params_set(coro, coll, config, coll->instance->context); + return input_coro; +} + +static FLB_INLINE int flb_input_is_threaded(struct flb_input_instance *ins) +{ + return ins->is_threaded; +} + +/* + * This function is used by the output plugins to return. It's mandatory + * as it will take care to signal the event loop letting know the flush + * callback has done. + * + * The signal emitted indicate the 'Task' number that have finished plus + * a return value. The return value is either FLB_OK, FLB_RETRY or FLB_ERROR. + * + * If the caller have requested an FLB_RETRY, it will be issued depending on the + * number of retries, if it has exceeded the 'retry_limit' option, an FLB_ERROR + * will be returned instead. + */ +static inline void flb_input_return(struct flb_coro *coro) { + int n; + uint64_t val; + struct flb_input_coro *input_coro; + struct flb_input_instance *ins; + + input_coro = (struct flb_input_coro *) coro->data; + ins = input_coro->ins; + + /* + * Message the event loop by identifying the message coming from an input + * coroutine and passing the input plugin ID that triggered the event. + */ + val = FLB_BITS_U64_SET(FLB_ENGINE_IN_CORO, ins->id); + n = flb_pipe_w(ins->ch_events[1], (void *) &val, sizeof(val)); + if (n == -1) { + flb_errno(); + } + + flb_input_coro_prepare_destroy(input_coro); +} + +static inline void flb_input_return_do(int ret) { + struct flb_coro *coro = flb_coro_get(); + + flb_input_return(coro); + flb_coro_yield(coro, FLB_TRUE); +} + +#define FLB_INPUT_RETURN(x) \ + flb_input_return_do(x); \ + return x; + +static inline int flb_input_buf_paused(struct flb_input_instance *i) +{ + if (i->mem_buf_status == FLB_INPUT_PAUSED) { + return FLB_TRUE; + } + if (i->storage_buf_status == FLB_INPUT_PAUSED) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static inline int flb_input_config_map_set(struct flb_input_instance *ins, + void *context) +{ + int ret; + + /* Process normal properties */ + if (ins->config_map) { + ret = flb_config_map_set(&ins->properties, ins->config_map, context); + + if (ret == -1) { + return -1; + } + } + + /* Net properties */ + if (ins->net_config_map) { + ret = flb_config_map_set(&ins->net_properties, ins->net_config_map, + &ins->net_setup); + if (ret == -1) { + return -1; + } + } + + return ret; +} + +int flb_input_register_all(struct flb_config *config); +struct flb_input_instance *flb_input_new(struct flb_config *config, + const char *input, void *data, + int public_only); +struct flb_input_instance *flb_input_get_instance(struct flb_config *config, + int ins_id); + +int flb_input_set_property(struct flb_input_instance *ins, + const char *k, const char *v); +const char *flb_input_get_property(const char *key, + struct flb_input_instance *ins); +#ifdef FLB_HAVE_METRICS +void *flb_input_get_cmt_instance(struct flb_input_instance *ins); +#endif + +int flb_input_check(struct flb_config *config); +void flb_input_set_context(struct flb_input_instance *ins, void *context); +int flb_input_channel_init(struct flb_input_instance *ins); + + +int flb_input_collector_start(int coll_id, struct flb_input_instance *ins); +int flb_input_collectors_start(struct flb_config *config); +int flb_input_collector_pause(int coll_id, struct flb_input_instance *ins); +int flb_input_collector_resume(int coll_id, struct flb_input_instance *ins); +int flb_input_collector_delete(int coll_id, struct flb_input_instance *ins); +int flb_input_collector_destroy(struct flb_input_collector *coll); +int flb_input_collector_fd(flb_pipefd_t fd, struct flb_config *config); +struct mk_event *flb_input_collector_get_event(int coll_id, + struct flb_input_instance *ins); +int flb_input_set_collector_time(struct flb_input_instance *ins, + int (*cb_collect) (struct flb_input_instance *, + struct flb_config *, void *), + time_t seconds, + long nanoseconds, + struct flb_config *config); +int flb_input_set_collector_event(struct flb_input_instance *ins, + int (*cb_collect) (struct flb_input_instance *, + struct flb_config *, void *), + flb_pipefd_t fd, + struct flb_config *config); +int flb_input_set_collector_socket(struct flb_input_instance *ins, + int (*cb_new_connection) (struct flb_input_instance *, + struct flb_config *, + void*), + flb_pipefd_t fd, + struct flb_config *config); +int flb_input_collector_running(int coll_id, struct flb_input_instance *ins); +int flb_input_coro_id_get(struct flb_input_instance *ins); +int flb_input_coro_finished(struct flb_config *config, int ins_id); + +int flb_input_instance_init(struct flb_input_instance *ins, + struct flb_config *config); +void flb_input_instance_exit(struct flb_input_instance *ins, + struct flb_config *config); +void flb_input_instance_destroy(struct flb_input_instance *ins); + +int flb_input_net_property_check(struct flb_input_instance *ins, + struct flb_config *config); +int flb_input_plugin_property_check(struct flb_input_instance *ins, + struct flb_config *config); + +int flb_input_init_all(struct flb_config *config); +void flb_input_pre_run_all(struct flb_config *config); +void flb_input_exit_all(struct flb_config *config); + +void *flb_input_flush(struct flb_input_instance *ins, size_t *size); + +int flb_input_test_pause_resume(struct flb_input_instance *ins, int sleep_seconds); +int flb_input_pause(struct flb_input_instance *ins); +int flb_input_pause_all(struct flb_config *config); +int flb_input_resume(struct flb_input_instance *ins); + +const char *flb_input_name(struct flb_input_instance *ins); +int flb_input_name_exists(const char *name, struct flb_config *config); + +void flb_input_net_default_listener(const char *listen, int port, + struct flb_input_instance *ins); + +int flb_input_log_check(struct flb_input_instance *ins, int l); + +struct mk_event_loop *flb_input_event_loop_get(struct flb_input_instance *ins); +int flb_input_upstream_set(struct flb_upstream *u, struct flb_input_instance *ins); +int flb_input_downstream_set(struct flb_downstream *stream, + struct flb_input_instance *ins); + +/* processors */ +int flb_input_instance_processors_load(struct flb_input_instance *ins, struct flb_cf_group *processors); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_input_chunk.h b/fluent-bit/include/fluent-bit/flb_input_chunk.h new file mode 100644 index 00000000..d535c7cc --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_input_chunk.h @@ -0,0 +1,126 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_INPUT_CHUNK_H +#define FLB_INPUT_CHUNK_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_routes_mask.h> + +#include <monkey/mk_core.h> +#include <msgpack.h> + +/* + * This variable defines a 'hint' size for new Chunks created, this + * value is passed to Chunk I/O. + */ +#define FLB_INPUT_CHUNK_SIZE 262144 /* 256KB (hint) */ +/* + * Defines a maximum size for a Chunk in the file system: note that despite + * this is considered a limit, a Chunk size might get greater than this. + */ +#define FLB_INPUT_CHUNK_FS_MAX_SIZE 2048000 /* 2MB */ + +/* Number of bytes reserved for Metadata Header on Chunks */ +#define FLB_INPUT_CHUNK_META_HEADER 4 + +/* Chunks magic bytes (starting from Fluent Bit v1.8.10) */ +#define FLB_INPUT_CHUNK_MAGIC_BYTE_0 (unsigned char) 0xF1 +#define FLB_INPUT_CHUNK_MAGIC_BYTE_1 (unsigned char) 0x77 + +/* Chunk types: Log, Metrics and Traces are supported */ +#define FLB_INPUT_CHUNK_TYPE_LOGS 0 +#define FLB_INPUT_CHUNK_TYPE_METRICS 1 +#define FLB_INPUT_CHUNK_TYPE_TRACES 2 + +#ifdef FLB_HAVE_CHUNK_TRACE +#define FLB_INPUT_CHUNK_HAS_TRACE 1 << 31 +#endif /* FLB_HAVE_CHUNK_TRACE */ + +/* Max length for Tag */ +#define FLB_INPUT_CHUNK_TAG_MAX (65535 - FLB_INPUT_CHUNK_META_HEADER) + +struct flb_input_chunk { + int event_type; /* chunk type: logs, metrics or traces */ + bool fs_counted; + int busy; /* buffer is being flushed */ + int fs_backlog; /* chunk originated from fs backlog */ + int sp_done; /* sp already processed this chunk */ +#ifdef FLB_HAVE_METRICS + int total_records; /* total records in the chunk */ + int added_records; /* recently added records */ +#endif + void *chunk; /* context of struct cio_chunk */ + off_t stream_off; /* stream offset */ + msgpack_packer mp_pck; /* msgpack packer */ + struct flb_input_instance *in; /* reference to parent input instance */ + struct flb_task *task; /* reference to the outgoing task */ +#ifdef FLB_HAVE_CHUNK_TRACE + struct flb_chunk_trace *trace; +#endif /* FLB_HAVE_CHUNK_TRACE */ + uint64_t routes_mask + [FLB_ROUTES_MASK_ELEMENTS]; /* track the output plugins the chunk routes to */ + struct mk_list _head; +}; + +struct flb_input_chunk *flb_input_chunk_create(struct flb_input_instance *in, int event_type, + const char *tag, int tag_len); +int flb_input_chunk_destroy(struct flb_input_chunk *ic, int del); +void flb_input_chunk_destroy_all(struct flb_input_instance *in); +int flb_input_chunk_destroy_corrupted(struct flb_input_chunk *ic, + const char *tag_buf, int tag_len, + int del); +int flb_input_chunk_write(void *data, const char *buf, size_t len); +int flb_input_chunk_write_at(void *data, off_t offset, + const char *buf, size_t len); +int flb_input_chunk_append_obj(struct flb_input_instance *in, + const char *tag, int tag_len, + msgpack_object data); +int flb_input_chunk_append_raw(struct flb_input_instance *in, + int event_type, + size_t records, + const char *tag, size_t tag_len, + const void *buf, size_t buf_size); + +const void *flb_input_chunk_flush(struct flb_input_chunk *ic, size_t *size); +int flb_input_chunk_release_lock(struct flb_input_chunk *ic); +flb_sds_t flb_input_chunk_get_name(struct flb_input_chunk *ic); +int flb_input_chunk_get_event_type(struct flb_input_chunk *ic); + +int flb_input_chunk_get_tag(struct flb_input_chunk *ic, + const char **tag_buf, int *tag_len); + +void flb_input_chunk_ring_buffer_cleanup(struct flb_input_instance *ins); +void flb_input_chunk_ring_buffer_collector(struct flb_config *ctx, void *data); +ssize_t flb_input_chunk_get_size(struct flb_input_chunk *ic); +size_t flb_input_chunk_set_limits(struct flb_input_instance *in); +size_t flb_input_chunk_total_size(struct flb_input_instance *in); +struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, + int event_type, + void *chunk); +int flb_input_chunk_set_up_down(struct flb_input_chunk *ic); +int flb_input_chunk_set_up(struct flb_input_chunk *ic); +int flb_input_chunk_down(struct flb_input_chunk *ic); +int flb_input_chunk_is_up(struct flb_input_chunk *ic); +void flb_input_chunk_update_output_instances(struct flb_input_chunk *ic, + size_t chunk_size); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_input_log.h b/fluent-bit/include/fluent-bit/flb_input_log.h new file mode 100644 index 00000000..80dab8f4 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_input_log.h @@ -0,0 +1,41 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_INPUT_LOG_H +#define FLB_INPUT_LOG_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_input.h> + +int flb_input_log_append(struct flb_input_instance *ins, + const char *tag, size_t tag_len, + const void *buf, size_t buf_size); + +int flb_input_log_append_records(struct flb_input_instance *ins, + size_t records, + const char *tag, size_t tag_len, + const void *buf, size_t buf_size); + +int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins, + size_t processor_starting_stage, + const char *tag, + size_t tag_len, + const void *buf, + size_t buf_size); +#endif diff --git a/fluent-bit/include/fluent-bit/flb_input_metric.h b/fluent-bit/include/fluent-bit/flb_input_metric.h new file mode 100644 index 00000000..7a7c1d71 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_input_metric.h @@ -0,0 +1,36 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_INPUT_METRIC_H +#define FLB_INPUT_METRIC_H + +#include <fluent-bit/flb_info.h> +#include <cmetrics/cmetrics.h> +#include <cmetrics/cmt_encode_msgpack.h> + +int flb_input_metrics_append(struct flb_input_instance *ins, + const char *tag, size_t tag_len, + struct cmt *cmt); + +int flb_input_metrics_append_skip_processor_stages( + struct flb_input_instance *ins, + size_t processor_starting_stage, + const char *tag, size_t tag_len, + struct cmt *cmt); +#endif diff --git a/fluent-bit/include/fluent-bit/flb_input_plugin.h b/fluent-bit/include/fluent-bit/flb_input_plugin.h new file mode 100644 index 00000000..a81e9390 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_input_plugin.h @@ -0,0 +1,63 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_INPUT_PLUGIN_H +#define FLB_INPUT_PLUGIN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_input.h> +#include <fluent-bit/flb_input_metric.h> +#include <fluent-bit/flb_log.h> + +#define flb_input_plugin_log_suppress_check(ins, fmt, ...) \ + flb_log_suppress_check(ins->log_suppress_interval, fmt, ##__VA_ARGS__) + + +#define flb_plg_error(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_ERROR)) \ + if (flb_input_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_ERROR, NULL, 0, "[input:%s:%s] " fmt, \ + ctx->p->name, flb_input_name(ctx), ##__VA_ARGS__) + +#define flb_plg_warn(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_WARN)) \ + if (flb_input_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_WARN, NULL, 0, "[input:%s:%s] " fmt, \ + ctx->p->name, flb_input_name(ctx), ##__VA_ARGS__) + +#define flb_plg_info(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_INFO)) \ + if (flb_input_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_INFO, NULL, 0, "[input:%s:%s] " fmt, \ + ctx->p->name, flb_input_name(ctx), ##__VA_ARGS__) + +#define flb_plg_debug(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_DEBUG)) \ + if (flb_input_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_DEBUG, NULL, 0, "[input:%s:%s] " fmt, \ + ctx->p->name, flb_input_name(ctx), ##__VA_ARGS__) + +#define flb_plg_trace(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_TRACE)) \ + if (flb_input_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_TRACE, NULL, 0, \ + "[input:%s:%s at %s:%i] " fmt, \ + ctx->p->name, flb_input_name(ctx), __FLB_FILENAME__, \ + __LINE__, ##__VA_ARGS__) +#endif diff --git a/fluent-bit/include/fluent-bit/flb_input_thread.h b/fluent-bit/include/fluent-bit/flb_input_thread.h new file mode 100644 index 00000000..abc16123 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_input_thread.h @@ -0,0 +1,107 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_INPUT_THREAD_H +#define FLB_INPUT_THREAD_H + +#include <fluent-bit/flb_input.h> +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_pipe.h> +#include <fluent-bit/flb_thread_pool.h> +#include <mpack/mpack.h> + +#define BUFFER_SIZE 65535 + +/* Message from parent to child thread */ +#define FLB_INPUT_THREAD_TO_PARENT (uint32_t) 1 +#define FLB_INPUT_THREAD_TO_THREAD (uint32_t) 2 + +/* Event types from parent to child thread */ +#define FLB_INPUT_THREAD_PAUSE (uint32_t) 1 +#define FLB_INPUT_THREAD_RESUME (uint32_t) 2 +#define FLB_INPUT_THREAD_EXIT (uint32_t) 3 +#define FLB_INPUT_THREAD_START_COLLECTORS (uint32_t) 4 +#define FLB_INPUT_THREAD_OK (uint32_t) 5 +#define FLB_INPUT_THREAD_ERROR (uint32_t) 6 +#define FLB_INPUT_THREAD_NOT_READY (uint32_t) 7 + +typedef void (*flb_input_thread_cb) (int write_fd, void *data); + +struct flb_input_thread { + pthread_mutex_t mutex; /* mutex used to synchronize the "exit" flag */ + bool exit; /* flag set by the main thread to tell worker to exit */ + FILE *write_file; /* std FILE wrapper around write fd */ + mpack_writer_t writer; /* mpack writer to serialize events */ + pthread_t thread; /* thread producing input */ + flb_pipefd_t read, write; /* pipe read/write fds */ + int coll_fd; /* collector fd */ + flb_input_thread_cb callback; /* user callback to run in the thread */ + void *data; /* user data passed to the callback */ + char buf[BUFFER_SIZE]; /* temporary buffer for incomplete msgpack data */ + size_t bufpos; /* current offset in the msgpack buffer */ +}; + +struct flb_input_thread_instance { + struct mk_event event; /* event context to associate events */ + struct mk_event event_local; /* local events inside the thread/event loop */ + struct mk_event_loop *evl; /* thread event loop context */ + flb_pipefd_t ch_parent_events[2]; /* communication between parent and thread */ + flb_pipefd_t ch_thread_events[2]; /* local messages in the thread event loop */ + struct flb_input_instance *ins; /* output plugin instance */ + struct flb_tp *tp; + struct flb_tp_thread *th; + struct flb_config *config; + + /* pthread initialization helpers for synchronization */ + int init_status; + pthread_mutex_t init_mutex; + pthread_cond_t init_condition; + + /* + * In multithread mode, we move some contexts to independent references per thread + * so we can avoid to have shared resources and mutexes. + * + * The following 'coro' fields maintains a state of co-routines inside the thread + * event loop. + * + * note: in single-thread mode, the same fields are in 'struct flb_inpu_instance'. + */ + int input_coro_id; + struct mk_list input_coro_list; + struct mk_list input_coro_list_destroy; +}; + +int flb_input_thread_instance_init(struct flb_config *config, + struct flb_input_instance *ins); +int flb_input_thread_instance_pre_run(struct flb_config *config, struct flb_input_instance *ins); + +int flb_input_thread_instance_pause(struct flb_input_instance *ins); +int flb_input_thread_instance_resume(struct flb_input_instance *ins); +int flb_input_thread_instance_exit(struct flb_input_instance *ins); + +int flb_input_thread_collectors_signal_start(struct flb_input_instance *ins); +int flb_input_thread_collectors_signal_wait(struct flb_input_instance *ins); +int flb_input_thread_collectors_start(struct flb_input_instance *ins); + +int flb_input_thread_init_fail(struct flb_input_instance *ins); +int flb_input_thread_is_ready(struct flb_input_instance *ins); +int flb_input_thread_wait_until_is_ready(struct flb_input_instance *ins); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_input_trace.h b/fluent-bit/include/fluent-bit/flb_input_trace.h new file mode 100644 index 00000000..41177f88 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_input_trace.h @@ -0,0 +1,36 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_INPUT_TRACE_H +#define FLB_INPUT_TRACE_H + +#include <fluent-bit/flb_info.h> +#include <ctraces/ctraces.h> + +int flb_input_trace_append(struct flb_input_instance *ins, + const char *tag, size_t tag_len, + struct ctrace *ctr); + +int flb_input_trace_append_skip_processor_stages( + struct flb_input_instance *ins, + size_t processor_starting_stage, + const char *tag, size_t tag_len, + struct ctrace *ctr); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_intermediate_metric.h b/fluent-bit/include/fluent-bit/flb_intermediate_metric.h new file mode 100644 index 00000000..594de9b9 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_intermediate_metric.h @@ -0,0 +1,48 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Fluent Bit intermediate metric representation */ + +#ifndef FLB_INTERMEDIATE_METRIC_H +#define FLB_INTERMEDIATE_METRIC_H + +#include <fluent-bit/flb_time.h> +#include <monkey/mk_core.h> +#include <msgpack.h> + +/* Metric Type- Gague or Counter */ +#define GAUGE 1 +#define COUNTER 2 + +/* Metric Unit */ +#define PERCENT "Percent" +#define BYTES "Bytes" + +struct flb_intermediate_metric +{ + msgpack_object key; + msgpack_object value; + int metric_type; + const char *metric_unit; + struct flb_time timestamp; + + struct mk_list _head; +}; + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_io.h b/fluent-bit/include/fluent-bit/flb_io.h new file mode 100644 index 00000000..90571bec --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_io.h @@ -0,0 +1,59 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IO_H +#define FLB_IO_H + +#include <monkey/mk_core.h> + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_coro.h> + +/* Coroutine status 'flb_coro.status' */ +#define FLB_IO_CONNECT 0 /* thread issue a connection request */ +#define FLB_IO_WRITE 1 /* thread wants to write() data */ + +/* Network operation modes */ +#define FLB_IO_TCP 1 /* use plain TCP */ +#define FLB_IO_TLS 2 /* use TLS/SSL layer */ +#define FLB_IO_OPT_TLS 4 /* use TCP and optional TLS */ +#define FLB_IO_ASYNC 8 /* use async mode (depends on event loop) */ +#define FLB_IO_TCP_KA 16 /* use async mode (depends on event loop) */ + +/* Other features */ +#define FLB_IO_IPV6 32 /* network I/O uses IPv6 */ + +struct flb_connection; + +int flb_io_net_accept(struct flb_connection *connection, + struct flb_coro *th); + +int flb_io_net_connect(struct flb_connection *u_conn, + struct flb_coro *th); + +int flb_io_net_write(struct flb_connection *connection, const void *data, + size_t len, size_t *out_len); + +ssize_t flb_io_net_read(struct flb_connection *connection, void *buf, size_t len); + +int flb_io_fd_write(int fd, const void *data, size_t len, size_t *out_len); + +ssize_t flb_io_fd_read(int fd, void *buf, size_t len); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_jsmn.h b/fluent-bit/include/fluent-bit/flb_jsmn.h new file mode 100644 index 00000000..319388e6 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_jsmn.h @@ -0,0 +1,26 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_JSMN_H +#define FLB_JSMN_H + +#define JSMN_HEADER +#include <jsmn/jsmn.h> + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_kafka.h b/fluent-bit/include/fluent-bit/flb_kafka.h new file mode 100644 index 00000000..7f12f345 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_kafka.h @@ -0,0 +1,43 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_KAFKA_H +#define FLB_KAFKA_H + +#include <monkey/mk_core.h> +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> + +#include <rdkafka.h> + +#define FLB_KAFKA_BROKERS "127.0.0.1" +#define FLB_KAFKA_TOPIC "fluent-bit" + +struct flb_kafka { + rd_kafka_t *rk; + char *brokers; +}; + +rd_kafka_conf_t *flb_kafka_conf_create(struct flb_kafka *kafka, + struct mk_list *properties, + int with_group_id); +rd_kafka_topic_partition_list_t *flb_kafka_parse_topics(const char *topics_str); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_kernel.h b/fluent-bit/include/fluent-bit/flb_kernel.h new file mode 100644 index 00000000..116b4a79 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_kernel.h @@ -0,0 +1,40 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_KERNEL_H +#define FLB_KERNEL_H + +#include <inttypes.h> +#include <monkey/mk_core.h> + +/* Numeric kernel version */ +#define FLB_KERNEL_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c)) + +struct flb_kernel { + uint8_t minor; + uint8_t major; + uint8_t patch; + uint32_t n_version; + mk_ptr_t s_version; +}; + +struct flb_kernel *flb_kernel_info(); +void flb_kernel_destroy(struct flb_kernel *kernel); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_kv.h b/fluent-bit/include/fluent-bit/flb_kv.h new file mode 100644 index 00000000..a6b23530 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_kv.h @@ -0,0 +1,43 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_KV_H +#define FLB_KV_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <monkey/mk_core.h> + +struct flb_kv { + flb_sds_t key; + flb_sds_t val; + struct mk_list _head; +}; + +void flb_kv_init(struct mk_list *list); +struct flb_kv *flb_kv_item_create_len(struct mk_list *list, + char *k_buf, size_t k_len, + char *v_buf, size_t v_len); +struct flb_kv *flb_kv_item_create(struct mk_list *list, + char *k_buf, char *v_buf); +void flb_kv_item_destroy(struct flb_kv *kv); +void flb_kv_release(struct mk_list *list); +const char *flb_kv_get_key_value(const char *key, struct mk_list *list); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_langinfo.h b/fluent-bit/include/fluent-bit/flb_langinfo.h new file mode 100644 index 00000000..6da30682 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_langinfo.h @@ -0,0 +1,144 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This is a dumb implementation of langinfo.h that only supports LC_TIME + * and "C" locale. We use this implemenatation to support locale-dependent + * specifiers of strptime(3) on Windows. + */ + +#ifndef FLB_LANGINFO_H +#define FLB_LANGINFO_H + +#ifndef _MSC_VER +#include <langinfo.h> +#else + +typedef int nl_item; + +#define D_T_FMT 0x00 +#define D_FMT 0x01 +#define T_FMT 0x02 + +#define DAY_1 0x03 +#define DAY_2 0x04 +#define DAY_3 0x05 +#define DAY_4 0x06 +#define DAY_5 0x07 +#define DAY_6 0x08 +#define DAY_7 0x09 + +#define ABDAY_1 0x0A +#define ABDAY_2 0x0B +#define ABDAY_3 0x0C +#define ABDAY_4 0x0D +#define ABDAY_5 0x0E +#define ABDAY_6 0x0F +#define ABDAY_7 0x10 + +#define MON_1 0x11 +#define MON_2 0x12 +#define MON_3 0x13 +#define MON_4 0x14 +#define MON_5 0x15 +#define MON_6 0x16 +#define MON_7 0x17 +#define MON_8 0x18 +#define MON_9 0x19 +#define MON_10 0x1A +#define MON_11 0x1B +#define MON_12 0x1C + +#define ABMON_1 0x1D +#define ABMON_2 0x1E +#define ABMON_3 0x1F +#define ABMON_4 0x20 +#define ABMON_5 0x21 +#define ABMON_6 0x22 +#define ABMON_7 0x23 +#define ABMON_8 0x24 +#define ABMON_9 0x25 +#define ABMON_10 0x26 +#define ABMON_11 0x27 +#define ABMON_12 0x28 + +#define AM_STR 0x29 +#define PM_STR 0x2A +#define T_FMT_AMPM 0x2B + +static const char *lc_time_c[] = { + "%a %b %e %H:%M:%S %Y", /* D_T_FMT */ + "%m/%d/%y", /* D_FMT */ + "%H:%M:%S", /* T_FMT */ + + "Sunday", /* DAY_1 */ + "Monday", /* DAY_2 */ + "Tuesday", /* DAY_3 */ + "Wednesday", /* DAY_4 */ + "Thursday", /* DAY_5 */ + "Friday", /* DAY_6 */ + "Saturday", /* DAY_7 */ + + "Sun", /* ABDAY_1 */ + "Mon", /* ABDAY_2 */ + "Tue", /* ABDAY_3 */ + "Wed", /* ABDAY_4 */ + "Thu", /* ABDAY_5 */ + "Fri", /* ABDAY_6 */ + "Sat", /* ABDAY_7 */ + + "January", /* MON_1 */ + "February", /* MON_2 */ + "March", /* MON_3 */ + "April", /* MON_4 */ + "May", /* MON_5 */ + "June", /* MON_6 */ + "July", /* MON_7 */ + "August", /* MON_8 */ + "September", /* MON_9 */ + "October", /* MON_10 */ + "November", /* MON_11 */ + "December", /* MON_12 */ + + "Jan", /* ABMON_1 */ + "Feb", /* ABMON_2 */ + "Mar", /* ABMON_3 */ + "Apr", /* ABMON_4 */ + "May", /* ABMON_5 */ + "Jun", /* ABMON_6 */ + "Jul", /* ABMON_7 */ + "Aug", /* ABMON_8 */ + "Sep", /* ABMON_9 */ + "Oct", /* ABMON_10 */ + "Nov", /* ABMON_11 */ + "Dec", /* ABMON_12 */ + + "AM", /* AM_STR */ + "PM", /* PM_STR */ + "%I:%M:%S %p", /* T_FMT_AMPM */ +}; + +static inline const char *nl_langinfo(nl_item item) +{ + if (item < 0 || 0x2B < item) + return ""; + return lc_time_c[item]; +} +#endif +#endif diff --git a/fluent-bit/include/fluent-bit/flb_lib.h b/fluent-bit/include/fluent-bit/flb_lib.h new file mode 100644 index 00000000..b5383dac --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_lib.h @@ -0,0 +1,90 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit Demo + * =============== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LIB_H +#define FLB_LIB_H + +#include <fluent-bit/flb_macros.h> +#include <fluent-bit/flb_config.h> + +/* Lib engine status */ +#define FLB_LIB_ERROR -1 +#define FLB_LIB_NONE 0 +#define FLB_LIB_OK 1 +#define FLB_LIB_NO_CONFIG_MAP 2 + +/* Library mode context data */ +struct flb_lib_ctx { + int status; + struct mk_event_loop *event_loop; + struct mk_event *event_channel; + struct flb_config *config; +}; + +/* Used on out_lib to define a callback and further opaque data */ +struct flb_lib_out_cb { + int (*cb) (void *record, size_t size, void *data); + void *data; +}; + +/* For Fluent Bit library callers, we only export the following symbols */ +typedef struct flb_lib_ctx flb_ctx_t; + +FLB_EXPORT void flb_init_env(); +FLB_EXPORT flb_ctx_t *flb_create(); +FLB_EXPORT void flb_destroy(flb_ctx_t *ctx); +FLB_EXPORT int flb_input(flb_ctx_t *ctx, const char *input, void *data); +FLB_EXPORT int flb_output(flb_ctx_t *ctx, const char *output, struct flb_lib_out_cb *cb); +FLB_EXPORT int flb_filter(flb_ctx_t *ctx, const char *filter, void *data); +FLB_EXPORT int flb_input_set(flb_ctx_t *ctx, int ffd, ...); +FLB_EXPORT int flb_input_property_check(flb_ctx_t *ctx, int ffd, char *key, char *val); +FLB_EXPORT int flb_output_property_check(flb_ctx_t *ctx, int ffd, char *key, char *val); +FLB_EXPORT int flb_filter_property_check(flb_ctx_t *ctx, int ffd, char *key, char *val); +FLB_EXPORT int flb_output_set(flb_ctx_t *ctx, int ffd, ...); +FLB_EXPORT int flb_output_set_test(flb_ctx_t *ctx, int ffd, char *test_name, + void (*out_callback) (void *, int, int, + void *, size_t, void *), + void *out_callback_data, + void *test_ctx); +FLB_EXPORT int flb_output_set_callback(flb_ctx_t *ctx, int ffd, char *name, + void (*cb)(char *, void *, void *)); + +FLB_EXPORT int flb_filter_set(flb_ctx_t *ctx, int ffd, ...); +FLB_EXPORT int flb_service_set(flb_ctx_t *ctx, ...); +FLB_EXPORT int flb_lib_free(void *data); +FLB_EXPORT double flb_time_now(); + +/* start stop the engine */ +FLB_EXPORT int flb_start(flb_ctx_t *ctx); +FLB_EXPORT int flb_start_trace(flb_ctx_t *ctx); +FLB_EXPORT int flb_stop(flb_ctx_t *ctx); +FLB_EXPORT int flb_loop(flb_ctx_t *ctx); + +/* data ingestion for "lib" input instance */ +FLB_EXPORT int flb_lib_push(flb_ctx_t *ctx, int ffd, const void *data, size_t len); +FLB_EXPORT int flb_lib_config_file(flb_ctx_t *ctx, const char *path); + +/* library context handling */ +FLB_EXPORT void flb_context_set(flb_ctx_t *ctx); +FLB_EXPORT flb_ctx_t *flb_context_get(); + +FLB_EXPORT void flb_cf_context_set(struct flb_cf *cf); +FLB_EXPORT struct flb_cf *flb_cf_context_get(); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_log.h b/fluent-bit/include/fluent-bit/flb_log.h new file mode 100644 index 00000000..ec3ca449 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_log.h @@ -0,0 +1,236 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOG_H +#define FLB_LOG_H + +#include <monkey/mk_core.h> +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_pipe.h> +#include <fluent-bit/flb_macros.h> +#include <fluent-bit/flb_thread_storage.h> +#include <fluent-bit/flb_worker.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_sds.h> +#include <inttypes.h> +#include <errno.h> +#include <stdarg.h> + +/* FIXME: this extern should be auto-populated from flb_thread_storage.h */ +extern FLB_TLS_DEFINE(struct flb_log, flb_log_ctx) + +/* Message types */ +#define FLB_LOG_OFF 0 +#define FLB_LOG_ERROR 1 +#define FLB_LOG_WARN 2 +#define FLB_LOG_INFO 3 /* default */ +#define FLB_LOG_DEBUG 4 +#define FLB_LOG_TRACE 5 +#define FLB_LOG_HELP 6 /* unused by log level */ + +#define FLB_LOG_IDEBUG 10 + +/* Logging outputs */ +#define FLB_LOG_STDERR 0 /* send logs to STDERR */ +#define FLB_LOG_FILE 1 /* write logs to a file */ +#define FLB_LOG_SOCKET 2 /* write logs to a unix socket */ + +#define FLB_LOG_EVENT MK_EVENT_NOTIFICATION +#define FLB_LOG_MNG 1024 + +#define FLB_LOG_CACHE_ENTRIES 10 +#define FLB_LOG_CACHE_TEXT_BUF_SIZE 1024 + +/* Logging main context */ +struct flb_log { + struct mk_event event; /* worker event for manager */ + flb_pipefd_t ch_mng[2]; /* worker channel manager */ + uint16_t type; /* log type */ + uint16_t level; /* level */ + char *out; /* FLB_LOG_FILE or FLB_LOG_SOCKET */ + pthread_t tid; /* thread ID */ + struct flb_worker *worker; /* non-real worker reference */ + struct mk_event_loop *evl; + + /* Initialization variables */ + int pth_init; + pthread_cond_t pth_cond; + pthread_mutex_t pth_mutex; +}; + +struct flb_log_cache_entry { + flb_sds_t buf; + uint64_t timestamp; + struct mk_list _head; +}; + +/* Structure to keep a reference of the last N number of entries */ +struct flb_log_cache { + int size; /* cache size */ + int timeout; /* cache timeout */ + struct mk_list entries; /* list for entries */ +}; + +/* + * This function is used by plugins interface to check if an incoming log message + * should be logged or not based in the log levels defined. + */ +static inline int flb_log_check_level(int level_set, int msg_level) +{ + if (msg_level <= level_set) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static inline int flb_log_check(int l) { + struct flb_worker *w; + w = (struct flb_worker *) FLB_TLS_GET(flb_worker_ctx); + if (!w && l <= 3) { + return FLB_TRUE; + } + + if (w == NULL || flb_worker_log_level(w) < l) { + return FLB_FALSE; + } + return FLB_TRUE; +} + +struct flb_log *flb_log_create(struct flb_config *config, int type, + int level, char *out); +int flb_log_set_level(struct flb_config *config, int level); +int flb_log_get_level_str(char *str); + +int flb_log_set_file(struct flb_config *config, char *out); + +int flb_log_destroy(struct flb_log *log, struct flb_config *config); +void flb_log_print(int type, const char *file, int line, const char *fmt, ...) FLB_FORMAT_PRINTF(4, 5); +int flb_log_is_truncated(int type, const char *file, int line, const char *fmt, ...) FLB_FORMAT_PRINTF(4, 5); + +struct flb_log_cache *flb_log_cache_create(int timeout_seconds, int size); +void flb_log_cache_destroy(struct flb_log_cache *cache); +struct flb_log_cache_entry *flb_log_cache_exists(struct flb_log_cache *cache, char *msg_buf, size_t msg_size); +struct flb_log_cache_entry *flb_log_cache_get_target(struct flb_log_cache *cache, uint64_t ts); + +int flb_log_cache_check_suppress(struct flb_log_cache *cache, char *msg_buf, size_t msg_size); + + +static inline int flb_log_suppress_check(int log_suppress_interval, const char *fmt, ...) +{ + int ret; + size_t size; + va_list args; + char buf[4096]; + struct flb_worker *w; + + if (log_suppress_interval <= 0) { + return FLB_FALSE; + } + + va_start(args, fmt); + size = vsnprintf(buf, sizeof(buf) - 1, fmt, args); + va_end(args); + + if (size == -1) { + return FLB_FALSE; + } + + w = flb_worker_get(); + if (!w) { + return FLB_FALSE; + } + + ret = flb_log_cache_check_suppress(w->log_cache, buf, size); + return ret; +} + + +/* Logging macros */ +#define flb_helper(fmt, ...) \ + flb_log_print(FLB_LOG_HELP, NULL, 0, fmt, ##__VA_ARGS__) + +#define flb_error(fmt, ...) \ + if (flb_log_check(FLB_LOG_ERROR)) \ + flb_log_print(FLB_LOG_ERROR, NULL, 0, fmt, ##__VA_ARGS__) + +#define flb_error_is_truncated(fmt, ...) \ + flb_log_check(FLB_LOG_ERROR) \ + ? flb_log_is_truncated(FLB_LOG_ERROR, NULL, 0, fmt, ##__VA_ARGS__) \ + : 0 + +#define flb_warn(fmt, ...) \ + if (flb_log_check(FLB_LOG_WARN)) \ + flb_log_print(FLB_LOG_WARN, NULL, 0, fmt, ##__VA_ARGS__) + +#define flb_warn_is_truncated(fmt, ... ) \ + flb_log_check(FLB_LOG_WARN) \ + ? flb_log_is_truncated(FLB_LOG_WARN, NULL, 0, fmt, ##__VA_ARGS__) \ + : 0 + +#define flb_info(fmt, ...) \ + if (flb_log_check(FLB_LOG_INFO)) \ + flb_log_print(FLB_LOG_INFO, NULL, 0, fmt, ##__VA_ARGS__) + +#define flb_info_is_truncated(fmt, ...) \ + flb_log_check(FLB_LOG_INFO) \ + ? flb_log_is_truncated(FLB_LOG_INFO, NULL, 0, fmt, ##__VA_ARGS__) \ + : 0 + +#define flb_debug(fmt, ...) \ + if (flb_log_check(FLB_LOG_DEBUG)) \ + flb_log_print(FLB_LOG_DEBUG, NULL, 0, fmt, ##__VA_ARGS__) + +#define flb_debug_is_truncated(fmt, ... ) \ + flb_log_check(FLB_LOG_DEBUG) \ + ? flb_log_is_truncated(FLB_LOG_DEBUG, NULL, 0, fmt, ##__VA_ARGS__) \ + : 0 + +#define flb_idebug(fmt, ...) \ + flb_log_print(FLB_LOG_IDEBUG, NULL, 0, fmt, ##__VA_ARGS__) + +#define flb_idebug_is_truncated(fmt, ...) \ + flb_log_is_truncated(FLB_LOG_IDEBUG, NULL, 0, fmt, ##__VA_ARGS__) + +#ifdef FLB_HAVE_TRACE +#define flb_trace(fmt, ...) \ + if (flb_log_check(FLB_LOG_TRACE)) \ + flb_log_print(FLB_LOG_TRACE, __FILE__, __LINE__, \ + fmt, ##__VA_ARGS__) + +#define flb_trace_is_truncated(fmt, ...) \ + flb_log_check(FLB_LOG_TRACE) \ + ? flb_log_is_truncated(FLB_LOG_TRACE, __FILE__, __LINE__, \ + fmt, ##__VA_ARGS__) \ + : 0 +#else +#define flb_trace(fmt, ...) do {} while(0) +#define flb_trace_is_truncated(fmt, ...) do {} while(0) +#endif + +int flb_log_worker_init(struct flb_worker *worker); +int flb_errno_print(int errnum, const char *file, int line); + +#ifdef __FLB_FILENAME__ +#define flb_errno() flb_errno_print(errno, __FLB_FILENAME__, __LINE__) +#else +#define flb_errno() flb_errno_print(errno, __FILE__, __LINE__) +#endif + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_log_event.h b/fluent-bit/include/fluent-bit/flb_log_event.h new file mode 100644 index 00000000..ca84d00f --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_log_event.h @@ -0,0 +1,45 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOG_EVENT_H +#define FLB_LOG_EVENT_H + +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_mp.h> + +#include <msgpack.h> + +#define FLB_LOG_EVENT_FORMAT_UNKNOWN 0 +#define FLB_LOG_EVENT_FORMAT_DEFAULT FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2 +#define FLB_LOG_EVENT_FORMAT_FORWARD_LEGACY 1 +#define FLB_LOG_EVENT_FORMAT_FORWARD 2 +#define FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V1 FLB_LOG_EVENT_FORMAT_FORWARD +#define FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2 4 + +struct flb_log_event { + msgpack_object *raw_timestamp; + struct flb_time timestamp; + msgpack_object *metadata; + int format; + msgpack_object *body; + msgpack_object *root; +}; + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_log_event_decoder.h b/fluent-bit/include/fluent-bit/flb_log_event_decoder.h new file mode 100644 index 00000000..d8d0787b --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_log_event_decoder.h @@ -0,0 +1,88 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOG_EVENT_DECODER_H +#define FLB_LOG_EVENT_DECODER_H + +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_mp.h> +#include <fluent-bit/flb_log_event.h> + +#include <msgpack.h> + + +#define FLB_EVENT_DECODER_SUCCESS 0 +#define FLB_EVENT_DECODER_ERROR_INITIALIZATION_FAILURE -1 +#define FLB_EVENT_DECODER_ERROR_INVALID_CONTEXT -2 +#define FLB_EVENT_DECODER_ERROR_INVALID_ARGUMENT -3 +#define FLB_EVENT_DECODER_ERROR_WRONG_ROOT_TYPE -4 +#define FLB_EVENT_DECODER_ERROR_WRONG_ROOT_SIZE -5 +#define FLB_EVENT_DECODER_ERROR_WRONG_HEADER_TYPE -6 +#define FLB_EVENT_DECODER_ERROR_WRONG_HEADER_SIZE -7 +#define FLB_EVENT_DECODER_ERROR_WRONG_TIMESTAMP_TYPE -8 +#define FLB_EVENT_DECODER_ERROR_WRONG_METADATA_TYPE -9 +#define FLB_EVENT_DECODER_ERROR_WRONG_BODY_TYPE -10 +#define FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE -11 +#define FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA -12 + +#define FLB_LOG_EVENT_EXPECTED_ROOT_ELEMENT_COUNT 2 +#define FLB_LOG_EVENT_EXPECTED_HEADER_ELEMENT_COUNT 2 + +struct flb_log_event_decoder { + int dynamically_allocated; + msgpack_unpacked unpacked_empty_map; + size_t previous_offset; + msgpack_unpacked unpacked_event; + size_t record_length; + const char *record_base; + int initialized; + msgpack_object *empty_map; + const char *buffer; + size_t offset; + size_t length; + int last_result; +}; + +void flb_log_event_decoder_reset(struct flb_log_event_decoder *context, + char *input_buffer, + size_t input_length); + +int flb_log_event_decoder_init(struct flb_log_event_decoder *context, + char *input_buffer, + size_t input_length); + +struct flb_log_event_decoder *flb_log_event_decoder_create(char *input_buffer, + size_t input_length); + +void flb_log_event_decoder_destroy(struct flb_log_event_decoder *context); + +int flb_log_event_decoder_decode_timestamp(msgpack_object *input, + struct flb_time *output); + +int flb_event_decoder_decode_object(struct flb_log_event_decoder *context, + struct flb_log_event *event, + msgpack_object *input); +int flb_log_event_decoder_get_last_result(struct flb_log_event_decoder *context); +int flb_log_event_decoder_next(struct flb_log_event_decoder *context, + struct flb_log_event *record); + +const char *flb_log_event_decoder_get_error_description(int error_code); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_log_event_encoder.h b/fluent-bit/include/fluent-bit/flb_log_event_encoder.h new file mode 100644 index 00000000..6979634a --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_log_event_encoder.h @@ -0,0 +1,315 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOG_EVENT_ENCODER_H +#define FLB_LOG_EVENT_ENCODER_H + +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_mp.h> +#include <fluent-bit/flb_log_event.h> +#include <fluent-bit/flb_log_event_encoder_dynamic_field.h> + +#include <msgpack.h> + +/* Note: + * 64 bit Windows : + * All of the size_t casts have been replaced with + * size_t pointer casts because there is an issue + * with msvc where it doesn't honor the type promotion + * when a small constant integer is hard-coded. + * + * This should not be a problem because according to + * the standard a size_t should be as large as the + * native register size just like pointers. + * + * 32 bit Windows : + * 64 bit integers are still defined as their specific + * types because the compiler handles them properly. + * + * Additionally, even though it would be preferrable to be + * able to use the optimize pragma to selectively disable + * the problematic optimizations it doesn't seem to work + * as expected. + */ + +#ifdef FLB_SYSTEM_WINDOWS +#ifdef _WIN64 +typedef size_t * flb_log_event_encoder_size_t; +typedef size_t * flb_log_event_encoder_int64_t; +typedef size_t * flb_log_event_encoder_uint64_t; +#else +typedef size_t * flb_log_event_encoder_size_t; +typedef int64_t flb_log_event_encoder_int64_t; +typedef uint64_t flb_log_event_encoder_uint64_t; +#endif +#else +typedef size_t flb_log_event_encoder_size_t; +typedef int64_t flb_log_event_encoder_int64_t; +typedef uint64_t flb_log_event_encoder_uint64_t; +#endif + +#define FLB_EVENT_ENCODER_SUCCESS 0 +#define FLB_EVENT_ENCODER_ERROR_UNSPECIFIED -1 +#define FLB_EVENT_ENCODER_ERROR_ALLOCATION_ERROR -2 +#define FLB_EVENT_ENCODER_ERROR_INVALID_CONTEXT -3 +#define FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT -4 +#define FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE -5 +#define FLB_EVENT_ENCODER_ERROR_INVALID_VALUE_TYPE -6 + +#define FLB_LOG_EVENT_APPEND_TERMINATOR_VALUE_TYPE 0 + +#define FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE 1 +#define FLB_LOG_EVENT_STRING_BODY_VALUE_TYPE 2 +#define FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE 3 +#define FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE 4 +#define FLB_LOG_EVENT_EXT_LENGTH_VALUE_TYPE 5 +#define FLB_LOG_EVENT_EXT_BODY_VALUE_TYPE 6 +#define FLB_LOG_EVENT_NULL_VALUE_TYPE 7 +#define FLB_LOG_EVENT_CHAR_VALUE_TYPE 8 +#define FLB_LOG_EVENT_INT8_VALUE_TYPE 9 +#define FLB_LOG_EVENT_INT16_VALUE_TYPE 10 +#define FLB_LOG_EVENT_INT32_VALUE_TYPE 11 +#define FLB_LOG_EVENT_INT64_VALUE_TYPE 12 +#define FLB_LOG_EVENT_UINT8_VALUE_TYPE 13 +#define FLB_LOG_EVENT_UINT16_VALUE_TYPE 14 +#define FLB_LOG_EVENT_UINT32_VALUE_TYPE 15 +#define FLB_LOG_EVENT_UINT64_VALUE_TYPE 16 +#define FLB_LOG_EVENT_DOUBLE_VALUE_TYPE 17 +#define FLB_LOG_EVENT_BOOLEAN_VALUE_TYPE 18 +#define FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE_TYPE 19 +#define FLB_LOG_EVENT_MSGPACK_RAW_VALUE_TYPE 20 +#define FLB_LOG_EVENT_TIMESTAMP_VALUE_TYPE 21 +#define FLB_LOG_EVENT_LEGACY_TIMESTAMP_VALUE_TYPE 22 +#define FLB_LOG_EVENT_FORWARD_V1_TIMESTAMP_VALUE_TYPE 23 +#define FLB_LOG_EVENT_FLUENT_BIT_V1_TIMESTAMP_VALUE_TYPE 24 +#define FLB_LOG_EVENT_FLUENT_BIT_V2_TIMESTAMP_VALUE_TYPE 25 + +#define FLB_LOG_EVENT_STRING_MIN_VALUE_TYPE FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE +#define FLB_LOG_EVENT_STRING_MAX_VALUE_TYPE FLB_LOG_EVENT_FLUENT_BIT_V2_TIMESTAMP_VALUE_TYPE + +#define FLB_LOG_EVENT_ROOT 1 +#define FLB_LOG_EVENT_METADATA 2 +#define FLB_LOG_EVENT_BODY 3 + +#define FLB_EVENT_ENCODER_VALUE_LIMIT 64 + +#define FLB_LOG_EVENT_APPEND_UNTIL_TERMINATOR -1 + +#define FLB_LOG_EVENT_VALUE_LIST_TERMINATOR() \ + (int) FLB_LOG_EVENT_APPEND_TERMINATOR_VALUE_TYPE + +#define FLB_LOG_EVENT_STRING_LENGTH_VALUE(length) \ + (int) FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE, \ + (flb_log_event_encoder_size_t) length + +#define FLB_LOG_EVENT_STRING_BODY_VALUE(buffer, length) \ + (int) FLB_LOG_EVENT_STRING_BODY_VALUE_TYPE, \ + (char *) buffer, \ + (flb_log_event_encoder_size_t) length + +#define FLB_LOG_EVENT_BINARY_LENGTH_VALUE(length) \ + (int) FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE, \ + (flb_log_event_encoder_size_t) length + +#define FLB_LOG_EVENT_BINARY_BODY_VALUE(buffer, length) \ + (int) FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE, \ + (char *) buffer, \ + (flb_log_event_encoder_size_t) length + +#define FLB_LOG_EVENT_EXT_LENGTH_VALUE(type_, length) \ + (int) FLB_LOG_EVENT_EXT_LENGTH_VALUE_TYPE, \ + (int) type_, \ + (flb_log_event_encoder_size_t) length + +#define FLB_LOG_EVENT_EXT_BODY_VALUE(buffer, length) \ + (int) FLB_LOG_EVENT_EXT_BODY_VALUE_TYPE, \ + (char *) buffer, \ + (flb_log_event_encoder_size_t) length + +#define FLB_LOG_EVENT_TIMESTAMP_VALUE(value) \ + (int) FLB_LOG_EVENT_TIMESTAMP_VALUE_TYPE, \ + (char *) value + +#define FLB_LOG_EVENT_LEGACY_TIMESTAMP_VALUE(value) \ + (int) FLB_LOG_EVENT_LEGACY_TIMESTAMP_VALUE_TYPE, \ + (char *) value + +#define FLB_LOG_EVENT_FORWARD_V1_TIMESTAMP_VALUE(value) \ + (int) FLB_LOG_EVENT_FORWARD_V1_TIMESTAMP_VALUE_TYPE, \ + (char *) value + +#define FLB_LOG_EVENT_FLUENT_BIT_V1_TIMESTAMP_VALUE(value) \ + (int) FLB_LOG_EVENT_FLUENT_BIT_V1_TIMESTAMP_VALUE_TYPE, \ + (char *) value + +#define FLB_LOG_EVENT_NULL_VALUE() \ + (int) FLB_LOG_EVENT_NULL_VALUE_TYPE + +#define FLB_LOG_EVENT_CHAR_VALUE(value) \ + (int) FLB_LOG_EVENT_CHAR_VALUE_TYPE, \ + (int) value + +#define FLB_LOG_EVENT_INT8_VALUE(value) \ + (int) FLB_LOG_EVENT_INT8_VALUE_TYPE, \ + (int) value + +#define FLB_LOG_EVENT_INT16_VALUE(value) \ + (int) FLB_LOG_EVENT_INT16_VALUE_TYPE, \ + (int) value + +#define FLB_LOG_EVENT_INT32_VALUE(value) \ + (int) FLB_LOG_EVENT_INT32_VALUE_TYPE, \ + (int32_t) value + +#define FLB_LOG_EVENT_INT64_VALUE(value) \ + (int) FLB_LOG_EVENT_INT64_VALUE_TYPE, \ + (flb_log_event_encoder_int64_t) value + +#define FLB_LOG_EVENT_UINT8_VALUE(value) \ + (int) FLB_LOG_EVENT_UINT8_VALUE_TYPE, \ + (unsigned int) value + +#define FLB_LOG_EVENT_UINT16_VALUE(value) \ + (int) FLB_LOG_EVENT_UINT16_VALUE_TYPE, \ + (unsigned int) value + +#define FLB_LOG_EVENT_UINT32_VALUE(value) \ + (int) FLB_LOG_EVENT_UINT32_VALUE_TYPE, \ + (uint32_t) value + +#define FLB_LOG_EVENT_UINT64_VALUE(value) \ + (int) FLB_LOG_EVENT_UINT64_VALUE_TYPE, \ + (flb_log_event_encoder_uint64_t) value + +#define FLB_LOG_EVENT_DOUBLE_VALUE(value) \ + (int) FLB_LOG_EVENT_DOUBLE_VALUE_TYPE, \ + (double) value + +#define FLB_LOG_EVENT_BOOLEAN_VALUE(value) \ + (int) FLB_LOG_EVENT_BOOLEAN_VALUE_TYPE, \ + (int) value + +#define FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(value) \ + (int) FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE_TYPE, \ + (char *) value + +#define FLB_LOG_EVENT_MSGPACK_RAW_VALUE(buffer, length) \ + (int) FLB_LOG_EVENT_MSGPACK_RAW_VALUE_TYPE, \ + (char *) buffer, \ + (flb_log_event_encoder_size_t) length + +#define FLB_LOG_EVENT_STRING_VALUE(buffer, length) \ + FLB_LOG_EVENT_STRING_LENGTH_VALUE(length), \ + FLB_LOG_EVENT_STRING_BODY_VALUE(buffer, length) + +#define FLB_LOG_EVENT_BINARY_VALUE(buffer, length) \ + FLB_LOG_EVENT_BINARY_LENGTH_VALUE(length), \ + FLB_LOG_EVENT_BINARY_BODY_VALUE(buffer, length) + +#define FLB_LOG_EVENT_EXT_VALUE(type_, buffer, length) \ + FLB_LOG_EVENT_EXT_LENGTH_VALUE(type, length), \ + FLB_LOG_EVENT_EXT_BODY_VALUE(buffer, length) + +#define FLB_LOG_EVENT_CSTRING_VALUE(buffer) \ + FLB_LOG_EVENT_STRING_VALUE(buffer, strlen(buffer)) + +struct flb_log_event_encoder { + int dynamically_allocated; + char *output_buffer; + size_t output_length; + int initialized; + + struct flb_time timestamp; + struct flb_log_event_encoder_dynamic_field metadata; + struct flb_log_event_encoder_dynamic_field body; + struct flb_log_event_encoder_dynamic_field root; + + msgpack_packer packer; + msgpack_sbuffer buffer; + + int format; +}; + +int flb_log_event_encoder_init(struct flb_log_event_encoder *context, + int format); + +struct flb_log_event_encoder *flb_log_event_encoder_create(int format); + +void flb_log_event_encoder_destroy(struct flb_log_event_encoder *context); + +void flb_log_event_encoder_reset(struct flb_log_event_encoder *context); + +void flb_log_event_encoder_claim_internal_buffer_ownership( + struct flb_log_event_encoder *context); + +int flb_log_event_encoder_emit_record(struct flb_log_event_encoder *context); +int flb_log_event_encoder_reset_record(struct flb_log_event_encoder *context); +int flb_log_event_encoder_begin_record(struct flb_log_event_encoder *context); +int flb_log_event_encoder_commit_record(struct flb_log_event_encoder *context); +int flb_log_event_encoder_rollback_record(struct flb_log_event_encoder *context); +int flb_log_event_encoder_emit_raw_record(struct flb_log_event_encoder *context, + const char *buffer, + size_t length); + +int flb_log_event_encoder_set_timestamp( + struct flb_log_event_encoder *context, + struct flb_time *timestamp); + +int flb_log_event_encoder_set_current_timestamp( + struct flb_log_event_encoder *context); + +int flb_log_event_encoder_append_metadata_values_unsafe( + struct flb_log_event_encoder *context, + ...); + +int flb_log_event_encoder_append_body_values_unsafe( + struct flb_log_event_encoder *context, + ...); + +int flb_log_event_encoder_append_root_values_unsafe( + struct flb_log_event_encoder *context, + ...); + +const char *flb_log_event_encoder_get_error_description(int error_code); + +#define flb_log_event_encoder_append_metadata_values(context, ...) \ + flb_log_event_encoder_append_metadata_values_unsafe( \ + context, \ + __VA_ARGS__, \ + FLB_LOG_EVENT_VALUE_LIST_TERMINATOR()); + +#define flb_log_event_encoder_append_body_values(context, ...) \ + flb_log_event_encoder_append_body_values_unsafe( \ + context, \ + __VA_ARGS__, \ + FLB_LOG_EVENT_VALUE_LIST_TERMINATOR()); + +#define flb_log_event_encoder_append_root_values(context, ...) \ + flb_log_event_encoder_append_root_values_unsafe( \ + context, \ + __VA_ARGS__, \ + FLB_LOG_EVENT_VALUE_LIST_TERMINATOR()); + +#include <fluent-bit/flb_log_event_encoder_primitives.h> +#include <fluent-bit/flb_log_event_encoder_root_macros.h> +#include <fluent-bit/flb_log_event_encoder_metadata_macros.h> +#include <fluent-bit/flb_log_event_encoder_body_macros.h> + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_log_event_encoder_body_macros.h b/fluent-bit/include/fluent-bit/flb_log_event_encoder_body_macros.h new file mode 100644 index 00000000..4945da8c --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_log_event_encoder_body_macros.h @@ -0,0 +1,174 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOG_EVENT_ENCODER_BODY_MACROS_H +#define FLB_LOG_EVENT_ENCODER_BODY_MACROS_H + +#define flb_log_event_encoder_append_body_binary_length(context, length) \ + flb_log_event_encoder_append_binary_length(context, FLB_LOG_EVENT_BODY, length) + +#define flb_log_event_encoder_append_body_binary_body(context, value, length) \ + flb_log_event_encoder_append_binary_body(context, FLB_LOG_EVENT_BODY, value, length) + +#define flb_log_event_encoder_append_body_ext_length(context, type, length) \ + flb_log_event_encoder_append_ext_length(context, FLB_LOG_EVENT_BODY, type, length) + +#define flb_log_event_encoder_append_body_ext_body(context, value, length) \ + flb_log_event_encoder_append_ext_body(context, FLB_LOG_EVENT_BODY, value, length) + +#define flb_log_event_encoder_append_body_string_length(context, length) \ + flb_log_event_encoder_append_string_length(context, FLB_LOG_EVENT_BODY, length) + +#define flb_log_event_encoder_append_body_string_body(context, value, length) \ + flb_log_event_encoder_append_string_body(context, FLB_LOG_EVENT_BODY, value, length) + +#define flb_log_event_encoder_append_body_int8(context, value) \ + flb_log_event_encoder_append_int8(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_int16(context, value) \ + flb_log_event_encoder_append_int16(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_int32(context, value) \ + flb_log_event_encoder_append_int32(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_int64(context, value) \ + flb_log_event_encoder_append_int64(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_uint8(context, value) \ + flb_log_event_encoder_append_uint8(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_uint16(context, value) \ + flb_log_event_encoder_append_uint16(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_uint32(context, value) \ + flb_log_event_encoder_append_uint32(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_uint64(context, value) \ + flb_log_event_encoder_append_uint64(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_double(context, value) \ + flb_log_event_encoder_append_double(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_boolean(context, value) \ + flb_log_event_encoder_append_boolean(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_character(context, value) \ + flb_log_event_encoder_append_character(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_binary(context, value, length) \ + flb_log_event_encoder_append_binary(context, FLB_LOG_EVENT_BODY, value, length) + +#define flb_log_event_encoder_append_body_string(context, value, length) \ + flb_log_event_encoder_append_string(context, FLB_LOG_EVENT_BODY, value, length) + +#define flb_log_event_encoder_append_body_ext(context, type, value, length) \ + flb_log_event_encoder_append_ext(context, FLB_LOG_EVENT_BODY, type, value, length) + +#define flb_log_event_encoder_append_body_cstring(context, value) \ + flb_log_event_encoder_append_cstring(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_null(context) \ + flb_log_event_encoder_append_null(context, FLB_LOG_EVENT_BODY) + +#define flb_log_event_encoder_append_body_msgpack_object(context, value) \ + flb_log_event_encoder_append_msgpack_object(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_raw_msgpack(context, value_buffer, value_size) \ + flb_log_event_encoder_append_raw_msgpack(context, FLB_LOG_EVENT_BODY, value_buffer, value_size) + +#define flb_log_event_encoder_append_body_timestamp(context, value) \ + flb_log_event_encoder_append_timestamp(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_legacy_timestamp(context, value) \ + flb_log_event_encoder_append_legacy_timestamp(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_forward_v1_timestamp(context, value) \ + flb_log_event_encoder_append_forward_v1_timestamp(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_fluent_bit_v1_timestamp(context, value) \ + flb_log_event_encoder_append_fluent_bit_v1_timestamp(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_append_body_fluent_bit_v2_timestamp(context, value) \ + flb_log_event_encoder_append_fluent_bit_v2_timestamp(context, FLB_LOG_EVENT_BODY, value) + +#define flb_log_event_encoder_body_begin_map(context) \ + flb_log_event_encoder_dynamic_field_begin_map(&(context->body)) + +#define flb_log_event_encoder_body_commit_map(context) \ + flb_log_event_encoder_dynamic_field_commit_map(&(context->body)) + +#define flb_log_event_encoder_body_rollback_map(context) \ + flb_log_event_encoder_dynamic_field_rollback_map(&(context->body)) + +#define flb_log_event_encoder_body_begin_array(context) \ + flb_log_event_encoder_dynamic_field_begin_array(&(context->body)) + +#define flb_log_event_encoder_body_commit_array(context) \ + flb_log_event_encoder_dynamic_field_commit_array(&(context->body)) + +#define flb_log_event_encoder_body_rollback_array(context) \ + flb_log_event_encoder_dynamic_field_rollback_array(&(context->body)) + + +static inline \ +int flb_log_event_encoder_set_body_from_msgpack_object( + struct flb_log_event_encoder *context, + msgpack_object *value) +{ + int result; + + result = flb_log_event_encoder_dynamic_field_reset(&context->body); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_append_body_msgpack_object( + context, value); + } + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_flush(&context->body); + } + + return result; +} + +static inline \ +int flb_log_event_encoder_set_body_from_raw_msgpack( + struct flb_log_event_encoder *context, + char *value_buffer, + size_t value_size) +{ + int result; + + result = flb_log_event_encoder_dynamic_field_reset(&context->body); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_append_body_raw_msgpack( + context, + value_buffer, + value_size); + } + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_flush(&context->body); + } + + return result; +} + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_log_event_encoder_dynamic_field.h b/fluent-bit/include/fluent-bit/flb_log_event_encoder_dynamic_field.h new file mode 100644 index 00000000..3ca90d44 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_log_event_encoder_dynamic_field.h @@ -0,0 +1,98 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOG_EVENT_ENCODER_DYNAMIC_FIELD_H +#define FLB_LOG_EVENT_ENCODER_DYNAMIC_FIELD_H + +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_mp.h> +#include <cfl/cfl_list.h> + +#include <msgpack.h> +#include <ctype.h> + +struct flb_log_event_encoder_dynamic_field_scope { + size_t offset; + struct flb_mp_map_header header; + int type; + struct cfl_list _head; +}; + +struct flb_log_event_encoder_dynamic_field { + int initialized; + size_t entry_count; + size_t data_offset; + msgpack_packer packer; + msgpack_sbuffer buffer; + struct cfl_list scopes; + + char *data; + size_t size; + int type; +}; + +struct flb_log_event_encoder_dynamic_field_scope * + flb_log_event_encoder_dynamic_field_scope_current( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_scope_enter( + struct flb_log_event_encoder_dynamic_field *field, + int type); + +int flb_log_event_encoder_dynamic_field_scope_leave( + struct flb_log_event_encoder_dynamic_field *field, + struct flb_log_event_encoder_dynamic_field_scope *scope, + int commit); + +int flb_log_event_encoder_dynamic_field_begin_map( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_begin_array( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_commit_map( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_commit_array( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_rollback_map( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_rollback_array( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_append( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_flush( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_reset( + struct flb_log_event_encoder_dynamic_field *field); + +int flb_log_event_encoder_dynamic_field_init( + struct flb_log_event_encoder_dynamic_field *field, + int type); + +void flb_log_event_encoder_dynamic_field_destroy( + struct flb_log_event_encoder_dynamic_field *field); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_log_event_encoder_metadata_macros.h b/fluent-bit/include/fluent-bit/flb_log_event_encoder_metadata_macros.h new file mode 100644 index 00000000..b412010e --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_log_event_encoder_metadata_macros.h @@ -0,0 +1,173 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOG_EVENT_ENCODER_METADATA_MACROS_H +#define FLB_LOG_EVENT_ENCODER_METADATA_MACROS_H + +#define flb_log_event_encoder_append_metadata_binary_length(context, length) \ + flb_log_event_encoder_append_binary_length(context, FLB_LOG_EVENT_METADATA, length) + +#define flb_log_event_encoder_append_metadata_binary_body(context, value, length) \ + flb_log_event_encoder_append_binary_body(context, FLB_LOG_EVENT_METADATA, value, length) + +#define flb_log_event_encoder_append_metadata_ext_length(context, type, length) \ + flb_log_event_encoder_append_ext_length(context, FLB_LOG_EVENT_METADATA, type, length) + +#define flb_log_event_encoder_append_metadata_ext_body(context, value, length) \ + flb_log_event_encoder_append_ext_body(context, FLB_LOG_EVENT_METADATA, value, length) + +#define flb_log_event_encoder_append_metadata_string_length(context, length) \ + flb_log_event_encoder_append_string_length(context, FLB_LOG_EVENT_METADATA, length) + +#define flb_log_event_encoder_append_metadata_string_body(context, value, length) \ + flb_log_event_encoder_append_string_body(context, FLB_LOG_EVENT_METADATA, value, length) + +#define flb_log_event_encoder_append_metadata_int8(context, value) \ + flb_log_event_encoder_append_int8(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_int16(context, value) \ + flb_log_event_encoder_append_int16(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_int32(context, value) \ + flb_log_event_encoder_append_int32(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_int64(context, value) \ + flb_log_event_encoder_append_int64(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_uint8(context, value) \ + flb_log_event_encoder_append_uint8(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_uint16(context, value) \ + flb_log_event_encoder_append_uint16(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_uint32(context, value) \ + flb_log_event_encoder_append_uint32(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_uint64(context, value) \ + flb_log_event_encoder_append_uint64(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_double(context, value) \ + flb_log_event_encoder_append_double(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_boolean(context, value) \ + flb_log_event_encoder_append_boolean(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_character(context, value) \ + flb_log_event_encoder_append_character(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_binary(context, value, length) \ + flb_log_event_encoder_append_binary(context, FLB_LOG_EVENT_METADATA, value, length) + +#define flb_log_event_encoder_append_metadata_string(context, value, length) \ + flb_log_event_encoder_append_string(context, FLB_LOG_EVENT_METADATA, value, length) + +#define flb_log_event_encoder_append_metadata_ext(context, type, value, length) \ + flb_log_event_encoder_append_ext(context, FLB_LOG_EVENT_METADATA, type, value, length) + +#define flb_log_event_encoder_append_metadata_cstring(context, value) \ + flb_log_event_encoder_append_cstring(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_null(context) \ + flb_log_event_encoder_append_null(context, FLB_LOG_EVENT_METADATA) + +#define flb_log_event_encoder_append_metadata_msgpack_object(context, value) \ + flb_log_event_encoder_append_msgpack_object(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_raw_msgpack(context, value_buffer, value_size) \ + flb_log_event_encoder_append_raw_msgpack(context, FLB_LOG_EVENT_METADATA, value_buffer, value_size) + +#define flb_log_event_encoder_append_metadata_timestamp(context, value) \ + flb_log_event_encoder_append_timestamp(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_legacy_timestamp(context, value) \ + flb_log_event_encoder_append_legacy_timestamp(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_forward_v1_timestamp(context, value) \ + flb_log_event_encoder_append_forward_v1_timestamp(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_fluent_bit_v1_timestamp(context, value) \ + flb_log_event_encoder_append_fluent_bit_v1_timestamp(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_append_metadata_fluent_bit_v2_timestamp(context, value) \ + flb_log_event_encoder_append_fluent_bit_v2_timestamp(context, FLB_LOG_EVENT_METADATA, value) + +#define flb_log_event_encoder_metadata_begin_map(context) \ + flb_log_event_encoder_dynamic_field_begin_map(&(context->metadata)) + +#define flb_log_event_encoder_metadata_commit_map(context) \ + flb_log_event_encoder_dynamic_field_commit_map(&(context->metadata)) + +#define flb_log_event_encoder_metadata_rollback_map(context) \ + flb_log_event_encoder_dynamic_field_rollback_map(&(context->metadata)) + +#define flb_log_event_encoder_metadata_begin_array(context) \ + flb_log_event_encoder_dynamic_field_begin_array(&(context->metadata)) + +#define flb_log_event_encoder_metadata_commit_array(context) \ + flb_log_event_encoder_dynamic_field_commit_array(&(context->metadata)) + +#define flb_log_event_encoder_metadata_rollback_array(context) \ + flb_log_event_encoder_dynamic_field_rollback_array(&(context->metadata)) + +static inline \ +int flb_log_event_encoder_set_metadata_from_msgpack_object( + struct flb_log_event_encoder *context, + msgpack_object *value) +{ + int result; + + result = flb_log_event_encoder_dynamic_field_reset(&context->metadata); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_append_metadata_msgpack_object( + context, value); + } + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_flush(&context->metadata); + } + + return result; +} + +static inline \ +int flb_log_event_encoder_set_metadata_from_raw_msgpack( + struct flb_log_event_encoder *context, + char *value_buffer, + size_t value_size) +{ + int result; + + result = flb_log_event_encoder_dynamic_field_reset(&context->metadata); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_append_metadata_raw_msgpack( + context, + value_buffer, + value_size); + } + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_flush(&context->metadata); + } + + return result; +} + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_log_event_encoder_primitives.h b/fluent-bit/include/fluent-bit/flb_log_event_encoder_primitives.h new file mode 100644 index 00000000..1a1400d5 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_log_event_encoder_primitives.h @@ -0,0 +1,312 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOG_EVENT_ENCODER_PRIMITIVES_H +#define FLB_LOG_EVENT_ENCODER_PRIMITIVES_H + +int flb_log_event_encoder_append_values_unsafe( + struct flb_log_event_encoder *context, + int field, + va_list arguments); + +int flb_log_event_encoder_append_binary_length( + struct flb_log_event_encoder *context, + int target_field, + size_t length); + +int flb_log_event_encoder_append_binary_body( + struct flb_log_event_encoder *context, + int target_field, + char *value, + size_t length); + +int flb_log_event_encoder_append_ext_length( + struct flb_log_event_encoder *context, + int target_field, + int8_t type, + size_t length); + +int flb_log_event_encoder_append_ext_body( + struct flb_log_event_encoder *context, + int target_field, + char *value, + size_t length); + +int flb_log_event_encoder_append_string_length( + struct flb_log_event_encoder *context, + int target_field, + size_t length); + +int flb_log_event_encoder_append_string_body( + struct flb_log_event_encoder *context, + int target_field, + char *value, + size_t length); + +int flb_log_event_encoder_append_int8( + struct flb_log_event_encoder *context, + int target_field, + int8_t value); + +int flb_log_event_encoder_append_int16( + struct flb_log_event_encoder *context, + int target_field, + int16_t value); + +int flb_log_event_encoder_append_int32( + struct flb_log_event_encoder *context, + int target_field, + int32_t value); + +int flb_log_event_encoder_append_int64( + struct flb_log_event_encoder *context, + int target_field, + int64_t value); + +int flb_log_event_encoder_append_uint8( + struct flb_log_event_encoder *context, + int target_field, + uint8_t value); + +int flb_log_event_encoder_append_uint16( + struct flb_log_event_encoder *context, + int target_field, + uint16_t value); + +int flb_log_event_encoder_append_uint32( + struct flb_log_event_encoder *context, + int target_field, + uint32_t value); + +int flb_log_event_encoder_append_uint64( + struct flb_log_event_encoder *context, + int target_field, + uint64_t value); + +int flb_log_event_encoder_append_double( + struct flb_log_event_encoder *context, + int target_field, + double value); + +int flb_log_event_encoder_append_boolean( + struct flb_log_event_encoder *context, + int target_field, + int value); + +int flb_log_event_encoder_append_character( + struct flb_log_event_encoder *context, + int target_field, + char value); + +int flb_log_event_encoder_append_binary( + struct flb_log_event_encoder *context, + int target_field, + char *value, + size_t length); + +int flb_log_event_encoder_append_string( + struct flb_log_event_encoder *context, + int target_field, + char *value, + size_t length); + +int flb_log_event_encoder_append_ext( + struct flb_log_event_encoder *context, + int target_field, + int8_t type, + char *value, + size_t length); + +int flb_log_event_encoder_append_cstring( + struct flb_log_event_encoder *context, + int target_field, + char *value); + +int flb_log_event_encoder_append_null( + struct flb_log_event_encoder *context, + int target_field); + +int flb_log_event_encoder_append_msgpack_object( + struct flb_log_event_encoder *context, + int target_field, + msgpack_object *value); + +int flb_log_event_encoder_append_raw_msgpack( + struct flb_log_event_encoder *context, + int target_field, + char *value_buffer, + size_t value_size); + +int flb_log_event_encoder_append_timestamp( + struct flb_log_event_encoder *context, + int target_field, + struct flb_time *value); + +int flb_log_event_encoder_append_legacy_timestamp( + struct flb_log_event_encoder *context, + int target_field, + struct flb_time *value); + +int flb_log_event_encoder_append_forward_v1_timestamp( + struct flb_log_event_encoder *context, + int target_field, + struct flb_time *timestamp); + +int flb_log_event_encoder_append_fluent_bit_v1_timestamp( + struct flb_log_event_encoder *context, + int target_field, + struct flb_time *value); + +int flb_log_event_encoder_append_fluent_bit_v2_timestamp( + struct flb_log_event_encoder *context, + int target_field, + struct flb_time *value); + +#define flb_log_event_encoder_append_values(context, field, ...) \ + flb_log_event_encoder_append_values_unsafe( \ + context, \ + field, \ + __VA_ARGS__, \ + FLB_LOG_EVENT_VALUE_LIST_TERMINATOR()); + +static inline \ +int flb_log_event_encoder_get_field( + struct flb_log_event_encoder *context, + int target_field, + struct flb_log_event_encoder_dynamic_field **field) +{ + if (target_field == FLB_LOG_EVENT_METADATA) { + *field = &context->metadata; + } + else if (target_field == FLB_LOG_EVENT_BODY) { + *field = &context->body; + } + else if (target_field == FLB_LOG_EVENT_ROOT) { + *field = &context->root; + } + else { + return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; + } + + return FLB_EVENT_ENCODER_SUCCESS; +} + +static inline \ +int flb_log_event_encoder_begin_map( + struct flb_log_event_encoder *context, + int target_field) +{ + struct flb_log_event_encoder_dynamic_field *field; + int result; + + result = flb_log_event_encoder_get_field(context, target_field, &field); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_begin_map(field); + } + + return result; +} + +static inline \ +int flb_log_event_encoder_commit_map( + struct flb_log_event_encoder *context, + int target_field) +{ + struct flb_log_event_encoder_dynamic_field *field; + int result; + + result = flb_log_event_encoder_get_field(context, target_field, &field); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_commit_map(field); + } + + return result; +} + +static inline \ +int flb_log_event_encoder_rollback_map( + struct flb_log_event_encoder *context, + int target_field) +{ + struct flb_log_event_encoder_dynamic_field *field; + int result; + + result = flb_log_event_encoder_get_field(context, target_field, &field); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_rollback_map(field); + } + + return result; +} + +static inline \ +int flb_log_event_encoder_begin_array( + struct flb_log_event_encoder *context, + int target_field) +{ + struct flb_log_event_encoder_dynamic_field *field; + int result; + + result = flb_log_event_encoder_get_field(context, target_field, &field); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_begin_array(field); + } + + return result; +} + +static inline \ +int flb_log_event_encoder_commit_array( + struct flb_log_event_encoder *context, + int target_field) +{ + struct flb_log_event_encoder_dynamic_field *field; + int result; + + result = flb_log_event_encoder_get_field(context, target_field, &field); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_commit_array(field); + } + + return result; +} + +static inline \ +int flb_log_event_encoder_rollback_array( + struct flb_log_event_encoder *context, + int target_field) +{ + struct flb_log_event_encoder_dynamic_field *field; + int result; + + result = flb_log_event_encoder_get_field(context, target_field, &field); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_rollback_array(field); + } + + return result; +} + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_log_event_encoder_root_macros.h b/fluent-bit/include/fluent-bit/flb_log_event_encoder_root_macros.h new file mode 100644 index 00000000..a199e1b1 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_log_event_encoder_root_macros.h @@ -0,0 +1,190 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOG_EVENT_ENCODER_ROOT_MACROS_H +#define FLB_LOG_EVENT_ENCODER_ROOT_MACROS_H + +#define flb_log_event_encoder_append_root_binary_length(context, length) \ + flb_log_event_encoder_append_binary_length(context, FLB_LOG_EVENT_ROOT, length) + +#define flb_log_event_encoder_append_root_binary_body(context, value, length) \ + flb_log_event_encoder_append_binary_body(context, FLB_LOG_EVENT_ROOT, value, length) + +#define flb_log_event_encoder_append_root_ext_length(context, type, length) \ + flb_log_event_encoder_append_ext_length(context, FLB_LOG_EVENT_ROOT, type, length) + +#define flb_log_event_encoder_append_root_ext_body(context, value, length) \ + flb_log_event_encoder_append_ext_body(context, FLB_LOG_EVENT_ROOT, value, length) + +#define flb_log_event_encoder_append_root_string_length(context, length) \ + flb_log_event_encoder_append_string_length(context, FLB_LOG_EVENT_ROOT, length) + +#define flb_log_event_encoder_append_root_string_body(context, value, length) \ + flb_log_event_encoder_append_string_body(context, FLB_LOG_EVENT_ROOT, value, length) + +#define flb_log_event_encoder_append_root_int8(context, value) \ + flb_log_event_encoder_append_int8(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_int16(context, value) \ + flb_log_event_encoder_append_int16(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_int32(context, value) \ + flb_log_event_encoder_append_int32(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_int64(context, value) \ + flb_log_event_encoder_append_int64(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_uint8(context, value) \ + flb_log_event_encoder_append_uint8(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_uint16(context, value) \ + flb_log_event_encoder_append_uint16(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_uint32(context, value) \ + flb_log_event_encoder_append_uint32(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_uint64(context, value) \ + flb_log_event_encoder_append_uint64(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_double(context, value) \ + flb_log_event_encoder_append_double(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_boolean(context, value) \ + flb_log_event_encoder_append_boolean(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_character(context, value) \ + flb_log_event_encoder_append_character(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_binary(context, value, length) \ + flb_log_event_encoder_append_binary(context, FLB_LOG_EVENT_ROOT, value, length) + +#define flb_log_event_encoder_append_root_string(context, value, length) \ + flb_log_event_encoder_append_string(context, FLB_LOG_EVENT_ROOT, value, length) + +#define flb_log_event_encoder_append_root_ext(context, type, value, length) \ + flb_log_event_encoder_append_ext(context, FLB_LOG_EVENT_ROOT, type, value, length) + +#define flb_log_event_encoder_append_root_cstring(context, value) \ + flb_log_event_encoder_append_cstring(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_null(context) \ + flb_log_event_encoder_append_null(context, FLB_LOG_EVENT_ROOT) + +#define flb_log_event_encoder_append_root_msgpack_object(context, value) \ + flb_log_event_encoder_append_msgpack_object(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_raw_msgpack(context, value_buffer, value_size) \ + flb_log_event_encoder_append_raw_msgpack(context, FLB_LOG_EVENT_ROOT, value_buffer, value_size) + +#define flb_log_event_encoder_append_root_timestamp(context, value) \ + flb_log_event_encoder_append_timestamp(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_legacy_timestamp(context, value) \ + flb_log_event_encoder_append_legacy_timestamp(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_forward_v1_timestamp(context, value) \ + flb_log_event_encoder_append_forward_v1_timestamp(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_fluent_bit_v1_timestamp(context, value) \ + flb_log_event_encoder_append_fluent_bit_v1_timestamp(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_append_root_fluent_bit_v2_timestamp(context, value) \ + flb_log_event_encoder_append_fluent_bit_v2_timestamp(context, FLB_LOG_EVENT_ROOT, value) + +#define flb_log_event_encoder_root_begin_map(context) \ + flb_log_event_encoder_dynamic_field_begin_map(&(context->root)) + +#define flb_log_event_encoder_root_commit_map(context) \ + flb_log_event_encoder_dynamic_field_commit_map(&(context->root)) + +#define flb_log_event_encoder_root_rollback_map(context) \ + flb_log_event_encoder_dynamic_field_rollback_map(&(context->root)) + +#define flb_log_event_encoder_root_begin_array(context) \ + flb_log_event_encoder_dynamic_field_begin_array(&(context->root)) + +#define flb_log_event_encoder_root_commit_array(context) \ + flb_log_event_encoder_dynamic_field_commit_array(&(context->root)) + +#define flb_log_event_encoder_root_rollback_array(context) \ + flb_log_event_encoder_dynamic_field_rollback_array(&(context->root)) + + +static inline \ +int flb_log_event_encoder_set_root_from_msgpack_object( + struct flb_log_event_encoder *context, + msgpack_object *value) +{ + int result; + + result = flb_log_event_encoder_dynamic_field_reset(&context->metadata); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_reset(&context->body); + } + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_reset(&context->root); + } + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_append_root_msgpack_object( + context, value); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_flush(&context->root); + } + } + + return result; +} + +static inline \ +int flb_log_event_encoder_set_root_from_raw_msgpack( + struct flb_log_event_encoder *context, + char *value_buffer, + size_t value_size) +{ + int result; + + result = flb_log_event_encoder_dynamic_field_reset(&context->metadata); + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_reset(&context->body); + } + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_reset(&context->root); + } + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_append_root_raw_msgpack( + context, + value_buffer, + value_size); + } + + if (result == FLB_EVENT_ENCODER_SUCCESS) { + result = flb_log_event_encoder_dynamic_field_flush(&context->root); + } + + return result; +} + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_lua.h b/fluent-bit/include/fluent-bit/flb_lua.h new file mode 100644 index 00000000..ada88dbc --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_lua.h @@ -0,0 +1,104 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LUA_H +#define FLB_LUA_H + +#include "lua.h" +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_luajit.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_time.h> + +#include <monkey/mk_core/mk_list.h> +#include <msgpack/pack.h> + +/* global variables in Lua */ +#define FLB_LUA_VAR_FLB_NULL "flb_null" + +#define FLB_LUA_L2C_TYPES_NUM_MAX 16 + +enum flb_lua_l2c_type_enum { + FLB_LUA_L2C_TYPE_INT, + FLB_LUA_L2C_TYPE_ARRAY, + FLB_LUA_L2C_TYPE_MAP +}; + +struct flb_lua_l2c_type { + flb_sds_t key; + int type; + struct mk_list _head; +}; + +struct flb_lua_l2c_config { + int l2c_types_num; /* number of l2c_types */ + struct mk_list l2c_types; /* data types (lua -> C) */ +}; + + +/* + * Metatable for Lua table. + * https://www.lua.org/manual/5.1/manual.html#2.8 + */ +struct flb_lua_metadata { + int initialized; + int data_type; /* Map or Array */ +}; + +static inline int flb_lua_metadata_init(struct flb_lua_metadata *meta) +{ + if (meta == NULL) { + return -1; + } + meta->initialized = FLB_TRUE; + meta->data_type = -1; + + return 0; +} + +/* convert from negative index to positive index */ +static inline int flb_lua_absindex(lua_State *l , int index) +{ +#if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM < 520 + if (index < 0) { + index = lua_gettop(l) + index + 1; + } +#else + index = lua_absindex(l, index); +#endif + return index; +} + +int flb_lua_arraylength(lua_State *l, int index); +void flb_lua_pushtimetable(lua_State *l, struct flb_time *tm); +int flb_lua_is_valid_func(lua_State *l, flb_sds_t func); +int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader); +void flb_lua_pushmsgpack(lua_State *l, msgpack_object *o); +void flb_lua_tomsgpack(lua_State *l, + msgpack_packer *pck, + int index, + struct flb_lua_l2c_config *l2cc); +void flb_lua_tompack(lua_State *l, + mpack_writer_t *writer, + int index, + struct flb_lua_l2c_config *l2cc); +void flb_lua_dump_stack(FILE *out, lua_State *l); +int flb_lua_enable_flb_null(lua_State *l); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_luajit.h b/fluent-bit/include/fluent-bit/flb_luajit.h new file mode 100644 index 00000000..440455d8 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_luajit.h @@ -0,0 +1,44 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LUAJIT_H +#define FLB_LUAJIT_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> + +#include <lauxlib.h> +#include <lua.h> +#include <lualib.h> + +/* Lua Context */ +struct flb_luajit { + lua_State *state; /* LuaJIT VM environment */ + void *config; /* Fluent Bit context */ + struct mk_list _head; /* Link to flb_config->lua */ +}; + +struct flb_luajit *flb_luajit_create(struct flb_config *config); +int flb_luajit_load_script(struct flb_luajit *lj, char *script); +int flb_luajit_load_buffer(struct flb_luajit *lj, char *string, size_t len, char *name); + +void flb_luajit_destroy(struct flb_luajit *lj); +int flb_luajit_destroy_all(struct flb_config *ctx); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_macros.h b/fluent-bit/include/fluent-bit/flb_macros.h new file mode 100644 index 00000000..31b6ca52 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_macros.h @@ -0,0 +1,58 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_MACROS_H +#define FLB_MACROS_H + +#include <monkey/mk_core.h> + +#define FLB_FALSE 0 +#define FLB_TRUE !FLB_FALSE + +/* Return values */ +#define FLB_ERROR 0 +#define FLB_OK 1 +#define FLB_RETRY 2 + +/* ala-printf format check */ +#if defined(__GNUC__) || defined(__clang__) +#define FLB_FORMAT_PRINTF(fmt, args) __attribute__ ((format (printf, fmt, args))) +#else +#define FLB_FORMAT_PRINTF(fmt, args) +#endif + +#ifdef _WIN32 +#define FLB_INLINE inline +#else +#define FLB_INLINE inline __attribute__((always_inline)) +#endif + +#define FLB_EXPORT MK_EXPORT + +#define flb_unlikely(x) mk_unlikely(x) +#define flb_likely(x) mk_likely(x) + +#define flb_bug(condition) do { \ + if (flb_unlikely((condition)!=0)) { \ + fprintf(stderr, "Bug found in %s() at %s:%d", \ + __FUNCTION__, __FILE__, __LINE__); \ + abort(); \ + } \ + } while(0) +#endif diff --git a/fluent-bit/include/fluent-bit/flb_mem.h b/fluent-bit/include/fluent-bit/flb_mem.h new file mode 100644 index 00000000..04d7bc55 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_mem.h @@ -0,0 +1,132 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_MEM_H +#define FLB_MEM_H + +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE +#endif + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_macros.h> + +#ifdef FLB_HAVE_JEMALLOC +#include <jemalloc/jemalloc.h> +#endif + +#include <stdlib.h> + +/* + * The following memory handling wrappers, aims to simplify the way to use + * the default memory allocator from the libc or an alternative one as Jemalloc. + * + * Here there is no error logging in case of failures, we defer that task to the + * caller. + */ +#ifdef FLB_HAVE_ATTRIBUTE_ALLOC_SIZE + #define FLB_ALLOCSZ_ATTR(x,...) __attribute__ ((alloc_size(x, ##__VA_ARGS__))) +#else + #define FLB_ALLOCSZ_ATTR(x,...) +#endif + +#ifdef FLB_HAVE_TESTS_OSSFUZZ +/* + * Return 1 or 0 based on a probability. + */ +int flb_malloc_p; +int flb_malloc_mod; + +static inline int flb_fuzz_get_probability(int val) { + flb_malloc_p += 1; + flb_malloc_p = flb_malloc_p % flb_malloc_mod; + if (val > flb_malloc_p) { + return 1; + } + return 0; +} +#endif + +static inline FLB_ALLOCSZ_ATTR(1) +void *flb_malloc(const size_t size) { + +#ifdef FLB_HAVE_TESTS_OSSFUZZ + // 1% chance of failure + if (flb_fuzz_get_probability(1)) { + return NULL; + } +#endif + + if (size == 0) { + return NULL; + } + + return malloc(size); +} + +static inline FLB_ALLOCSZ_ATTR(1, 2) +void *flb_calloc(size_t n, const size_t size) { + if (size == 0) { + return NULL; + } +#ifdef FLB_HAVE_TESTS_OSSFUZZ + // Add chance of failure. Used by fuzzing to test error-handling code. + if (flb_fuzz_get_probability(1)) { + return NULL; + } +#endif + + return calloc(n, size); +} + +static inline FLB_ALLOCSZ_ATTR(2) +void *flb_realloc(void *ptr, const size_t size) +{ + return realloc(ptr, size); +} + +static inline FLB_ALLOCSZ_ATTR(3) +void *flb_realloc_z(void *ptr, const size_t old_size, const size_t new_size) +{ + void *tmp; + void *p; + size_t diff; + + tmp = flb_realloc(ptr, new_size); + if (!tmp) { + return NULL; + } + + if (new_size > old_size) { + diff = new_size - old_size; + p = ((char *) tmp + old_size); + memset(p, 0, diff); + } + + return tmp; +} + + +static inline void flb_free(void *ptr) { + free(ptr); +} + +#undef FLB_ALLOCSZ_ATTR + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_meta.h b/fluent-bit/include/fluent-bit/flb_meta.h new file mode 100644 index 00000000..91e5a4c2 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_meta.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_META_H +#define FLB_META_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> + +int flb_meta_run(struct flb_config *ctx, const char *cmd, const char *params); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_metrics.h b/fluent-bit/include/fluent-bit/flb_metrics.h new file mode 100644 index 00000000..4db8d4ca --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_metrics.h @@ -0,0 +1,85 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> +#include <monkey/mk_core.h> + +#ifdef FLB_HAVE_METRICS +#ifndef FLB_METRICS_H +#define FLB_METRICS_H + +/* CMetrics */ +#include <cmetrics/cmetrics.h> +#include <cmetrics/cmt_counter.h> +#include <cmetrics/cmt_gauge.h> +#include <cmetrics/cmt_untyped.h> +#include <cmetrics/cmt_cat.h> +#include <cmetrics/cmt_decode_msgpack.h> +#include <cmetrics/cmt_encode_influx.h> +#include <cmetrics/cmt_encode_text.h> +#include <cmetrics/cmt_encode_prometheus.h> +#include <cmetrics/cmt_encode_prometheus_remote_write.h> +#include <cmetrics/cmt_encode_msgpack.h> +#include <cmetrics/cmt_encode_splunk_hec.h> + +/* Metrics IDs for general purpose (used by core and Plugins */ +#define FLB_METRIC_N_RECORDS 0 +#define FLB_METRIC_N_BYTES 1 +#define FLB_METRIC_N_DROPPED 2 +#define FLB_METRIC_N_ADDED 3 + +/* Genaral output plugin metrics */ +#define FLB_METRIC_OUT_OK_RECORDS 10 /* proc_records */ +#define FLB_METRIC_OUT_OK_BYTES 11 /* proc_bytes */ +#define FLB_METRIC_OUT_ERROR 12 /* errors */ +#define FLB_METRIC_OUT_RETRY 13 /* retries */ +#define FLB_METRIC_OUT_RETRY_FAILED 14 /* retries_failed */ +#define FLB_METRIC_OUT_DROPPED_RECORDS 15 /* dropped_records_total */ +#define FLB_METRIC_OUT_RETRIED_RECORDS 16 /* retried_records_total */ + +struct flb_metric { + int id; + int title_len; + char title[64]; + size_t val; + struct mk_list _head; +}; + +struct flb_metrics { + int title_len; /* Title string length */ + char title[64]; /* Title or id for this metrics context */ + int count; /* Total count of metrics registered */ + struct mk_list list; /* Head of metrics list */ +}; + +struct flb_metrics *flb_metrics_create(const char *title); +int flb_metrics_title(const char *title, struct flb_metrics *metrics); + +struct flb_metric *flb_metrics_get_id(int id, struct flb_metrics *metrics); +int flb_metrics_add(int id, const char *title, struct flb_metrics *metrics); +int flb_metrics_sum(int id, size_t val, struct flb_metrics *metrics); +int flb_metrics_print(struct flb_metrics *metrics); +int flb_metrics_dump_values(char **out_buf, size_t *out_size, + struct flb_metrics *me); +int flb_metrics_destroy(struct flb_metrics *metrics); +int flb_metrics_fluentbit_add(struct flb_config *ctx, struct cmt *cmt); + +#endif +#endif /* FLB_HAVE_METRICS */ diff --git a/fluent-bit/include/fluent-bit/flb_metrics_exporter.h b/fluent-bit/include/fluent-bit/flb_metrics_exporter.h new file mode 100644 index 00000000..ceb52d89 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_metrics_exporter.h @@ -0,0 +1,42 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit/flb_info.h> +#ifdef FLB_HAVE_METRICS + +#ifndef FLB_METRICS_EXPORTER_H +#define FLB_METRICS_EXPORTER_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_metrics.h> + +struct flb_me { + int fd; + struct flb_config *config; + struct mk_event event; +}; + +int flb_me_fd_event(int fd, struct flb_me *me); +struct flb_me *flb_me_create(struct flb_config *ctx); +int flb_me_destroy(struct flb_me *me); +struct cmt *flb_me_get_cmetrics(struct flb_config *ctx); + +#endif +#endif /* FLB_HAVE_METRICS */ diff --git a/fluent-bit/include/fluent-bit/flb_mp.h b/fluent-bit/include/fluent-bit/flb_mp.h new file mode 100644 index 00000000..536edb7b --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_mp.h @@ -0,0 +1,79 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_MP_H +#define FLB_MP_H + +#include <msgpack.h> + +#define FLB_MP_MAP MSGPACK_OBJECT_MAP +#define FLB_MP_ARRAY MSGPACK_OBJECT_ARRAY + +int flb_mp_count(const void *data, size_t bytes); +int flb_mp_count_remaining(const void *data, size_t bytes, size_t *remaining_bytes); +int flb_mp_validate_log_chunk(const void *data, size_t bytes, + int *out_records, size_t *processed_bytes); +int flb_mp_validate_metric_chunk(const void *data, size_t bytes, + int *out_series, size_t *processed_bytes); + +void flb_mp_set_map_header_size(char *buf, int arr_size); + + +/* + * Map header handling functions + */ +struct flb_mp_map_header { + off_t offset; + size_t entries; + void *data; +}; + + +/* */ +struct flb_mp_accessor_match { + int matched; + msgpack_object *start_key; + msgpack_object *key; + msgpack_object *val; + struct flb_record_accessor *ra; +}; + +/* A context to abstract usage of record accessor when multiple patterns exists */ +struct flb_mp_accessor { + int matches_size; + struct flb_mp_accessor_match *matches; + struct mk_list ra_list; +}; + +int flb_mp_map_header_init(struct flb_mp_map_header *mh, msgpack_packer *mp_pck); +int flb_mp_map_header_append(struct flb_mp_map_header *mh); +void flb_mp_map_header_end(struct flb_mp_map_header *mh); + +int flb_mp_array_header_init(struct flb_mp_map_header *mh, msgpack_packer *mp_pck); +int flb_mp_array_header_append(struct flb_mp_map_header *mh); +void flb_mp_array_header_end(struct flb_mp_map_header *mh); + +/* mp accessor api */ +struct flb_mp_accessor *flb_mp_accessor_create(struct mk_list *slist_patterns); +void flb_mp_accessor_destroy(struct flb_mp_accessor *mpa); +int flb_mp_accessor_keys_remove(struct flb_mp_accessor *mpa, + msgpack_object *map, + void **out_buf, size_t *out_size); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_net_dns.h b/fluent-bit/include/fluent-bit/flb_net_dns.h new file mode 100644 index 00000000..3a36406d --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_net_dns.h @@ -0,0 +1,29 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_NET_DNS_H +#define FLB_NET_DNS_H + +struct flb_net_dns { + struct mk_list lookups; + struct mk_list lookups_drop; +}; + +#endif + diff --git a/fluent-bit/include/fluent-bit/flb_network.h b/fluent-bit/include/fluent-bit/flb_network.h new file mode 100644 index 00000000..eab45787 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_network.h @@ -0,0 +1,212 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_NETWORK_H +#define FLB_NETWORK_H + +#include <fluent-bit/flb_compat.h> +#include <fluent-bit/flb_socket.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_uri.h> +#include <fluent-bit/flb_upstream_conn.h> +#include <fluent-bit/flb_net_dns.h> +#include <ares.h> + +#define FLB_NETWORK_DEFAULT_BACKLOG_SIZE 128 +#define FLB_NETWORK_UNIX_SOCKET_PEER_ADDRESS_TEMPLATE "pid_%s" +#define FLB_NETWORK_ADDRESS_UNAVAILABLE "unavailable" + +/* FLB_NETWORK_MAX_UNIX_ADDRESS_LENGTH should be enough for the + * string "PID " plus the string form of a signed 32 bit integer + * and a NULL character. + */ + +/* Network connection setup */ +struct flb_net_setup { + /* enable/disable keepalive support */ + char keepalive; + + /* max time in seconds that a keepalive connection can be idle */ + int keepalive_idle_timeout; + + /* max time in seconds to wait for a established connection */ + int connect_timeout; + + /* max time in seconds an incoming connection can take including the + * TLS handshake + */ + int accept_timeout; + + /* accept timeout log error (default: true) */ + int accept_timeout_log_error; + + /* max time in seconds to wait for blocking io calls */ + int io_timeout; + + /* connect timeout log error (default: true) */ + int connect_timeout_log_error; + + /* network interface to bind and use to send data */ + flb_sds_t source_address; + + /* maximum of times a keepalive connection can be used */ + int keepalive_max_recycle; + + /* dns mode : TCP or UDP */ + char *dns_mode; + + /* dns resolver : LEGACY or ASYNC */ + char *dns_resolver; + + /* prioritize ipv4 results when trying to establish a connection*/ + int dns_prefer_ipv4; + + /* maximum number of allowed active TCP connections */ + int max_worker_connections; +}; + +/* Defines a host service and it properties */ +struct flb_net_host { + int ipv6; /* IPv6 required ? */ + flb_sds_t address; /* Original address */ + int port; /* TCP port */ + flb_sds_t name; /* Hostname */ + flb_sds_t listen; /* Listen interface */ + struct flb_uri *uri; /* Extra URI parameters */ +}; + +/* Defines an async DNS lookup context */ +struct flb_dns_lookup_context { + struct mk_event response_event; /* c-ares socket event */ + int ares_socket_registered; + struct ares_socket_functions ares_socket_functions; + int *udp_timeout_detected; + int ares_socket_created; + int ares_socket_type; + void *ares_channel; + int *result_code; + struct mk_event_loop *event_loop; + struct flb_coro *coroutine; + struct flb_sched_timer *udp_timer; + int finished; + int dropped; + struct flb_net_dns *dns_ctx; + struct addrinfo **result; + /* result is a synthetized result, don't call freeaddrinfo on it */ + struct mk_list _head; +}; + +#define FLB_DNS_LOOKUP_CONTEXT_FOR_EVENT(event) \ + ((struct flb_dns_lookup_context *) \ + &((uint8_t *) event)[-offsetof(struct flb_dns_lookup_context, response_event)]) + +#define FLB_DNS_LEGACY 'L' +#define FLB_DNS_ASYNC 'A' + +#define FLB_DNS_USE_TCP 'T' +#define FLB_DNS_USE_UDP 'U' + +#ifndef TCP_FASTOPEN +#define TCP_FASTOPEN 23 +#endif + +/* General initialization of the networking layer */ +void flb_net_lib_init(); +void flb_net_ctx_init(struct flb_net_dns *dns_ctx); + +void flb_net_dns_ctx_init(); +struct flb_net_dns *flb_net_dns_ctx_get(); +void flb_net_dns_ctx_set(struct flb_net_dns *dns_ctx); + +/* Generic functions */ +void flb_net_setup_init(struct flb_net_setup *net); +int flb_net_host_set(const char *plugin_name, struct flb_net_host *host, const char *address); + +/* DNS handling */ +void flb_net_dns_lookup_context_cleanup(struct flb_net_dns *dns_ctx); + +/* TCP options */ +int flb_net_socket_reset(flb_sockfd_t fd); +int flb_net_socket_tcp_nodelay(flb_sockfd_t fd); +int flb_net_socket_blocking(flb_sockfd_t fd); +int flb_net_socket_nonblocking(flb_sockfd_t fd); +int flb_net_socket_rcv_buffer(flb_sockfd_t fd, int rcvbuf); +int flb_net_socket_tcp_fastopen(flb_sockfd_t sockfd); + +/* Socket handling */ +flb_sockfd_t flb_net_socket_create(int family, int nonblock); +flb_sockfd_t flb_net_socket_create_udp(int family, int nonblock); +flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, + char *source_addr, int connect_timeout, + int is_async, + void *async_ctx, + struct flb_connection *u_conn); + +flb_sockfd_t flb_net_udp_connect(const char *host, unsigned long port, + char *source_addr); + +int flb_net_tcp_fd_connect(flb_sockfd_t fd, const char *host, unsigned long port); +flb_sockfd_t flb_net_server(const char *port, const char *listen_addr); +flb_sockfd_t flb_net_server_udp(const char *port, const char *listen_addr); +flb_sockfd_t flb_net_server_unix(const char *listen_path, int stream_mode, + int backlog); +int flb_net_bind(flb_sockfd_t fd, const struct sockaddr *addr, + socklen_t addrlen, int backlog); +int flb_net_bind_udp(flb_sockfd_t fd, const struct sockaddr *addr, + socklen_t addrlen); +flb_sockfd_t flb_net_accept(flb_sockfd_t server_fd); + +int flb_net_address_to_str(int family, const struct sockaddr *addr, + char *output_buffer, size_t output_buffer_size); + +int flb_net_socket_peer_address(flb_sockfd_t fd, + struct sockaddr_storage *output_buffer); + +int flb_net_socket_address_info(flb_sockfd_t fd, + struct sockaddr_storage *address, + unsigned short int *port_output_buffer, + char *str_output_buffer, + int str_output_buffer_size, + size_t *str_output_data_size); + +int flb_net_socket_peer_ip_str(flb_sockfd_t fd, + char *output_buffer, + int output_buffer_size, + size_t *output_data_size, + int *output_address_family); + +int flb_net_socket_peer_ip_raw(flb_sockfd_t fd, + char *output_buffer, + int output_buffer_size, + size_t *output_data_size, + int *output_address_family); + +int flb_net_socket_peer_port(flb_sockfd_t fd, + unsigned short int *output_buffer); + +int flb_net_socket_peer_info(flb_sockfd_t fd, + unsigned short int *port_output_buffer, + struct sockaddr_storage *raw_output_buffer, + char *str_output_buffer, + int str_output_buffer_size, + size_t *str_output_data_size); + +size_t flb_network_address_size(struct sockaddr_storage *address); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_oauth2.h b/fluent-bit/include/fluent-bit/flb_oauth2.h new file mode 100644 index 00000000..0bcb981a --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_oauth2.h @@ -0,0 +1,73 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OAUTH2_H +#define FLB_OAUTH2_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/tls/flb_tls.h> + +#define FLB_OAUTH2_PORT "443" +#define FLB_OAUTH2_HTTP_ENCODING "application/x-www-form-urlencoded" + +struct flb_oauth2 { + flb_sds_t auth_url; + flb_sds_t payload; + + /* Parsed URL */ + flb_sds_t host; + flb_sds_t port; + flb_sds_t uri; + + /* Token times set by the caller */ + time_t issued; + time_t expires; + + /* Token info after successful auth */ + flb_sds_t access_token; + flb_sds_t token_type; + uint64_t expires_in; + + /* TLS Context */ +#ifdef FLB_HAVE_TLS + struct flb_tls *tls; +#else + void *tls; +#endif + + /* Upstream context */ + struct flb_upstream *u; +}; + +struct flb_oauth2 *flb_oauth2_create(struct flb_config *config, + const char *auth_url, int expire_sec); +void flb_oauth2_destroy(struct flb_oauth2 *ctx); +int flb_oauth2_token_len(struct flb_oauth2 *ctx); +void flb_oauth2_payload_clear(struct flb_oauth2 *ctx); +int flb_oauth2_payload_append(struct flb_oauth2 *ctx, + const char *key_str, int key_len, + const char *val_str, int val_len); +char *flb_oauth2_token_get(struct flb_oauth2 *ctx); +int flb_oauth2_token_expired(struct flb_oauth2 *ctx); + +int flb_oauth2_parse_json_response(const char *json_data, size_t json_size, + struct flb_oauth2 *ctx); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_output.h b/fluent-bit/include/fluent-bit/flb_output.h new file mode 100644 index 00000000..661054b7 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_output.h @@ -0,0 +1,1101 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUTPUT_H +#define FLB_OUTPUT_H + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE +#endif + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include <monkey/mk_core.h> +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_bits.h> +#include <fluent-bit/flb_io.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_config_map.h> +#include <fluent-bit/flb_network.h> +#include <fluent-bit/flb_engine.h> +#include <fluent-bit/flb_task.h> +#include <fluent-bit/flb_coro.h> +#include <fluent-bit/flb_callback.h> +#include <fluent-bit/flb_mem.h> +#include <fluent-bit/flb_str.h> +#include <fluent-bit/flb_http_client.h> +#include <fluent-bit/tls/flb_tls.h> +#include <fluent-bit/flb_output_thread.h> +#include <fluent-bit/flb_upstream.h> +#include <fluent-bit/flb_upstream_ha.h> +#include <fluent-bit/flb_event.h> +#include <fluent-bit/flb_processor.h> + +#include <cmetrics/cmetrics.h> +#include <cmetrics/cmt_gauge.h> +#include <cmetrics/cmt_counter.h> +#include <cmetrics/cmt_decode_msgpack.h> +#include <cmetrics/cmt_encode_msgpack.h> + +#include <ctraces/ctraces.h> +#include <ctraces/ctr_decode_msgpack.h> +#include <ctraces/ctr_encode_msgpack.h> +#include <ctraces/ctr_mpack_utils_defs.h> + +#ifdef FLB_HAVE_REGEX +#include <fluent-bit/flb_regex.h> +#endif + +/* Output plugin masks */ +#define FLB_OUTPUT_NET 32 /* output address may set host and port */ +#define FLB_OUTPUT_PLUGIN_CORE 0 +#define FLB_OUTPUT_PLUGIN_PROXY 1 +#define FLB_OUTPUT_NO_MULTIPLEX 512 /* run one task at a time, one task per flush */ +#define FLB_OUTPUT_PRIVATE 1024 +#define FLB_OUTPUT_SYNCHRONOUS 2048 /* run one task at a time, no flush cycle limit */ + + +/* + * Event type handlers + * + * These types are defined by creating a mask using numbers. However, it's important + * to note that the masks used in this process are different from the ones used + * in flb_event.h. The original chunk values are not actually masks, but rather set + * numbers starting from 0; this is for compatibility reasons. + */ +#define FLB_OUTPUT_LOGS 1 +#define FLB_OUTPUT_METRICS 2 +#define FLB_OUTPUT_TRACES 4 + +#define FLB_OUTPUT_FLUSH_COMPAT_OLD_18() \ + const void *data = event_chunk->data; \ + size_t bytes = event_chunk->size; \ + int tag_len = flb_sds_len(event_chunk->tag); \ + const char *tag = event_chunk->tag; + +struct flb_output_flush; + +/* + * Tests callbacks + * =============== + */ +struct flb_test_out_formatter { + /* + * Runtime Library Mode + * ==================== + * When the runtime library enable the test formatter mode, it needs to + * keep a reference of the context and other information: + * + * - rt_ctx : context created by flb_create() + * + * - rt_ffd : this plugin assigned 'integer' created by flb_output() + * + * - rt_step_calback: intermediary function to receive the results of + * the formatter plugin test function. + * + * - rt_data: opaque data type for rt_step_callback() + */ + + /* runtime library context */ + void *rt_ctx; + + /* runtime library: assigned plugin integer */ + int rt_ffd; + + /* + * "runtime step callback": this function pointer is used by Fluent Bit + * library mode to reference a test function that must retrieve the + * results of 'callback'. Consider this an intermediary function to + * transfer the results to the runtime test. + * + * This function is private and should not be set manually in the plugin + * code, it's set on src/flb_lib.c . + */ + void (*rt_out_callback) (void *, int, int, void *, size_t, void *); + + /* + * opaque data type passed by the runtime library to be used on + * rt_step_test(). + */ + void *rt_data; + + /* optional context for flush callback */ + void *flush_ctx; + + /* + * Callback + * ========= + * "Formatter callback": it references the plugin function that performs + * data formatting (msgpack -> local data). This entry is mostly to + * expose the plugin local function. + */ + int (*callback) (/* Fluent Bit context */ + struct flb_config *, + /* plugin that ingested the records */ + struct flb_input_instance *, + void *, /* plugin instance context */ + void *, /* optional flush context */ + int, /* event type */ + const char *, /* tag */ + int, /* tag length */ + const void *, /* incoming msgpack data */ + size_t, /* incoming msgpack size */ + void **, /* output buffer */ + size_t *); /* output buffer size */ +}; + +struct flb_output_plugin { + /* + * a 'mask' to define what kind of data the plugin can manage: + * + * - FLB_OUTPUT_LOGS + * - FLB_OUTPUT_METRICS + */ + int event_type; + + /* + * The type defines if this is a core-based plugin or it's handled by + * some specific proxy. + */ + int type; + void *proxy; + + int flags; + + /* The plugin name */ + char *name; + + /* Plugin description */ + char *description; + + struct flb_config_map *config_map; + + /* + * Output network info: + * + * An output plugin can be specified just using it shortname or using the + * complete network address format, e.g: + * + * $ fluent-bit -i cpu -o plugin://hostname:port/uri + * + * where: + * + * plugin = the output plugin shortname + * name = IP address or hostname of the target + * port = target TCP port + * uri = extra information that may be used by the plugin + */ + struct flb_net_host host; + + /* Initalization */ + int (*cb_init) (struct flb_output_instance *, struct flb_config *, void *); + + /* Pre run */ + int (*cb_pre_run) (void *, struct flb_config *); + + /* Flush callback */ + void (*cb_flush) (struct flb_event_chunk *, + struct flb_output_flush *, + struct flb_input_instance *, + void *, + struct flb_config *); + + /* Exit */ + int (*cb_exit) (void *, struct flb_config *); + + /* Destroy */ + void (*cb_destroy) (struct flb_output_plugin *); + + /* Default number of worker threads */ + int workers; + + /* Tests */ + struct flb_test_out_formatter test_formatter; + + /* Link to global list from flb_config->outputs */ + struct mk_list _head; +}; + +// constants for retry_limit +#define FLB_OUT_RETRY_UNLIMITED -1 +#define FLB_OUT_RETRY_NONE 0 + +/* + * Each initialized plugin must have an instance, same plugin may be + * loaded more than one time. + * + * An instance try to contain plugin data separating what is fixed data + * and the variable one that is generated when the plugin is invoked. + */ +struct flb_output_instance { + struct mk_event event; /* events handler */ + + struct flb_processor *processor; + + /* + * a 'mask' to define what kind of data the plugin can manage: + * + * - FLB_OUTPUT_LOGS + * - FLB_OUTPUT_METRICS + */ + int event_type; + int id; /* instance id */ + int log_level; /* instance log level */ + int log_suppress_interval; /* log suppression interval */ + char name[32]; /* numbered name (cpu -> cpu.0) */ + char *alias; /* alias name for the instance */ + int flags; /* inherit flags from plugin */ + int test_mode; /* running tests? (default:off) */ + flb_pipefd_t ch_events[2]; /* channel for events */ + struct flb_output_plugin *p; /* original plugin */ + void *context; /* plugin configuration context */ + + + /* Plugin properties */ + int retry_limit; /* max of retries allowed */ + int use_tls; /* bool, try to use TLS for I/O */ + char *match; /* match rule for tag/routing */ +#ifdef FLB_HAVE_REGEX + struct flb_regex *match_regex; /* match rule (regex) based on Tags */ +#endif + +#ifdef FLB_HAVE_TLS + int tls_verify; /* Verify certs (default: true) */ + int tls_debug; /* mbedtls debug level */ + char *tls_vhost; /* Virtual hostname for SNI */ + char *tls_ca_path; /* Path to certificates */ + char *tls_ca_file; /* CA root cert */ + char *tls_crt_file; /* Certificate */ + char *tls_key_file; /* Cert Key */ + char *tls_key_passwd; /* Cert Key Password */ +#endif + + /* + * network info: + * + * An input plugin can be specified just using it shortname or using the + * complete network address format, e.g: + * + * $ fluent-bit -i cpu -o plugin://hostname:port/uri + * + * where: + * + * plugin = the output plugin shortname + * name = IP address or hostname of the target + * port = target TCP port + * uri = extra information that may be used by the plugin + */ + struct flb_net_host host; + + /* + * Optional data passed to the plugin, this info is useful when + * running Fluent Bit in library mode and the target plugin needs + * some specific data from it caller. + */ + void *data; + + /* Output handler configuration */ + void *out_context; + +#ifdef FLB_HAVE_TLS + struct flb_tls *tls; +#else + void *tls; +#endif + + /* + * configuration properties: incoming properties set by the caller. This + * list is what the instance received by either a configuration file or + * through the command line arguments. This list is validated by the + * plugin. + */ + struct mk_list properties; + + /* + * configuration map: a new API is landing on Fluent Bit v1.4 that allows + * plugins to specify at registration time the allowed configuration + * properties and it data types. Config map is an optional API for now + * and some plugins will take advantage of it. When the API is used, the + * config map will validate the configuration, set default values + * and merge the 'properties' (above) into the map. + */ + struct mk_list *config_map; + + /* General network options like timeouts and keepalive */ + struct flb_net_setup net_setup; + struct mk_list *net_config_map; + struct mk_list net_properties; + + struct mk_list *tls_config_map; + + struct mk_list _head; /* link to config->inputs */ + + /* + * CMetrics + * -------- + */ + struct cmt *cmt; /* parent context */ + struct cmt_counter *cmt_proc_records; /* m: output_proc_records */ + struct cmt_counter *cmt_proc_bytes; /* m: output_proc_bytes */ + struct cmt_counter *cmt_errors; /* m: output_errors */ + struct cmt_counter *cmt_retries; /* m: output_retries */ + struct cmt_counter *cmt_retries_failed; /* m: output_retries_failed */ + struct cmt_counter *cmt_dropped_records; /* m: output_dropped_records */ + struct cmt_counter *cmt_retried_records; /* m: output_retried_records */ + + /* m: output_upstream_total_connections */ + struct cmt_gauge *cmt_upstream_total_connections; + /* m: output_upstream_busy_connections */ + struct cmt_gauge *cmt_upstream_busy_connections; + + /* OLD Metrics API */ +#ifdef FLB_HAVE_METRICS + struct flb_metrics *metrics; /* metrics */ +#endif + + /* Callbacks context */ + struct flb_callback *callback; + + /* Tests */ + struct flb_test_out_formatter test_formatter; + + /* + * Buffer counter: it counts the total of disk space (filesystem) used by buffers + */ + size_t fs_chunks_size; + + /* + * Buffer counter: it counts the total of disk space (filesystem) awaiting to be + * loaded (in backlog) + */ + size_t fs_backlog_chunks_size; + + /* + * Buffer limit: optional limit set by configuration so this output instance + * cannot buffer more than total_limit_size (bytes unit). + * + * Note that this is the limit set to the filesystem buffer mechanism so the + * input instance routered to this output plugin should configure to use + * filesystem as buffer type. + */ + size_t total_limit_size; + + /* Queue for singleplexed tasks */ + struct flb_task_queue *singleplex_queue; + + /* Thread Pool: this is optional for the caller */ + int tp_workers; + struct flb_tp *tp; + + /* If the thread pool was created, this flag is turned on */ + int is_threaded; + + /* List of upstreams */ + struct mk_list upstreams; + + /* + * flush context and co-routines + * ----------------------------- + * Every invocation of flush() output callback runs under a co-routine, this + * co-routine context (struct flb_coro) is wrapped inside the structure + * 'flb_output_flush' which is added to the 'flush_list' linked list. + * + * In order to assign the coro 'id', we use the 'coro_id' incremental + * counter to generate the next id. co-routine id's aims to be held + * in 14 bits so the range goes from 0 to 16383. + * + * When the 'flush context' needs to be destroyed, it's moved out from the + * 'flush_list' and placed into 'flush_list_destroy', a cleanup function will + * destroy the remaining resources. + * + * note on multi-threading mode + * ---------------------------- + * Every output instance in threaded mode has it own flush context which + * has similar fields like 'coro_id', 'flush_list' and 'flush_list_destroy'. + * + * On that mode, field fields are not used. + */ + int flush_id; + struct mk_list flush_list; + struct mk_list flush_list_destroy; + + /* Keep a reference to the original context this instance belongs to */ + struct flb_config *config; +}; + +/* + * [note] this has been renamed from flb_output_coro to flb_output_flush. + * + * This structure represents the context of a flush invocation with internal + * information about the output instance being called plus other internal details. + */ +struct flb_output_flush { + int id; /* out-thread ID */ + const void *buffer; /* output buffer */ + struct flb_task *task; /* Parent flb_task */ + struct flb_config *config; /* FLB context */ + struct flb_output_instance *o_ins; /* output instance */ + struct flb_coro *coro; /* parent coro addr */ + + /* + * if the original event_chunk has been processed, a new + * temporary event_chunk is created, so the flush callback + * receives new data. + */ + struct flb_event_chunk *processed_event_chunk; + + struct mk_list _head; /* Link to flb_task->threads */ +}; + +static FLB_INLINE int flb_output_is_threaded(struct flb_output_instance *ins) +{ + return ins->is_threaded; +} + +/* When an output_thread is going to be destroyed, this callback is triggered */ +static FLB_INLINE void flb_output_flush_destroy(struct flb_output_flush *out_flush) +{ + flb_debug("[out flush] cb_destroy coro_id=%i", out_flush->id); + + mk_list_del(&out_flush->_head); + flb_coro_destroy(out_flush->coro); + flb_free(out_flush); +} + +/* + * libco do not support parameters in the entrypoint function due to the + * complexity of implementation in terms of architecture and compiler, but + * it provide a workaround using a global structure as a middle entry-point + * that achieve the same stuff. + */ +struct flb_out_flush_params { + struct flb_event_chunk *event_chunk; /* event chunk */ + struct flb_output_flush *out_flush; /* output flush */ + struct flb_input_instance *i_ins; /* input instance */ + struct flb_output_plugin *out_plugin; /* output plugin context */ + void *out_context; /* custom plugin context */ + struct flb_config *config; /* Fluent Bit context */ + struct flb_coro *coro; /* coroutine context */ +}; + +extern FLB_TLS_DEFINE(struct flb_out_flush_params, out_flush_params); + +#define FLB_OUTPUT_RETURN(x) \ + flb_output_return_do(x); \ + return + +static inline void flb_output_return_do(int x); + +static FLB_INLINE void output_params_set(struct flb_output_flush *out_flush, + struct flb_coro *coro, + struct flb_task *task, + struct flb_output_plugin *out_plugin, + void *out_context, + struct flb_config *config) +{ + int s = sizeof(struct flb_out_flush_params); + struct flb_out_flush_params *params; + + params = (struct flb_out_flush_params *) FLB_TLS_GET(out_flush_params); + if (!params) { + params = (struct flb_out_flush_params *) flb_malloc(s); + if (!params) { + flb_errno(); + return; + } + } + + /* Callback parameters in order */ + if (out_flush->processed_event_chunk) { + params->event_chunk = out_flush->processed_event_chunk; + } + else { + params->event_chunk = task->event_chunk; + } + params->out_flush = out_flush; + params->i_ins = task->i_ins; + params->out_context = out_context; + params->config = config; + params->out_plugin = out_plugin; + params->coro = coro; + + FLB_TLS_SET(out_flush_params, params); + co_switch(coro->callee); +} + +static FLB_INLINE void output_pre_cb_flush(void) +{ + int route_status; + struct flb_coro *coro; + struct flb_output_plugin *out_p; + struct flb_out_flush_params *params; + struct flb_out_flush_params persisted_params; + + params = (struct flb_out_flush_params *) FLB_TLS_GET(out_flush_params); + if (!params) { + flb_error("[output] no co-routines params defined, unexpected"); + return; + } + + /* + * Until this point the th->callee already set the variables, so we + * wait until the core wanted to resume so we really trigger the + * output callback. + * + * Persist params locally incase ptr data is changed while switched out. + */ + coro = params->coro; + persisted_params = *params; + + co_switch(coro->caller); + + /* Continue, we will resume later */ + out_p = persisted_params.out_plugin; + + flb_task_acquire_lock(persisted_params.out_flush->task); + + route_status = flb_task_get_route_status( + persisted_params.out_flush->task, + persisted_params.out_flush->o_ins); + + if (route_status == FLB_TASK_ROUTE_DROPPED) { + flb_task_release_lock(persisted_params.out_flush->task); + + FLB_OUTPUT_RETURN(FLB_ERROR); + } + + flb_task_activate_route(persisted_params.out_flush->task, + persisted_params.out_flush->o_ins); + + flb_task_release_lock(persisted_params.out_flush->task); + + out_p->cb_flush(persisted_params.event_chunk, + persisted_params.out_flush, + persisted_params.i_ins, + persisted_params.out_context, + persisted_params.config); +} + +void flb_output_flush_prepare_destroy(struct flb_output_flush *out_flush); +int flb_output_flush_id_get(struct flb_output_instance *ins); + +static FLB_INLINE +struct flb_output_flush *flb_output_flush_create(struct flb_task *task, + struct flb_input_instance *i_ins, + struct flb_output_instance *o_ins, + struct flb_config *config) +{ + int ret; + size_t records; + void *p_buf; + size_t p_size; + size_t stack_size; + struct flb_coro *coro; + struct flb_output_flush *out_flush; + struct flb_out_thread_instance *th_ins; + struct flb_event_chunk *evc; + struct flb_event_chunk *tmp; + char *resized_serialization_buffer; + size_t serialization_buffer_offset; + char *serialized_context_buffer; + size_t serialized_context_size; + struct cmt *metrics_context; + struct ctrace *trace_context; + size_t chunk_offset; + + /* Custom output coroutine info */ + out_flush = (struct flb_output_flush *) flb_calloc(1, sizeof(struct flb_output_flush)); + if (!out_flush) { + flb_errno(); + return NULL; + } + + /* Create a new co-routine */ + coro = flb_coro_create(out_flush); + if (!coro) { + flb_free(out_flush); + return NULL; + } + + /* + * Each co-routine receives an 'id', the value is always incremental up to + * 16383. + */ + out_flush->id = flb_output_flush_id_get(o_ins); + out_flush->o_ins = o_ins; + out_flush->task = task; + out_flush->buffer = task->event_chunk->data; + out_flush->config = config; + out_flush->coro = coro; + out_flush->processed_event_chunk = NULL; + + /* Logs processor */ + evc = task->event_chunk; + if (flb_processor_is_active(o_ins->processor)) { + if (evc->type == FLB_EVENT_TYPE_LOGS) { + /* run the processor */ + ret = flb_processor_run(o_ins->processor, + 0, + FLB_PROCESSOR_LOGS, + evc->tag, flb_sds_len(evc->tag), + evc->data, evc->size, + &p_buf, &p_size); + if (ret == -1 || p_size == 0) { + flb_coro_destroy(coro); + flb_free(out_flush); + return NULL; + } + + records = flb_mp_count(p_buf, p_size); + if (records == 0) { + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + return NULL; + } + + tmp = flb_event_chunk_create(evc->type, records, evc->tag, flb_sds_len(evc->tag), p_buf, p_size); + if (!tmp) { + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + return NULL; + } + out_flush->processed_event_chunk = tmp; + } + else if (evc->type == FLB_EVENT_TYPE_METRICS) { + p_buf = flb_calloc(evc->size * 2, sizeof(char)); + + if (p_buf == NULL) { + flb_coro_destroy(coro); + flb_free(out_flush); + + return NULL; + } + + p_size = evc->size; + + chunk_offset = 0; + serialization_buffer_offset = 0; + + while ((ret = cmt_decode_msgpack_create( + &metrics_context, + (char *) evc->data, + evc->size, + &chunk_offset)) == CMT_DECODE_MSGPACK_SUCCESS) { + ret = flb_processor_run(o_ins->processor, + 0, + FLB_PROCESSOR_METRICS, + evc->tag, + flb_sds_len(evc->tag), + (char *) metrics_context, + 0, + NULL, + NULL); + + if (ret == 0) { + ret = cmt_encode_msgpack_create(metrics_context, + &serialized_context_buffer, + &serialized_context_size); + + cmt_destroy(metrics_context); + + if (ret != 0) { + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + + return NULL; + } + + if ((serialization_buffer_offset + + serialized_context_size) > p_size) { + resized_serialization_buffer = \ + flb_realloc(p_buf, p_size + serialized_context_size); + + if (resized_serialization_buffer == NULL) { + cmt_encode_msgpack_destroy(serialized_context_buffer); + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + + return NULL; + } + + p_size += serialized_context_size; + p_buf = resized_serialization_buffer; + } + + memcpy(&(((char *) p_buf)[serialization_buffer_offset]), + serialized_context_buffer, + serialized_context_size); + + serialization_buffer_offset += serialized_context_size; + + cmt_encode_msgpack_destroy(serialized_context_buffer); + } + } + + if (serialization_buffer_offset == 0) { + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + + return NULL; + } + + out_flush->processed_event_chunk = flb_event_chunk_create( + evc->type, + 0, + evc->tag, + flb_sds_len(evc->tag), + p_buf, + p_size); + + if (out_flush->processed_event_chunk == NULL) { + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + + return NULL; + } + } + else if (evc->type == FLB_EVENT_TYPE_TRACES) { + p_buf = flb_calloc(evc->size * 2, sizeof(char)); + + if (p_buf == NULL) { + flb_coro_destroy(coro); + flb_free(out_flush); + + return NULL; + } + + p_size = evc->size; + + chunk_offset = 0; + serialization_buffer_offset = 0; + + while ((ret = ctr_decode_msgpack_create( + &trace_context, + (char *) evc->data, + evc->size, + &chunk_offset)) == CTR_DECODE_MSGPACK_SUCCESS) { + ret = flb_processor_run(o_ins->processor, + 0, + FLB_PROCESSOR_TRACES, + evc->tag, + flb_sds_len(evc->tag), + (char *) trace_context, + 0, + NULL, + NULL); + + if (ret == 0) { + ret = ctr_encode_msgpack_create(trace_context, + &serialized_context_buffer, + &serialized_context_size); + + ctr_destroy(trace_context); + + if (ret != 0) { + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + + return NULL; + } + + if ((serialization_buffer_offset + + serialized_context_size) > p_size) { + resized_serialization_buffer = \ + flb_realloc(p_buf, p_size + serialized_context_size); + + if (resized_serialization_buffer == NULL) { + ctr_encode_msgpack_destroy(serialized_context_buffer); + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + + return NULL; + } + + p_size += serialized_context_size; + p_buf = resized_serialization_buffer; + } + + memcpy(&(((char *) p_buf)[serialization_buffer_offset]), + serialized_context_buffer, + serialized_context_size); + + serialization_buffer_offset += serialized_context_size; + + ctr_encode_msgpack_destroy(serialized_context_buffer); + } + } + + if (serialization_buffer_offset == 0) { + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + + return NULL; + } + + out_flush->processed_event_chunk = flb_event_chunk_create( + evc->type, + 0, + evc->tag, + flb_sds_len(evc->tag), + p_buf, + p_size); + + if (out_flush->processed_event_chunk == NULL) { + flb_coro_destroy(coro); + flb_free(out_flush); + flb_free(p_buf); + + return NULL; + } + } + } + + coro->caller = co_active(); + coro->callee = co_create(config->coro_stack_size, + output_pre_cb_flush, &stack_size); + + if (coro->callee == NULL) { + flb_coro_destroy(coro); + if (out_flush->processed_event_chunk) { + flb_free(out_flush->processed_event_chunk->data); + flb_event_chunk_destroy(out_flush->processed_event_chunk); + } + flb_free(out_flush); + return NULL; + } + +#ifdef FLB_HAVE_VALGRIND + coro->valgrind_stack_id = \ + VALGRIND_STACK_REGISTER(coro->callee, ((char *) coro->callee) + stack_size); +#endif + + if (o_ins->is_threaded == FLB_TRUE) { + th_ins = flb_output_thread_instance_get(); + + pthread_mutex_lock(&th_ins->flush_mutex); + mk_list_add(&out_flush->_head, &th_ins->flush_list); + pthread_mutex_unlock(&th_ins->flush_mutex); + } + else { + mk_list_add(&out_flush->_head, &o_ins->flush_list); + } + + /* Workaround for makecontext() */ + output_params_set(out_flush, coro, task, o_ins->p, o_ins->context, config); + return out_flush; +} + +/* + * This function is used by the output plugins to return. It's mandatory + * as it will take care to signal the event loop letting know the flush + * callback has done. + * + * The signal emmited indicate the 'Task' number that have finished plus + * a return value. The return value is either FLB_OK, FLB_RETRY or FLB_ERROR. + */ +static inline void flb_output_return(int ret, struct flb_coro *co) { + int n; + int pipe_fd; + uint32_t set; + uint64_t val; + struct flb_task *task; + struct flb_output_flush *out_flush; + struct flb_output_instance *o_ins; + struct flb_out_thread_instance *th_ins = NULL; + + out_flush = (struct flb_output_flush *) co->data; + o_ins = out_flush->o_ins; + task = out_flush->task; + + flb_task_acquire_lock(task); + + flb_task_deactivate_route(task, o_ins); + + flb_task_release_lock(task); + + if (out_flush->processed_event_chunk) { + if (task->event_chunk->data != out_flush->processed_event_chunk->data) { + flb_free(out_flush->processed_event_chunk->data); + } + + flb_event_chunk_destroy(out_flush->processed_event_chunk); + out_flush->processed_event_chunk = NULL; + } + + /* + * To compose the signal event the relevant info is: + * + * - Unique Task events id: 2 in this case + * - Return value: FLB_OK (0), FLB_ERROR (1) or FLB_RETRY (2) + * - Task ID + * - Output Instance ID (struct flb_output_instance)->id + * + * We put together the return value with the task_id on the 32 bits at right + */ + set = FLB_TASK_SET(ret, task->id, o_ins->id); + val = FLB_BITS_U64_SET(2 /* FLB_ENGINE_TASK */, set); + + /* + * Set the target pipe channel: if this return code is running inside a + * thread pool worker, use the specific worker pipe/event loop to handle + * the return status, otherwise use the channel connected to the parent + * event loop. + */ + if (flb_output_is_threaded(o_ins) == FLB_TRUE) { + /* Retrieve the thread instance and prepare pipe channel */ + th_ins = flb_output_thread_instance_get(); + pipe_fd = th_ins->ch_thread_events[1]; + } + else { + pipe_fd = out_flush->o_ins->ch_events[1]; + } + + /* Notify the event loop about our return status */ + n = flb_pipe_w(pipe_fd, (void *) &val, sizeof(val)); + if (n == -1) { + flb_errno(); + } + + /* + * Prepare the co-routine to be destroyed: real-destroy happens in the + * event loop cleanup functions. + */ + flb_output_flush_prepare_destroy(out_flush); +} + +/* return the number of co-routines running in the instance */ +static inline int flb_output_coros_size(struct flb_output_instance *ins) +{ + int size = 0; + + if (flb_output_is_threaded(ins) == FLB_TRUE) { + /* + * On threaded mode, we need to count the active co-routines of + * every running thread of the thread pool. + */ + size = flb_output_thread_pool_coros_size(ins); + } + else { + size = mk_list_size(&ins->flush_list); + } + + return size; +} + +static inline void flb_output_return_do(int x) +{ + struct flb_coro *coro; + + coro = flb_coro_get(); + flb_output_return(x, coro); + /* + * Each co-routine handler have different ways to handle a return, + * just use the wrapper. + */ + flb_coro_yield(coro, FLB_TRUE); +} + +#define FLB_OUTPUT_RETURN(x) \ + flb_output_return_do(x); \ + return + +static inline int flb_output_config_map_set(struct flb_output_instance *ins, + void *context) +{ + int ret; + + /* Process normal properties */ + if (ins->config_map) { + ret = flb_config_map_set(&ins->properties, ins->config_map, context); + if (ret == -1) { + return -1; + } + } + + /* Net properties */ + if (ins->net_config_map) { + ret = flb_config_map_set(&ins->net_properties, ins->net_config_map, + &ins->net_setup); + if (ret == -1) { + return -1; + } + } + + return 0; +} + +int flb_output_help(struct flb_output_instance *ins, void **out_buf, size_t *out_size); + +struct flb_output_instance *flb_output_get_instance(struct flb_config *config, + int out_id); +int flb_output_flush_finished(struct flb_config *config, int out_id); + +int flb_output_task_singleplex_enqueue(struct flb_task_queue *queue, + struct flb_task_retry *retry, + struct flb_task *task, + struct flb_output_instance *out_ins, + struct flb_config *config); +int flb_output_task_singleplex_flush_next(struct flb_task_queue *queue); +struct flb_output_instance *flb_output_new(struct flb_config *config, + const char *output, void *data, + int public_only); +const char *flb_output_name(struct flb_output_instance *in); +int flb_output_set_property(struct flb_output_instance *out, + const char *k, const char *v); +const char *flb_output_get_property(const char *key, struct flb_output_instance *ins); +#ifdef FLB_HAVE_METRICS +void *flb_output_get_cmt_instance(struct flb_output_instance *ins); +#endif +void flb_output_net_default(const char *host, int port, + struct flb_output_instance *ins); +const char *flb_output_name(struct flb_output_instance *ins); +void flb_output_pre_run(struct flb_config *config); +void flb_output_exit(struct flb_config *config); +void flb_output_set_context(struct flb_output_instance *ins, void *context); +int flb_output_instance_destroy(struct flb_output_instance *ins); +int flb_output_net_property_check(struct flb_output_instance *ins, + struct flb_config *config); +int flb_output_plugin_property_check(struct flb_output_instance *ins, + struct flb_config *config); +int flb_output_init_all(struct flb_config *config); +int flb_output_check(struct flb_config *config); +int flb_output_log_check(struct flb_output_instance *ins, int l); + +int flb_output_upstream_set(struct flb_upstream *u, struct flb_output_instance *ins); +int flb_output_upstream_ha_set(void *ha, struct flb_output_instance *ins); + +void flb_output_prepare(); +int flb_output_set_http_debug_callbacks(struct flb_output_instance *ins); + +int flb_output_task_flush(struct flb_task *task, + struct flb_output_instance *out_ins, + struct flb_config *config); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_output_plugin.h b/fluent-bit/include/fluent-bit/flb_output_plugin.h new file mode 100644 index 00000000..6340202d --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_output_plugin.h @@ -0,0 +1,64 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUTPUT_PLUGIN_H +#define FLB_OUTPUT_PLUGIN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_output.h> +#include <fluent-bit/flb_log.h> +#include <cmetrics/cmetrics.h> +#include <cmetrics/cmt_encode_text.h> +#include <cmetrics/cmt_encode_prometheus.h> + +#define flb_output_plugin_log_suppress_check(ins, fmt, ...) \ + flb_log_suppress_check(ins->log_suppress_interval, fmt, ##__VA_ARGS__) + +#define flb_plg_error(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_ERROR)) \ + if (flb_output_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_ERROR, NULL, 0, "[output:%s:%s] " fmt, \ + ctx->p->name, flb_output_name(ctx), ##__VA_ARGS__) + +#define flb_plg_warn(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_WARN)) \ + if (flb_output_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_WARN, NULL, 0, "[output:%s:%s] " fmt, \ + ctx->p->name, flb_output_name(ctx), ##__VA_ARGS__) + +#define flb_plg_info(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_INFO)) \ + if (flb_output_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_INFO, NULL, 0, "[output:%s:%s] " fmt, \ + ctx->p->name, flb_output_name(ctx), ##__VA_ARGS__) + +#define flb_plg_debug(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_DEBUG)) \ + if (flb_output_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_DEBUG, NULL, 0, "[output:%s:%s] " fmt, \ + ctx->p->name, flb_output_name(ctx), ##__VA_ARGS__) + +#define flb_plg_trace(ctx, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, FLB_LOG_TRACE)) \ + if (flb_output_plugin_log_suppress_check(ctx, fmt, ##__VA_ARGS__) == FLB_FALSE) \ + flb_log_print(FLB_LOG_TRACE, NULL, 0, \ + "[output:%s:%s at %s:%i] " fmt, \ + ctx->p->name, flb_output_name(ctx), __FLB_FILENAME__, \ + __LINE__, ##__VA_ARGS__) +#endif diff --git a/fluent-bit/include/fluent-bit/flb_output_thread.h b/fluent-bit/include/fluent-bit/flb_output_thread.h new file mode 100644 index 00000000..4100c9c1 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_output_thread.h @@ -0,0 +1,109 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUTPUT_THREAD_H +#define FLB_OUTPUT_THREAD_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_upstream.h> +#include <fluent-bit/flb_upstream_queue.h> + +/* + * For every 'upstream' registered in the output plugin initialization, we create + * a local entry so we can manage the connections queues locally, on this way we + * avoid sharing a single list with the other threads. + */ +struct flb_out_thread_upstream { + /* output instance upstream connection context */ + struct flb_upstream *u; + + /* + * Local implementation of upstream queues: same as in the co-routines case, we + * implement our own lists of upstream connections so they can be only reused inside + * the same thread. + * + * The flb_upstream_queue structure have the following queues: + * + * - av_queue: connections in a persistent state (keepalive) ready to be + * used. + * + * - busy_queue: connections doing I/O, being used by a co-routine. + * + * - destroy_queue: connections that cannot be longer used, the connections linked + * to this list will be destroyed in the event loop once there is + * no pending events associated. + * + * note: in single-thread mode, the same fields are in 'struct flb_upstream' + */ + struct flb_upstream_queue queue; + + /* Link to struct flb_out_thread_instance->upstreams */ + struct mk_list _head; +}; + +struct flb_out_thread_instance { + struct mk_event event; /* event context to associate events */ + struct mk_event_loop *evl; /* thread event loop context */ + struct flb_bucket_queue *evl_bktq; /* bucket queue for evl track event priority */ + flb_pipefd_t ch_parent_events[2]; /* channel to receive parent notifications */ + flb_pipefd_t ch_thread_events[2]; /* channel to send messages local event loop */ + struct flb_output_instance *ins; /* output plugin instance */ + struct flb_config *config; + struct flb_tp_thread *th; + struct mk_list _head; + + /* + * In multithread mode, we move some contexts to independent references per thread + * so we can avoid to have shared resources and mutexes. + * + * The following 'coro' fields maintains a state of co-routines inside the thread + * event loop. + * + * note: in single-thread mode, the same fields are in 'struct flb_output_instance'. + */ + int flush_id; /* coroutine id counter */ + struct mk_list flush_list; /* flush context list */ + struct mk_list flush_list_destroy; /* flust context destroy list */ + + /* + * If the main engine (parent thread) needs to query the number of active + * 'flushes' running by a threaded instance, then the access to the 'flush_list' + * must be protected: we use 'flush_mutex for that purpose. + */ + pthread_mutex_t flush_mutex; /* mutex for 'flush_list' */ + + /* List of mapped 'upstream' contexts */ + struct mk_list upstreams; +}; + +int flb_output_thread_pool_create(struct flb_config *config, + struct flb_output_instance *ins); +int flb_output_thread_pool_coros_size(struct flb_output_instance *ins); +void flb_output_thread_pool_destroy(struct flb_output_instance *ins); +int flb_output_thread_pool_start(struct flb_output_instance *ins); +int flb_output_thread_pool_flush(struct flb_task *task, + struct flb_output_instance *out_ins, + struct flb_config *config); + + +void flb_output_thread_instance_init(); +struct flb_out_thread_instance *flb_output_thread_instance_get(); +void flb_output_thread_instance_set(struct flb_out_thread_instance *th_ins); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_pack.h b/fluent-bit/include/fluent-bit/flb_pack.h new file mode 100644 index 00000000..8e4c8831 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_pack.h @@ -0,0 +1,125 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PACK_H +#define FLB_PACK_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_jsmn.h> +#include <fluent-bit/flb_config.h> + +#include <msgpack.h> + +/* JSON types */ +#define FLB_PACK_JSON_UNDEFINED JSMN_UNDEFINED +#define FLB_PACK_JSON_OBJECT JSMN_OBJECT +#define FLB_PACK_JSON_ARRAY JSMN_ARRAY +#define FLB_PACK_JSON_STRING JSMN_STRING +#define FLB_PACK_JSON_PRIMITIVE JSMN_PRIMITIVE + +/* Date formats */ +#define FLB_PACK_JSON_DATE_DOUBLE 0 +#define FLB_PACK_JSON_DATE_ISO8601 1 +#define FLB_PACK_JSON_DATE_EPOCH 2 +#define FLB_PACK_JSON_DATE_JAVA_SQL_TIMESTAMP 3 +#define FLB_PACK_JSON_DATE_EPOCH_MS 4 + +/* Specific ISO8601 format */ +#define FLB_PACK_JSON_DATE_ISO8601_FMT "%Y-%m-%dT%H:%M:%S" + +/* Specific Java SQL Timestamp format */ +#define FLB_PACK_JSON_DATE_JAVA_SQL_TIMESTAMP_FMT "%Y-%m-%d %H:%M:%S" + +#define FBL_PACK_JSON_DATE_FORMAT_DESCRIPTION "Specify the format of the date, " \ + "supported formats: double, " \ + "iso8601 (e.g: 2018-05-30T09:39:52.000681Z), " \ + "java_sql_timestamp (e.g: 2018-05-30 09:39:52.000681, useful for AWS Athena), "\ + "and epoch." + +/* JSON formats (modes) */ +#define FLB_PACK_JSON_FORMAT_NONE 0 +#define FLB_PACK_JSON_FORMAT_JSON 1 +#define FLB_PACK_JSON_FORMAT_STREAM 2 +#define FLB_PACK_JSON_FORMAT_LINES 3 + +struct flb_pack_state { + int multiple; /* support multiple jsons? */ + int tokens_count; /* number of parsed tokens */ + int tokens_size; /* total tokens in the array */ + int last_byte; /* last byte of a full msg */ + jsmntok_t *tokens; /* tokens array */ + jsmn_parser parser; /* parser state */ + char *buf_data; /* temporary buffer */ + size_t buf_size; /* temporary buffer size */ + size_t buf_len; /* temporary buffer length */ +}; + +int flb_pack_init(struct flb_config *config); +int flb_json_tokenise(const char *js, size_t len, struct flb_pack_state *state); + + +int flb_pack_json(const char *js, size_t len, char **buffer, size_t *size, + int *root_type, size_t *consumed); +int flb_pack_json_recs(const char *js, size_t len, char **buffer, size_t *size, + int *root_type, int *out_records, size_t *consumed); + +int flb_pack_state_init(struct flb_pack_state *s); +void flb_pack_state_reset(struct flb_pack_state *s); + +int flb_pack_json_state(const char *js, size_t len, + char **buffer, int *size, + struct flb_pack_state *state); +int flb_pack_json_valid(const char *json, size_t len); + +flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, + int json_format, int date_format, + flb_sds_t date_key); +int flb_pack_to_json_format_type(const char *str); +int flb_pack_to_json_date_type(const char *str); + +void flb_pack_print(const char *data, size_t bytes); +int flb_msgpack_to_json(char *json_str, size_t str_len, + const msgpack_object *obj); +char* flb_msgpack_to_json_str(size_t size, const msgpack_object *obj); +flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size); + +int flb_pack_time_now(msgpack_packer *pck); +int flb_msgpack_expand_map(char *map_data, size_t map_size, + msgpack_object_kv **obj_arr, int obj_arr_len, + char** out_buf, int* out_size); + +struct flb_gelf_fields { + flb_sds_t timestamp_key; + flb_sds_t host_key; + flb_sds_t short_message_key; + flb_sds_t full_message_key; + flb_sds_t level_key; +}; + +flb_sds_t flb_msgpack_to_gelf(flb_sds_t *s, msgpack_object *o, + struct flb_time *tm, + struct flb_gelf_fields *fields); + +flb_sds_t flb_msgpack_raw_to_gelf(char *buf, size_t buf_size, + struct flb_time *tm, + struct flb_gelf_fields *fields); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_parser.h b/fluent-bit/include/fluent-bit/flb_parser.h new file mode 100644 index 00000000..7d5cfcbc --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_parser.h @@ -0,0 +1,117 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit/flb_info.h> + +#if !defined(FLB_PARSER_H) && defined(FLB_HAVE_REGEX) +#define FLB_PARSER_H + +#include <fluent-bit/flb_regex.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_time.h> +#include <msgpack.h> + +#define FLB_PARSER_REGEX 1 +#define FLB_PARSER_JSON 2 +#define FLB_PARSER_LTSV 3 +#define FLB_PARSER_LOGFMT 4 + +struct flb_parser_types { + char *key; + int key_len; + int type; +}; + +struct flb_parser { + /* configuration */ + int type; /* parser type */ + char *name; /* format name */ + char *p_regex; /* pattern for main regular expression */ + int skip_empty; /* skip empty regex matches */ + char *time_fmt; /* time format */ + char *time_fmt_full; /* original given time format */ + char *time_key; /* field name that contains the time */ + int time_offset; /* fixed UTC offset */ + int time_keep; /* keep time field */ + int time_strict; /* parse time field strictly */ + int logfmt_no_bare_keys; /* in logfmt parsers, require all keys to have values */ + char *time_frac_secs; /* time format have fractional seconds ? */ + struct flb_parser_types *types; /* type casting */ + int types_len; + + /* Field decoders */ + struct mk_list *decoders; + + /* internal */ + int time_with_year; /* do time_fmt consider a year (%Y) ? */ + char *time_fmt_year; + int time_with_tz; /* do time_fmt consider a timezone ? */ + struct flb_regex *regex; + struct mk_list _head; +}; + +enum { + FLB_PARSER_TYPE_INT = 1, + FLB_PARSER_TYPE_FLOAT, + FLB_PARSER_TYPE_BOOL, + FLB_PARSER_TYPE_STRING, + FLB_PARSER_TYPE_HEX, +}; + +static inline time_t flb_parser_tm2time(const struct flb_tm *src) +{ + struct tm tmp; + time_t res; + + tmp = src->tm; + res = timegm(&tmp) - flb_tm_gmtoff(src); + return res; +} + + +struct flb_parser *flb_parser_create(const char *name, const char *format, + const char *p_regex, + int skip_empty, + const char *time_fmt, const char *time_key, + const char *time_offset, + int time_keep, + int time_strict, + int logfmt_no_bare_keys, + struct flb_parser_types *types, + int types_len, + struct mk_list *decoders, + struct flb_config *config); +int flb_parser_conf_file_stat(const char *file, struct flb_config *config); +int flb_parser_conf_file(const char *file, struct flb_config *config); +void flb_parser_destroy(struct flb_parser *parser); +struct flb_parser *flb_parser_get(const char *name, struct flb_config *config); +int flb_parser_do(struct flb_parser *parser, const char *buf, size_t length, + void **out_buf, size_t *out_size, struct flb_time *out_time); + +void flb_parser_exit(struct flb_config *config); +int flb_parser_tzone_offset(const char *str, int len, int *tmdiff); +int flb_parser_time_lookup(const char *time, size_t tsize, time_t now, + struct flb_parser *parser, + struct flb_tm *tm, double *ns); +int flb_parser_typecast(const char *key, int key_len, + const char *val, int val_len, + msgpack_packer *pck, + struct flb_parser_types *types, + int types_len); +#endif diff --git a/fluent-bit/include/fluent-bit/flb_parser_decoder.h b/fluent-bit/include/fluent-bit/flb_parser_decoder.h new file mode 100644 index 00000000..7657d691 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_parser_decoder.h @@ -0,0 +1,68 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PARSER_DECODER_H +#define FLB_PARSER_DECODER_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_config_format.h> + +/* Decoder behavior */ +#define FLB_PARSER_DEC_DEFAULT 0 /* results place as separate keys */ +#define FLB_PARSER_DEC_AS 1 /* results replace current key/value */ + +/* Decoder Backend */ +#define FLB_PARSER_DEC_JSON 0 /* decode_json() */ +#define FLB_PARSER_DEC_ESCAPED 1 /* decode_escaped() */ +#define FLB_PARSER_DEC_ESCAPED_UTF8 2 /* decode_escaped_utf8() */ +#define FLB_PARSER_DEC_MYSQL_QUOTED 3 /* decode_mysql_quoted() */ + +/* Decoder actions */ +#define FLB_PARSER_ACT_NONE 0 +#define FLB_PARSER_ACT_TRY_NEXT 1 +#define FLB_PARSER_ACT_DO_NEXT 2 + +#define FLB_PARSER_DEC_BUF_SIZE 1024*8 /* 8KB */ + +struct flb_parser_dec_rule { + int type; /* decode_field, decode_field_as */ + int backend; /* backend handler: json, escaped */ + int action; /* actions: try_next, do_next */ + + /* Link to flb_parser_dec->rules list head */ + struct mk_list _head; +}; + +struct flb_parser_dec { + flb_sds_t key; + flb_sds_t buffer; /* temporary buffer for decoding work */ + int add_extra_keys; /* if type == FLB_PARSER_DEC_DEFAULT, flag is True */ + struct mk_list rules; /* list head for decoder key rules */ + struct mk_list _head; /* link to parser->decoders */ +}; + +struct mk_list *flb_parser_decoder_list_create(struct flb_cf_section *section); + +int flb_parser_decoder_list_destroy(struct mk_list *list); +int flb_parser_decoder_do(struct mk_list *decoders, + const char *in_buf, size_t in_size, + char **out_buf, size_t *out_size); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_pipe.h b/fluent-bit/include/fluent-bit/flb_pipe.h new file mode 100644 index 00000000..4ff1d79b --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_pipe.h @@ -0,0 +1,47 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PIPE_H +#define FLB_PIPE_H + +#include <fluent-bit/flb_compat.h> + +#ifdef _WIN32 +#include <event.h> +#define flb_pipefd_t evutil_socket_t +#define flb_sockfd_t evutil_socket_t +#define flb_pipe_w(fd, buf, len) send(fd, buf, len, 0) +#define flb_pipe_r(fd, buf, len) recv(fd, buf, len, 0) +#define FLB_PIPE_WOULDBLOCK() (WSAGetLastError() == WSAEWOULDBLOCK) +#else +#define flb_pipefd_t int +#define flb_sockfd_t int +#define flb_pipe_w(fd, buf, len) write(fd, buf, len) +#define flb_pipe_r(fd, buf, len) read(fd, buf, len) +#define FLB_PIPE_WOULDBLOCK() (errno == EAGAIN || errno == EWOULDBLOCK) +#endif + +int flb_pipe_create(flb_pipefd_t pipefd[2]); +void flb_pipe_destroy(flb_pipefd_t pipefd[2]); +int flb_pipe_close(flb_pipefd_t fd); +int flb_pipe_set_nonblocking(flb_pipefd_t fd); +ssize_t flb_pipe_read_all(int fd, void *buf, size_t count); +ssize_t flb_pipe_write_all(int fd, const void *buf, size_t count); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_plugin.h b/fluent-bit/include/fluent-bit/flb_plugin.h new file mode 100644 index 00000000..b53b4c55 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_plugin.h @@ -0,0 +1,54 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PLUGIN_H +#define FLB_PLUGIN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <monkey/mk_core.h> + +#define FLB_PLUGIN_INPUT 1 +#define FLB_PLUGIN_FILTER 2 +#define FLB_PLUGIN_OUTPUT 3 +#define FLB_PLUGIN_PROCESSOR 4 + +/* Informational contexts for discovered dynamic plugins */ +struct flb_plugin { + int type; /* plugin type */ + flb_sds_t path; /* path for .so file */ + void *dso_handle; /* shared object handler */ + struct mk_list _head; /* link to struct flb_plugins */ +}; + +struct flb_plugins { + struct mk_list input; + struct mk_list processor; + struct mk_list filter; + struct mk_list output; +}; + +struct flb_plugins *flb_plugin_create(); +int flb_plugin_load(char *path, struct flb_plugins *ctx, + struct flb_config *config); +int flb_plugin_load_router(char *path, struct flb_config *config); +int flb_plugin_load_config_file(const char *file, struct flb_config *config); +void flb_plugin_destroy(struct flb_plugins *ctx); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_plugin_proxy.h b/fluent-bit/include/fluent-bit/flb_plugin_proxy.h new file mode 100644 index 00000000..08182141 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_plugin_proxy.h @@ -0,0 +1,85 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PLUGIN_PROXY_H +#define FLB_PLUGIN_PROXY_H + +#include <monkey/mk_core.h> +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_output.h> +#include <fluent-bit/flb_input_thread.h> + +/* Plugin Types */ +#define FLB_PROXY_INPUT_PLUGIN 1 +#define FLB_PROXY_OUTPUT_PLUGIN 2 + +/* Proxies available */ +#define FLB_PROXY_GOLANG 11 + +struct flb_plugin_proxy_def { + /* Fields populated once remote flb_cb_register() is called */ + int type; /* defined by FLB_PROXY_[INPUT|OUTPUT]_PLUGIN */ + int proxy; /* proxy type */ + int flags; + char *name; /* plugin short name */ + char *description; /* plugin description */ +}; + +/* Proxy context */ +struct flb_plugin_proxy { + /* Fields populated once remote flb_cb_register() is called */ + struct flb_plugin_proxy_def *def; + + /* Internal */ + struct flb_api *api; /* API context to export functions */ + void *instance; /* input/output instance */ + void *dso_handler; /* dso handler - dlopen(2) */ + void *data; /* opaque data type for specific proxy handler */ + struct mk_list _head; /* link to parent config->proxies */ +}; + +/* This is the context for proxy plugins */ +struct flb_plugin_proxy_context { + /* This context is set by the remote init and is passed to remote flush */ + void *remote_context; + /* A proxy ptr is needed to detect the proxy type/lang (OUTPUT/GOLANG) */ + struct flb_plugin_proxy *proxy; +}; + +struct flb_plugin_input_proxy_context { + int coll_fd; + /* A proxy ptr is needed to store the proxy type/lang (OUTPUT/GOLANG) */ + struct flb_plugin_proxy *proxy; +}; + +void *flb_plugin_proxy_symbol(struct flb_plugin_proxy *proxy, + const char *symbol); + +int flb_plugin_proxy_output_init(struct flb_plugin_proxy *proxy, + struct flb_output_instance *o_ins, + struct flb_config *config); + +int flb_plugin_proxy_register(struct flb_plugin_proxy *proxy, + struct flb_config *config); + +struct flb_plugin_proxy *flb_plugin_proxy_create(const char *dso_path, int type, + struct flb_config *config); +int flb_plugin_proxy_load_all(struct flb_config *config); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_plugins.h.in b/fluent-bit/include/fluent-bit/flb_plugins.h.in new file mode 100644 index 00000000..e1491b68 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_plugins.h.in @@ -0,0 +1,100 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PLUGINS_H +#define FLB_PLUGINS_H + +#include <monkey/mk_core.h> +#include <fluent-bit/flb_custom.h> +#include <fluent-bit/flb_input.h> +#include <fluent-bit/flb_output.h> +#include <fluent-bit/flb_filter.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_log.h> + +@FLB_IN_PLUGINS_DECL@ +@FLB_OUT_PLUGINS_DECL@ +@FLB_FILTER_PLUGINS_DECL@ +@FLB_PROCESSOR_PLUGINS_DECL@ + +int flb_plugins_register(struct flb_config *config) +{ + struct flb_custom_plugin *custom; + struct flb_input_plugin *in; + struct flb_output_plugin *out; + struct flb_filter_plugin *filter; + struct flb_processor_plugin *processor; + +@FLB_CUSTOM_PLUGINS_ADD@ +@FLB_IN_PLUGINS_ADD@ +@FLB_OUT_PLUGINS_ADD@ +@FLB_FILTER_PLUGINS_ADD@ +@FLB_PROCESSOR_PLUGINS_ADD@ + + return 0; +} + +void flb_plugins_unregister(struct flb_config *config) +{ + struct mk_list *tmp; + struct mk_list *head; + struct flb_custom_plugin *custom; + struct flb_input_plugin *in; + struct flb_output_plugin *out; + struct flb_filter_plugin *filter; + struct flb_processor_plugin *processor; + + mk_list_foreach_safe(head, tmp, &config->custom_plugins) { + custom = mk_list_entry(head, struct flb_custom_plugin, _head); + mk_list_del(&custom->_head); + flb_free(custom); + } + + mk_list_foreach_safe(head, tmp, &config->in_plugins) { + in = mk_list_entry(head, struct flb_input_plugin, _head); + if(in->cb_destroy) { + in->cb_destroy(in); + } + mk_list_del(&in->_head); + flb_free(in); + } + + mk_list_foreach_safe(head, tmp, &config->out_plugins) { + out = mk_list_entry(head, struct flb_output_plugin, _head); + if(out->cb_destroy) { + out->cb_destroy(out); + } + mk_list_del(&out->_head); + flb_free(out); + } + + mk_list_foreach_safe(head, tmp, &config->filter_plugins) { + filter = mk_list_entry(head, struct flb_filter_plugin, _head); + mk_list_del(&filter->_head); + flb_free(filter); + } + + mk_list_foreach_safe(head, tmp, &config->processor_plugins) { + processor = mk_list_entry(head, struct flb_processor_plugin, _head); + mk_list_del(&processor->_head); + flb_free(processor); + } +} + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_processor.h b/fluent-bit/include/fluent-bit/flb_processor.h new file mode 100644 index 00000000..591604fb --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_processor.h @@ -0,0 +1,260 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2023 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PROCESSOR_H +#define FLB_PROCESSOR_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_config_map.h> +#include <fluent-bit/flb_config_format.h> + +#include <ctraces/ctraces.h> +#include <cmetrics/cmetrics.h> + +/* Processor plugin result values */ +#define FLB_PROCESSOR_SUCCESS 0 +#define FLB_PROCESSOR_FAILURE -1 + +/* Processor event types */ +#define FLB_PROCESSOR_LOGS 1 +#define FLB_PROCESSOR_METRICS 2 +#define FLB_PROCESSOR_TRACES 4 + +/* Type of processor unit: 'pipeline filter' or 'native unit' */ +#define FLB_PROCESSOR_UNIT_NATIVE 0 +#define FLB_PROCESSOR_UNIT_FILTER 1 + + +/* The current values mean the processor stack will + * wait for 2 seconds at most in 50 millisecond increments + * for each processor unit. + * + * This is the worst case scenario and in reality there will + * be no wait in 99.9% of the cases. + */ +#define FLB_PROCESSOR_LOCK_RETRY_LIMIT 40 +#define FLB_PROCESSOR_LOCK_RETRY_DELAY 50000 + +/* These forward definitions are necessary in order to avoid + * inclussion conflicts. + */ + +struct flb_log_event; +struct flb_input_instance; +struct flb_log_event_decoder; +struct flb_log_event_encoder; +struct flb_processor_instance; + +struct flb_processor_unit { + int event_type; + int unit_type; + flb_sds_t name; + size_t stage; + + /* + * Opaque data type for custom reference (for pipeline filters this + * contains the filter instance context. + */ + void *ctx; + + /* This lock is meant to cover the case where two output plugin + * worker threads flb_output_flush_create calls overlap which + * could cause flb_processor_run to be invoked by both of them + * at the same time with the same context. + * + * This could cause certain non thread aware filters such as + * filter_lua to modify internal structures leading to corruption + * and crashes. + */ + pthread_mutex_t lock; + /* + * pipeline filters needs to be linked somewhere since the destroy + * function will do the mk_list_del(). To avoid corruptions we link + * normal filters here, this list is never iterated or used besides + * for this purpose. + */ + struct mk_list unused_list; + + /* link to struct flb_processor->(logs, metrics, traces) list */ + struct mk_list _head; + + /* link to parent processor */ + void *parent; +}; + +struct flb_processor { + int is_active; + + /* user-defined processor name */ + flb_sds_t name; + + /* lists for different types */ + struct mk_list logs; + struct mk_list metrics; + struct mk_list traces; + + size_t stage_count; + /* + * opaque data type to reference anything specific from the caller, for input + * plugins this will contain the input instance context. + */ + void *data; + int source_plugin_type; + + /* Fluent Bit context */ + struct flb_config *config; +}; + +struct flb_processor_plugin { + int flags; /* Flags (not available at the moment */ + char *name; /* Processor short name */ + char *description; /* Description */ + + /* Config map */ + struct flb_config_map *config_map; + + /* Callbacks */ + int (*cb_init) (struct flb_processor_instance *, + void *, + int, + struct flb_config *); + + int (*cb_process_logs) (struct flb_processor_instance *, + struct flb_log_event_encoder *, + struct flb_log_event *, + const char *, + int); + + int (*cb_process_metrics) (struct flb_processor_instance *, + struct cmt *, + const char *, + int); + + int (*cb_process_traces) (struct flb_processor_instance *, + struct ctrace *, + const char *, + int); + + int (*cb_exit) (struct flb_processor_instance *); + + struct mk_list _head; /* Link to parent list (config->filters) */ +}; + +struct flb_processor_instance { + int id; /* instance id */ + int log_level; /* instance log level */ + char name[32]; /* numbered name */ + char *alias; /* alias name */ + void *context; /* Instance local context */ + void *data; + struct flb_processor_plugin *p; /* original plugin */ + struct mk_list properties; /* config properties */ + struct mk_list *config_map; /* configuration map */ + + struct flb_log_event_decoder *log_decoder; + struct flb_log_event_encoder *log_encoder; + + /* + * CMetrics + * -------- + */ + struct cmt *cmt; /* parent context */ + + /* Keep a reference to the original context this instance belongs to */ + struct flb_config *config; +}; + + +/* Processor stack */ + +struct flb_processor *flb_processor_create(struct flb_config *config, + char *name, + void *source_plugin_instance, + int source_plugin_type); + +int flb_processor_is_active(struct flb_processor *proc); + +int flb_processor_init(struct flb_processor *proc); +void flb_processor_destroy(struct flb_processor *proc); + +int flb_processor_run(struct flb_processor *proc, + size_t starting_stage, + int type, + const char *tag, size_t tag_len, + void *data, size_t data_size, + void **out_buf, size_t *out_size); + + +struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, + int event_type, + char *unit_name); +void flb_processor_unit_destroy(struct flb_processor_unit *pu); +int flb_processor_unit_set_property(struct flb_processor_unit *pu, const char *k, struct cfl_variant *v); + +int flb_processors_load_from_config_format_group(struct flb_processor *proc, struct flb_cf_group *g); + +/* Processor plugin instance */ + +struct flb_processor_instance *flb_processor_instance_create( + struct flb_config *config, + const char *name, + void *data); + +void flb_processor_instance_destroy( + struct flb_processor_instance *ins); + +int flb_processor_instance_init( + struct flb_processor_instance *ins, + void *source_plugin_instance, + int source_plugin_type, + struct flb_config *config); + +void flb_processor_instance_exit( + struct flb_processor_instance *ins, + struct flb_config *config); + +void flb_processor_instance_set_context( + struct flb_processor_instance *ins, + void *context); + +int flb_processor_instance_check_properties( + struct flb_processor_instance *ins, + struct flb_config *config); + +int flb_processor_instance_set_property( + struct flb_processor_instance *ins, + const char *k, const char *v); + +const char *flb_processor_instance_get_property( + const char *key, + struct flb_processor_instance *ins); + +const char *flb_processor_instance_get_name( + struct flb_processor_instance *ins); + +static inline int flb_processor_instance_config_map_set( + struct flb_processor_instance *ins, + void *context) +{ + return flb_config_map_set(&ins->properties, ins->config_map, context); +} + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_processor_plugin.h b/fluent-bit/include/fluent-bit/flb_processor_plugin.h new file mode 100644 index 00000000..39d1a18a --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_processor_plugin.h @@ -0,0 +1,47 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FILTER_PLUGIN_H +#define FLB_FILTER_PLUGIN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_processor.h> +#include <fluent-bit/flb_log.h> + +#define flb_plg_log(ctx, level, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, level)) \ + flb_log_print(level, NULL, 0, "[processor:%s:%s] " fmt, \ + ctx->p->name, \ + flb_processor_instance_get_name(ctx), ##__VA_ARGS__) + +#define flb_plg_error(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_ERROR, fmt, ##__VA_ARGS__) + +#define flb_plg_warn(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_WARN, fmt, ##__VA_ARGS__) + +#define flb_plg_info(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_INFO, fmt, ##__VA_ARGS__) + +#define flb_plg_debug(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_DEBUG, fmt, ##__VA_ARGS__) + +#define flb_plg_trace(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_TRACE, fmt, ##__VA_ARGS__) +#endif diff --git a/fluent-bit/include/fluent-bit/flb_pthread.h b/fluent-bit/include/fluent-bit/flb_pthread.h new file mode 100644 index 00000000..2bad3a62 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_pthread.h @@ -0,0 +1,30 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PTHREAD_H +#define FLB_PTHREAD_H + +#include <fluent-bit/flb_info.h> +#ifdef FLB_SYSTEM_WINDOWS +#include <monkey/mk_core/external/winpthreads.h> +#else +#include <pthread.h> +#endif + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_ra_key.h b/fluent-bit/include/fluent-bit/flb_ra_key.h new file mode 100644 index 00000000..397dee70 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_ra_key.h @@ -0,0 +1,73 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_RA_KEY_H +#define FLB_RA_KEY_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_regex.h> +#include <fluent-bit/record_accessor/flb_ra_parser.h> +#include <monkey/mk_core.h> +#include <msgpack.h> + +enum ra_types { + FLB_RA_BOOL = 0, + FLB_RA_INT, + FLB_RA_FLOAT, + FLB_RA_STRING, + FLB_RA_NULL +}; + +/* condition value types */ +typedef union { + bool boolean; + int64_t i64; + double f64; + flb_sds_t string; +} ra_val; + +/* Represent any value object */ +struct flb_ra_value { + int type; + msgpack_object o; + ra_val val; +}; + +struct flb_ra_value *flb_ra_key_to_value(flb_sds_t ckey, + msgpack_object map, + struct mk_list *subkeys); +void flb_ra_key_value_destroy(struct flb_ra_value *v); + +int flb_ra_key_value_get(flb_sds_t ckey, msgpack_object map, + struct mk_list *subkeys, + msgpack_object **start_key, + msgpack_object **out_key, msgpack_object **out_val); + +int flb_ra_key_strcmp(flb_sds_t ckey, msgpack_object map, + struct mk_list *subkeys, char *str, int len); +int flb_ra_key_regex_match(flb_sds_t ckey, msgpack_object map, + struct mk_list *subkeys, struct flb_regex *regex, + struct flb_regex_search *result); +int flb_ra_key_value_append(struct flb_ra_parser *rp, msgpack_object obj, + msgpack_object *in_val, msgpack_packer *mp_pck); +int flb_ra_key_value_update(struct flb_ra_parser *rp, msgpack_object obj, + msgpack_object *in_key, msgpack_object *in_val, + msgpack_packer *mp_pck); +#endif diff --git a/fluent-bit/include/fluent-bit/flb_random.h b/fluent-bit/include/fluent-bit/flb_random.h new file mode 100644 index 00000000..d92ee7f3 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_random.h @@ -0,0 +1,29 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_RANDOM_H +#define FLB_RANDOM_H + +/* + * Fill buffer with the random bytes. Return 0 on success; + * -1 on error. + */ +int flb_random_bytes(unsigned char *buf, int len); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_record_accessor.h b/fluent-bit/include/fluent-bit/flb_record_accessor.h new file mode 100644 index 00000000..8dbcce3e --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_record_accessor.h @@ -0,0 +1,68 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_RECORD_ACCESSOR_H +#define FLB_RECORD_ACCESSOR_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_regex.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_sds_list.h> +#include <monkey/mk_core.h> +#include <msgpack.h> + +struct flb_record_accessor { + size_t size_hint; + flb_sds_t pattern; + struct mk_list list; /* List of parsed strings */ + struct mk_list _head; /* Head to custom list (only used by flb_mp.h) */ +}; + +int flb_ra_subkey_count(struct flb_record_accessor *ra); +struct flb_record_accessor *flb_ra_create(char *str, int translate_env); +void flb_ra_destroy(struct flb_record_accessor *ra); +void flb_ra_dump(struct flb_record_accessor *ra); +flb_sds_t flb_ra_translate(struct flb_record_accessor *ra, + char *tag, int tag_len, + msgpack_object map, struct flb_regex_search *result); +flb_sds_t flb_ra_translate_check(struct flb_record_accessor *ra, + char *tag, int tag_len, + msgpack_object map, struct flb_regex_search *result, + int check); +int flb_ra_is_static(struct flb_record_accessor *ra); +int flb_ra_strcmp(struct flb_record_accessor *ra, msgpack_object map, + char *str, int len); +int flb_ra_regex_match(struct flb_record_accessor *ra, msgpack_object map, + struct flb_regex *regex, + struct flb_regex_search *result); + +int flb_ra_get_kv_pair(struct flb_record_accessor *ra, msgpack_object map, + msgpack_object **start_key, + msgpack_object **out_key, msgpack_object **out_val); + +struct flb_ra_value *flb_ra_get_value_object(struct flb_record_accessor *ra, + msgpack_object map); +int flb_ra_append_kv_pair(struct flb_record_accessor *ra, msgpack_object map, + void **out_map, size_t *out_size, msgpack_object *in_val); +int flb_ra_update_kv_pair(struct flb_record_accessor *ra, msgpack_object map, + void **out_map, size_t *out_size, + msgpack_object *in_key, msgpack_object *in_val); +flb_sds_t flb_ra_create_str_from_list(struct flb_sds_list *str_list); +struct flb_record_accessor *flb_ra_create_from_list(struct flb_sds_list *str_list, int translate_env); +#endif diff --git a/fluent-bit/include/fluent-bit/flb_regex.h b/fluent-bit/include/fluent-bit/flb_regex.h new file mode 100644 index 00000000..4fb10a91 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_regex.h @@ -0,0 +1,68 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_REGEX_H +#define FLB_REGEX_H + +#include <fluent-bit/flb_info.h> + +#ifdef FLB_HAVE_REGEX + +#include <fluent-bit/flb_compat.h> + +#include <stdlib.h> +#include <stddef.h> + +struct flb_regex { + void *regex; +}; + +struct flb_regex_search { + int last_pos; + void *region; + const char *str; + void (*cb_match) (const char *, /* name */ + const char *, size_t, /* value */ + void *); /* caller data */ + void *data; +}; + +int flb_regex_init(); +struct flb_regex *flb_regex_create(const char *pattern); +ssize_t flb_regex_do(struct flb_regex *r, const char *str, size_t slen, + struct flb_regex_search *result); + +int flb_regex_match(struct flb_regex *r, unsigned char *str, size_t slen); + +int flb_regex_parse(struct flb_regex *r, struct flb_regex_search *result, + void (*cb_match) (const char *, /* name */ + const char *, size_t, /* value */ + void *), /* caller data */ + void *data); +int flb_regex_destroy(struct flb_regex *r); +int flb_regex_results_get(struct flb_regex_search *result, int i, + ptrdiff_t *start, ptrdiff_t *end); +void flb_regex_results_release(struct flb_regex_search *result); +int flb_regex_results_size(struct flb_regex_search *result); + +void flb_regex_exit(); + +#endif + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_reload.h b/fluent-bit/include/fluent-bit/flb_reload.h new file mode 100644 index 00000000..7066505a --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_reload.h @@ -0,0 +1,35 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_RELOAD_H +#define FLB_RELOAD_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_lib.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_config_format.h> + +#define FLB_RELOAD_IDLE 0 +#define FLB_RELOAD_IN_PROGRESS 1 + +int flb_reload_property_check_all(struct flb_config *config); +int flb_reload_reconstruct_cf(struct flb_cf *src_cf, struct flb_cf *dest_cf); +int flb_reload(flb_ctx_t *ctx, struct flb_cf *cf_opts); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_ring_buffer.h b/fluent-bit/include/fluent-bit/flb_ring_buffer.h new file mode 100644 index 00000000..6f11186e --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_ring_buffer.h @@ -0,0 +1,44 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_RING_BUFFER_H +#define FLB_RING_BUFFER_H + +#include <fluent-bit/flb_pipe.h> + +struct flb_ring_buffer { + void *ctx; /* pointer to backend context */ + void *event_loop; /* event loop where this ring buffer emits flush request signals */ + int flush_pending; /* flag meant to prevent flush request signal flood */ + void *signal_event; /* event loop entry for the flush request signal */ + flb_pipefd_t signal_channels[2]; /* flush request signaling channel */ + uint64_t data_window; /* 0% - 100% occupancy window flush request */ + uint64_t data_size; /* ring buffer size */ + void *data_buf; /* ring buffer */ +}; + +struct flb_ring_buffer *flb_ring_buffer_create(uint64_t size); +void flb_ring_buffer_destroy(struct flb_ring_buffer *rb); + +int flb_ring_buffer_add_event_loop(struct flb_ring_buffer *rb, void *evl, uint8_t window_size); + +int flb_ring_buffer_write(struct flb_ring_buffer *rb, void *ptr, size_t size); +int flb_ring_buffer_read(struct flb_ring_buffer *rb, void *ptr, size_t size); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_router.h b/fluent-bit/include/fluent-bit/flb_router.h new file mode 100644 index 00000000..aa3c8bf1 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_router.h @@ -0,0 +1,60 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ROUTER_H +#define FLB_ROUTER_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_input.h> +#include <fluent-bit/flb_output.h> + +struct flb_router_path { + struct flb_output_instance *ins; + struct mk_list _head; +}; + +static inline int flb_router_match_type(int in_event_type, + struct flb_output_instance *o_ins) +{ + if (in_event_type == FLB_INPUT_LOGS && + !(o_ins->event_type & FLB_OUTPUT_LOGS)) { + return FLB_FALSE; + } + else if (in_event_type == FLB_INPUT_METRICS && + !(o_ins->event_type & FLB_OUTPUT_METRICS)) { + return FLB_FALSE; + } + else if (in_event_type == FLB_INPUT_TRACES && + !(o_ins->event_type & FLB_OUTPUT_TRACES)) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +int flb_router_connect(struct flb_input_instance *in, + struct flb_output_instance *out); +int flb_router_connect_direct(struct flb_input_instance *in, + struct flb_output_instance *out); + +int flb_router_match(const char *tag, int tag_len, + const char *match, void *match_regex); +int flb_router_io_set(struct flb_config *config); +void flb_router_exit(struct flb_config *config); +#endif diff --git a/fluent-bit/include/fluent-bit/flb_routes_mask.h b/fluent-bit/include/fluent-bit/flb_routes_mask.h new file mode 100644 index 00000000..113bb138 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_routes_mask.h @@ -0,0 +1,65 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This file defines a few utility methods for handling the routing masks + * used by input chunks to keep track of which output plugins they are + * routed to. + */ +#ifndef FLB_ROUTES_MASK_H +#define FLB_ROUTES_MASK_H + +#include <limits.h> + +/* + * The routing mask is an array integers used to store a bitfield. Each + * bit represents the unique id of an output plugin. For example, the 9th + * bit in the routes_mask represents the output plugin with id = 9. + * + * A value of 1 in the bitfield means that output plugin is selected + * and a value of zero means that output is deselected. + * + * The size of the bitmask array limits the number of output plugins + * The router can route to. For example: with a value of 4 using + * 64-bit integers the bitmask can represent up to 256 output plugins + */ +#define FLB_ROUTES_MASK_ELEMENTS 4 + +/* + * How many bits are in each element of the bitmask array + */ +#define FLB_ROUTES_MASK_ELEMENT_BITS (sizeof(uint64_t) * CHAR_BIT) + +/* + * The maximum number of routes that can be stored in the array + */ +#define FLB_ROUTES_MASK_MAX_VALUE (FLB_ROUTES_MASK_ELEMENTS * FLB_ROUTES_MASK_ELEMENT_BITS) + + +/* forward declaration */ +struct flb_input_instance; + + +int flb_routes_mask_set_by_tag(uint64_t *routes_mask, const char *tag, int tag_len, struct flb_input_instance *in); +int flb_routes_mask_get_bit(uint64_t *routes_mask, int value); +void flb_routes_mask_set_bit(uint64_t *routes_mask, int value); +void flb_routes_mask_clear_bit(uint64_t *routes_mask, int value); +int flb_routes_mask_is_empty(uint64_t *routes_mask); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_s3_local_buffer.h b/fluent-bit/include/fluent-bit/flb_s3_local_buffer.h new file mode 100644 index 00000000..535d9c0a --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_s3_local_buffer.h @@ -0,0 +1,80 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef FLB_HAVE_AWS + +#ifndef flb_local_buffer_H +#define flb_local_buffer_H + +struct flb_local_chunk { + /* identifies this chunk in the buffer dir; created with simple_hash fn */ + flb_sds_t key; + /* the original fluent tag for this data */ + flb_sds_t tag; + flb_sds_t file_path; + size_t size; + struct timespec ts; + time_t create_time; + + /* times this chunk could not be sent */ + int failures; + + struct mk_list _head; +}; + +struct flb_local_buffer { + char *dir; + struct flb_output_instance *ins; + + struct mk_list chunks; +}; + +/* + * "Initializes" the local buffer from the file system + * Reads buffer directory and finds any existing files + * This ensures the plugin will still send buffered data even if FB is restarted + */ +int flb_init_local_buffer(struct flb_local_buffer *store); + +/* + * Stores data in the local file system + * 'c' should be NULL if no local chunk suitable for this data has been created yet + */ +int flb_buffer_put(struct flb_local_buffer *store, struct flb_local_chunk *c, + const char *tag, char *data, size_t bytes); + +/* + * Returns the chunk associated with the given tag + */ +struct flb_local_chunk *flb_chunk_get(struct flb_local_buffer *store, const char *tag); + +/* + * Recursively creates directories + */ +int flb_mkdir_all(const char *dir); + +/* Removes all files associated with a chunk once it has been removed */ +int flb_remove_chunk_files(struct flb_local_chunk *c); + +void flb_chunk_destroy(struct flb_local_chunk *c); + +void flb_local_buffer_destroy_chunks(struct flb_local_buffer *store); + +#endif +#endif /* FLB_HAVE_AWS */ diff --git a/fluent-bit/include/fluent-bit/flb_scheduler.h b/fluent-bit/include/fluent-bit/flb_scheduler.h new file mode 100644 index 00000000..e730bc33 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_scheduler.h @@ -0,0 +1,152 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SCHEDULER_H +#define FLB_SCHEDULER_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_pipe.h> +#include <fluent-bit/flb_task.h> +#include <fluent-bit/flb_output.h> + +/* Sched contstants */ +#define FLB_SCHED_CAP 2000 +#define FLB_SCHED_BASE 5 +#define FLB_SCHED_REQUEST_FRAME 10 + +/* Timer types */ +#define FLB_SCHED_TIMER_REQUEST 1 /* timerfd */ +#define FLB_SCHED_TIMER_FRAME 2 /* timer frame checker */ +#define FLB_SCHED_TIMER_CB_ONESHOT 3 /* one-shot callback timer */ +#define FLB_SCHED_TIMER_CB_PERM 4 /* permanent callback timer */ + +struct flb_sched; + +/* + * A sched timer struct belongs to an event triggered by the scheduler. This + * is a generic type and keeps two fields as a reference for further + * handling: + * + * - type: event type used to route to the proper handler + * - data: opaque data type used by the target handler + */ +struct flb_sched_timer { + struct mk_event event; + int active; + int type; + void *data; + struct flb_sched *sched; + + /* + * Custom timer specific data: + * + * - timer_fd = timer file descriptor + * - cb = callback to be triggerd upon expiration + */ + int timer_fd; + void (*cb)(struct flb_config *, void *); + + /* Parent context */ + struct flb_config *config; + + /* link to flb_sched->timers */ + struct mk_list _head; +}; + +/* Struct representing a FLB_SCHED_TIMER_REQUEST */ +struct flb_sched_request { + flb_pipefd_t fd; + time_t created; + time_t timeout; + void *data; + struct flb_sched_timer *timer; /* parent timer linked from */ + struct mk_list _head; /* link to flb_sched->[requests|wait] */ +}; + +/* Scheduler context */ +struct flb_sched { + + /* + * Scheduler lists: + * + * The scheduler is used to issue 'retries' of flush requests when these + * cannot be processed and the output plugins ask for a retry. + * + * If a retry have not reached a limit and is allowed, it needs to be + * queued: + * + * - For retries that will happen within the next 60 seconds, they are + * placed in the 'requests' list and a timeout is registered with + * the operating system (timerfd). + * + * - For retries that will happen 'after' 60 seconds, they are queued + * into the requests_wait' list. They stay there until their + * flush time happens within the next 60 seconds, so they are moved + * to the sched_requests list and a further timeout is created. + */ + struct mk_list requests; + struct mk_list requests_wait; + + /* Timers: list of timers for different purposes */ + struct mk_list timers; + + /* + * Timers_Drop: list of invalidated timers that needs to + * be free()d once the event loop finish the cycle. + */ + struct mk_list timers_drop; + + /* Frame timer context */ + flb_pipefd_t frame_fd; + + struct mk_event_loop *evl; + struct flb_config *config; +}; + +int flb_sched_request_create(struct flb_config *config, + void *data, int tries); +int flb_sched_request_destroy(struct flb_sched_request *req); +int flb_sched_event_handler(struct flb_config *config, struct mk_event *event); + +struct flb_sched *flb_sched_create(struct flb_config *config, + struct mk_event_loop *evl); + +int flb_sched_destroy(struct flb_sched *sched); + +struct flb_sched_timer *flb_sched_timer_create(struct flb_sched *sched); +int flb_sched_timer_destroy(struct flb_sched_timer *timer); + +int flb_sched_request_invalidate(struct flb_config *config, void *data); + +int flb_sched_timer_cb_create(struct flb_sched *sched, int type, int ms, + void (*cb)(struct flb_config *, void *), + void *data, struct flb_sched_timer **out_timer); +int flb_sched_timer_cb_disable(struct flb_sched_timer *timer); +int flb_sched_timer_cb_destroy(struct flb_sched_timer *timer); +void flb_sched_timer_invalidate(struct flb_sched_timer *timer); +int flb_sched_timer_cleanup(struct flb_sched *sched); +int flb_sched_retry_now(struct flb_config *config, + struct flb_task_retry *retry); + +/* Sched context api for multithread environment */ +void flb_sched_ctx_init(); +struct flb_sched *flb_sched_ctx_get(); +void flb_sched_ctx_set(struct flb_sched *sched); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_sds.h b/fluent-bit/include/fluent-bit/flb_sds.h new file mode 100644 index 00000000..8443c054 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_sds.h @@ -0,0 +1,113 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * The following SDS interface is a clone/strip-down version of the original + * SDS library created by Antirez at https://github.com/antirez/sds. + */ + +#ifndef FLB_SDS_H +#define FLB_SDS_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_macros.h> +#include <stdlib.h> +#include <string.h> +#include <inttypes.h> + +#define FLB_SDS_HEADER_SIZE (sizeof(uint64_t) + sizeof(uint64_t)) + +typedef char *flb_sds_t; + +#pragma pack(push, 1) +struct flb_sds { + uint64_t len; /* used */ + uint64_t alloc; /* excluding the header and null terminator */ + char buf[]; +}; +#pragma pack(pop) + +#define FLB_SDS_HEADER(s) ((struct flb_sds *) (s - FLB_SDS_HEADER_SIZE)) + +static inline size_t flb_sds_len(flb_sds_t s) +{ + return (size_t) FLB_SDS_HEADER(s)->len; +} + +static inline int flb_sds_is_empty(flb_sds_t s) +{ + if (flb_sds_len(s) == 0) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static inline void flb_sds_len_set(flb_sds_t s, size_t len) +{ + FLB_SDS_HEADER(s)->len = len; +} + +static inline size_t flb_sds_alloc(flb_sds_t s) +{ + return (size_t) FLB_SDS_HEADER(s)->alloc; +} + +static inline size_t flb_sds_avail(flb_sds_t s) +{ + struct flb_sds *h; + + h = FLB_SDS_HEADER(s); + return (size_t) (h->alloc - h->len); +} + +static inline int flb_sds_cmp(flb_sds_t s, const char *str, int len) +{ + if (flb_sds_len(s) != len) { + return -1; + } + + return strncmp(s, str, len); +} + +static inline int flb_sds_casecmp(flb_sds_t s, const char *str, int len) +{ + if (flb_sds_len(s) != len) { + return -1; + } + + return strncasecmp(s, str, len); +} + +flb_sds_t flb_sds_create(const char *str); +flb_sds_t flb_sds_create_len(const char *str, int len); +flb_sds_t flb_sds_create_size(size_t size); +int flb_sds_trim(flb_sds_t s); +flb_sds_t flb_sds_cat(flb_sds_t s, const char *str, int len); +flb_sds_t flb_sds_cat_esc(flb_sds_t s, const char *str, int len, + char *esc, size_t esc_size); +flb_sds_t flb_sds_cat_utf8(flb_sds_t *sds, const char *str, int len); +int flb_sds_cat_safe(flb_sds_t *buf, const char *str, int len); +flb_sds_t flb_sds_increase(flb_sds_t s, size_t len); +flb_sds_t flb_sds_copy(flb_sds_t s, const char *str, int len); +void flb_sds_destroy(flb_sds_t s); +flb_sds_t flb_sds_printf(flb_sds_t *sds, const char *fmt, ...) FLB_FORMAT_PRINTF(2, 3); +int flb_sds_snprintf(flb_sds_t *str, size_t size, const char *fmt, ...) FLB_FORMAT_PRINTF(3, 4); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_sds_list.h b/fluent-bit/include/fluent-bit/flb_sds_list.h new file mode 100644 index 00000000..36394ee5 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_sds_list.h @@ -0,0 +1,44 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SDS_LIST_H +#define FLB_SDS_LIST_H + +#include <fluent-bit/flb_sds.h> +#include <monkey/mk_core.h> + +struct flb_sds_list_entry { + flb_sds_t str; + struct mk_list _head; +}; + +struct flb_sds_list { + struct mk_list strs; +}; + +size_t flb_sds_list_size(struct flb_sds_list *list); +struct flb_sds_list *flb_sds_list_create(); +int flb_sds_list_destroy(struct flb_sds_list*); +int flb_sds_list_add(struct flb_sds_list*, char*, size_t); +int flb_sds_list_del(struct flb_sds_list_entry*); +int flb_sds_list_del_last_entry(struct flb_sds_list*); +int flb_sds_list_destroy_str_array(char **array); +char **flb_sds_list_create_str_array(struct flb_sds_list *list); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_signv4.h b/fluent-bit/include/fluent-bit/flb_signv4.h new file mode 100644 index 00000000..7b03953e --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_signv4.h @@ -0,0 +1,47 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_http_client.h> +#include <fluent-bit/flb_aws_credentials.h> + +#ifdef FLB_HAVE_SIGNV4 + +#ifndef FLB_SIGNV4_H +#define FLB_SIGNV4_H + +/* Request is not Amazon S3 PutObject */ +#define S3_MODE_NONE 0 +/* Set the x-amz-content-sha256 header with the sha value */ +#define S3_MODE_SIGNED_PAYLOAD 1 +/* Set the x-amz-content-sha256 header with the value UNSIGNED-PAYLOAD */ +#define S3_MODE_UNSIGNED_PAYLOAD 2 + +flb_sds_t flb_signv4_uri_normalize_path(char *uri, size_t len); + +flb_sds_t flb_signv4_do(struct flb_http_client *c, int normalize_uri, + int amz_date, + time_t t_now, + char *region, char *service, + int s3_mode, + struct mk_list *unsigned_headers, /* flb_slist */ + struct flb_aws_provider *provider); + +#endif +#endif /* FLB_HAVE_SIGNV4 */ diff --git a/fluent-bit/include/fluent-bit/flb_slist.h b/fluent-bit/include/fluent-bit/flb_slist.h new file mode 100644 index 00000000..c563a86d --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_slist.h @@ -0,0 +1,45 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SLIST_H +#define FLB_SLIST_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <monkey/mk_core.h> + +struct flb_slist_entry { + flb_sds_t str; + struct mk_list _head; +}; + +int flb_slist_create(struct mk_list *list); +int flb_slist_add(struct mk_list *head, const char *str); +int flb_slist_add_n(struct mk_list *head, const char *str, int len); +int flb_slist_add_sds(struct mk_list *head, flb_sds_t str); + +void flb_slist_destroy(struct mk_list *list); +int flb_slist_split_string(struct mk_list *list, const char *str, + int separator, int max_split); +int flb_slist_split_tokens(struct mk_list *list, const char *str, int max_split); + +void flb_slist_dump(struct mk_list *list); +struct flb_slist_entry *flb_slist_entry_get(struct mk_list *list, int n); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_snappy.h b/fluent-bit/include/fluent-bit/flb_snappy.h new file mode 100644 index 00000000..f1bba7e5 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_snappy.h @@ -0,0 +1,57 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/* Fluent Bit
+ * ==========
+ * Copyright (C) 2015-2022 The Fluent Bit Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FLB_SNAPPY_H
+#define FLB_SNAPPY_H
+
+#include <fluent-bit/flb_info.h>
+#include <cfl/cfl_list.h>
+
+#include <stdio.h>
+
+#define FLB_SNAPPY_STREAM_IDENTIFIER_STRING "sNaPpY"
+#define FLB_SNAPPY_FRAME_SIZE_LIMIT 65540
+
+#define FLB_SNAPPY_FRAME_TYPE_STREAM_IDENTIFIER 0xFF
+#define FLB_SNAPPY_FRAME_TYPE_COMPRESSED_DATA 0x00
+#define FLB_SNAPPY_FRAME_TYPE_UNCOMPRESSED_DATA 0x01
+#define FLB_SNAPPY_FRAME_TYPE_RESERVED_UNSKIPPABLE_BASE 0x02
+#define FLB_SNAPPY_FRAME_TYPE_RESERVED_UNSKIPPABLE_TOP 0x7F
+#define FLB_SNAPPY_FRAME_TYPE_RESERVED_SKIPPABLE_BASE 0x80
+#define FLB_SNAPPY_FRAME_TYPE_RESERVED_SKIPPABLE_TOP 0xFD
+#define FLB_SNAPPY_FRAME_TYPE_PADDING 0xFE
+
+struct flb_snappy_data_chunk {
+ int dynamically_allocated_buffer;
+ char *buffer;
+ size_t length;
+
+ struct cfl_list _head;
+};
+
+int flb_snappy_compress(char *in_data, size_t in_len,
+ char **out_data, size_t *out_len);
+
+int flb_snappy_uncompress(char *in_data, size_t in_len,
+ char **out_data, size_t *out_size);
+
+int flb_snappy_uncompress_framed_data(char *in_data, size_t in_len,
+ char **out_data, size_t *out_len);
+
+#endif
diff --git a/fluent-bit/include/fluent-bit/flb_socket.h b/fluent-bit/include/fluent-bit/flb_socket.h new file mode 100644 index 00000000..9f5365ad --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_socket.h @@ -0,0 +1,54 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SOCKET_H +#define FLB_SOCKET_H + +#include <fluent-bit/flb_compat.h> + +#ifdef _WIN32 +#include <event.h> + +#define flb_sockfd_t evutil_socket_t + +#define flb_socket_close(fd) evutil_closesocket(fd) +#define flb_socket_error(fd) evutil_socket_geterror(fd) + +#define FLB_EINPROGRESS(e) ((e) == WSAEWOULDBLOCK) +#define FLB_WOULDBLOCK() (WSAGetLastError() == WSAEWOULDBLOCK) + +#define FLB_INVALID_SOCKET ((flb_sockfd_t) -1) +#else +#include <sys/types.h> +#include <sys/socket.h> + +#define flb_sockfd_t int + +#define flb_socket_close(fd) close(fd) + +#define FLB_EINPROGRESS(e) ((e) == EINTR || (e) == EINPROGRESS) +#define FLB_WOULDBLOCK() (errno == EAGAIN || errno == EWOULDBLOCK) + +#define FLB_INVALID_SOCKET ((flb_sockfd_t) -1) + +int flb_socket_error(int fd); + +#endif + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_sosreport.h b/fluent-bit/include/fluent-bit/flb_sosreport.h new file mode 100644 index 00000000..34a1946c --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_sosreport.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SOSREPORT_H +#define FLB_SOSREPORT_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> + +int flb_sosreport(struct flb_config *config); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_sqldb.h b/fluent-bit/include/fluent-bit/flb_sqldb.h new file mode 100644 index 00000000..a5cd7c54 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_sqldb.h @@ -0,0 +1,45 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SQLDB_H +#define FLB_SQLDB_H + +#include <sqlite3.h> +#include <fluent-bit.h> + +struct flb_sqldb { + char *path; /* physical path of the database */ + char *desc; /* database description */ + int shared; /* is it a shared handler ? */ + int users; /* number of active users */ + void *parent; /* if shared, ref to parent */ + sqlite3 *handler; /* SQLite3 handler */ + struct mk_list _head; /* Link to config->sqldb_list */ +}; + +struct flb_sqldb *flb_sqldb_open(const char *path, const char *desc, + struct flb_config *config); +int flb_sqldb_close(struct flb_sqldb *db); + +int flb_sqldb_query(struct flb_sqldb *db, const char *sql, + int (*callback) (void *, int, char **, char **), + void *data); +int64_t flb_sqldb_last_id(struct flb_sqldb *db); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_stacktrace.h b/fluent-bit/include/fluent-bit/flb_stacktrace.h new file mode 100644 index 00000000..ea4c00a7 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_stacktrace.h @@ -0,0 +1,75 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_STACKTRACE_H +#define FLB_STACKTRACE_H + +#include <fluent-bit/flb_info.h> + +/* Libbacktrace support */ +#if defined(FLB_HAVE_LIBBACKTRACE) +#include <backtrace.h> +#include <backtrace-supported.h> + +struct flb_stacktrace { + struct backtrace_state *state; + int error; + int line; +}; + +static void flb_stacktrace_error_callback(void *data, + const char *msg, int errnum) +{ + struct flb_stacktrace *ctx = data; + fprintf(stderr, "ERROR: %s (%d)", msg, errnum); + ctx->error = 1; +} + +static int flb_stacktrace_print_callback(void *data, uintptr_t pc, + const char *filename, int lineno, + const char *function) +{ + struct flb_stacktrace *p = data; + + fprintf(stderr, "#%-2i 0x%-17lx in %s() at %s:%d\n", + p->line, + (unsigned long) pc, + function == NULL ? "???" : function, + filename == NULL ? "???" : filename + sizeof(FLB_SOURCE_DIR), + lineno); + p->line++; + return 0; +} + +static inline void flb_stacktrace_init(char *prog, struct flb_stacktrace *st) +{ + memset(st, '\0', sizeof(struct flb_stacktrace)); + st->state = backtrace_create_state(prog, + BACKTRACE_SUPPORTS_THREADS, + flb_stacktrace_error_callback, NULL); +} + +static inline void flb_stacktrace_print(struct flb_stacktrace *st) +{ + backtrace_full(st->state, 3, flb_stacktrace_print_callback, + flb_stacktrace_error_callback, st); +} + +#endif +#endif diff --git a/fluent-bit/include/fluent-bit/flb_storage.h b/fluent-bit/include/fluent-bit/flb_storage.h new file mode 100644 index 00000000..2adaaa07 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_storage.h @@ -0,0 +1,86 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_STORAGE_H +#define FLB_STORAGE_H + +#include <fluent-bit/flb_info.h> +#include <chunkio/chunkio.h> +#include <chunkio/cio_stats.h> + +/* Storage type */ +#define FLB_STORAGE_FS CIO_STORE_FS /* 0 */ +#define FLB_STORAGE_MEM CIO_STORE_MEM /* 1 */ +#define FLB_STORAGE_MEMRB 10 + +/* Storage defaults */ +#define FLB_STORAGE_BL_MEM_LIMIT "100M" +#define FLB_STORAGE_MAX_CHUNKS_UP 128 + +struct flb_storage_metrics { + int fd; + + struct cmt *cmt; + + /* cmetrics */ + struct cmt_gauge *cmt_chunks; /* total number of chunks */ + struct cmt_gauge *cmt_mem_chunks; /* number of chunks up in memory */ + struct cmt_gauge *cmt_fs_chunks; /* total number of filesystem chunks */ + struct cmt_gauge *cmt_fs_chunks_up; /* number of filesystem chunks up in memory */ + struct cmt_gauge *cmt_fs_chunks_down; /* number of filesystem chunks down */ +}; + +/* + * The storage structure helps to associate the contexts between + * input instances and the chunkio context and further streams. + * + * Each input instance have a stream associated. + */ +struct flb_storage_input { + int type; /* CIO_STORE_FS | CIO_STORE_MEM */ + struct cio_stream *stream; + struct cio_ctx *cio; +}; + +static inline char *flb_storage_get_type(int type) +{ + switch(type) { + case FLB_STORAGE_FS: + return "'filesystem' (memory + filesystem)"; + case FLB_STORAGE_MEM: + return "'memory' (memory only)"; + case FLB_STORAGE_MEMRB: + return "'memrb' (memory ring buffer)"; + }; + + return NULL; +} + +int flb_storage_create(struct flb_config *ctx); +int flb_storage_input_create(struct cio_ctx *cio, + struct flb_input_instance *in); +void flb_storage_destroy(struct flb_config *ctx); +void flb_storage_input_destroy(struct flb_input_instance *in); + +struct flb_storage_metrics *flb_storage_metrics_create(struct flb_config *ctx); + +/* cmetrics */ +int flb_storage_metrics_update(struct flb_config *config, struct flb_storage_metrics *sm); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_str.h b/fluent-bit/include/fluent-bit/flb_str.h new file mode 100644 index 00000000..5ddc3e50 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_str.h @@ -0,0 +1,76 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_STR_H +#define FLB_STR_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_macros.h> +#include <fluent-bit/flb_mem.h> + +#include <stdlib.h> +#include <string.h> + +static inline char *flb_strndup(const char *s, size_t n) +{ + char *str; + + str = (char *) flb_malloc(n + 1); + if (!str) { + return NULL; + } + memcpy(str, s, n); + str[n] = '\0'; + + return str; +} + +static inline char *flb_strdup(const char *s) +{ + return flb_strndup(s, strlen(s)); +} + +/* emptyval checks whether a string has a non-null value "". */ +static inline int flb_str_emptyval(const char *s) +{ + if (s != NULL && strcmp(s, "") == 0) { + return FLB_TRUE; + } + return FLB_FALSE; +} + +/* + Trim the `c` character sequence to the right of the `*str` string and return a copy. + * @param *str Source string; + * @param c Character to be trimmed. + * @returns a new string, which is a trimmed copy of `*str`. +*/ +static inline char *flb_rtrim(const char *str, char c) { + ssize_t pos = strlen(str); + + while(c == str[--pos]); + + if (pos < 0){ + return NULL; + } + + return flb_strndup(str, pos+1); +} + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_stream.h b/fluent-bit/include/fluent-bit/flb_stream.h new file mode 100644 index 00000000..95d8e79c --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_stream.h @@ -0,0 +1,208 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_STREAM_H +#define FLB_STREAM_H + +#include <fluent-bit/flb_io.h> +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_socket.h> +#include <fluent-bit/flb_config.h> + +#define FLB_DOWNSTREAM 1 +#define FLB_UPSTREAM 2 + +#define FLB_TRANSPORT_UNSET 0 +#define FLB_TRANSPORT_TCP 1 +#define FLB_TRANSPORT_UDP 2 +#define FLB_TRANSPORT_UNIX_STREAM 3 +#define FLB_TRANSPORT_UNIX_DGRAM 4 + +/* Base stream from which both downstream and upstream inherit */ +struct flb_stream { + int dynamically_allocated; + int thread_safety_flag; + int transport; + int flags; + int type; + + pthread_mutex_t list_mutex; + struct flb_tls *tls_context; + struct flb_config *config; + struct flb_net_setup net; + + struct mk_list _head; +}; + +static inline int flb_stream_is_shutting_down(struct flb_stream *stream) +{ + return stream->config->is_shutting_down; +} + +static inline void flb_stream_setup(struct flb_stream *stream, + int type, + int transport, + int flags, + struct flb_tls *tls_context, + struct flb_config *config, + struct flb_net_setup *net_setup) +{ + stream->thread_safety_flag = FLB_FALSE; + stream->tls_context = tls_context; + stream->transport = transport; + stream->config = config; + stream->flags = flags; + stream->type = type; + + /* Set default networking setup values */ + if (net_setup == NULL) { + flb_net_setup_init(&stream->net); + } + else { + memcpy(&stream->net, net_setup, sizeof(struct flb_net_setup)); + } +} + +static inline int flb_stream_get_flags(struct flb_stream *stream) +{ + return stream->flags; +} + +static inline void flb_stream_set_flags(struct flb_stream *stream, int flags) +{ + stream->flags = flags; +} + +static inline int flb_stream_get_flag_status(struct flb_stream *stream, int flag) +{ + return ((flb_stream_get_flags(stream) & flag) != 0); +} + +static inline void flb_stream_enable_flags(struct flb_stream *stream, int flag) +{ + flb_stream_set_flags(stream, flb_stream_get_flags(stream) | flag); +} + +static inline void flb_stream_disable_flags(struct flb_stream *stream, int flag) +{ + flb_stream_set_flags(stream, flb_stream_get_flags(stream) & ~flag); +} + +static inline void flb_stream_enable_async_mode(struct flb_stream *stream) +{ + flb_stream_enable_flags(stream, FLB_IO_ASYNC); +} + +static inline void flb_stream_disable_async_mode(struct flb_stream *stream) +{ + flb_stream_disable_flags(stream, FLB_IO_ASYNC); +} + +static inline int flb_stream_is_async(struct flb_stream *stream) +{ + return flb_stream_get_flag_status(stream, FLB_IO_ASYNC); +} + +static inline void flb_stream_enable_keepalive(struct flb_stream *stream) +{ + flb_stream_enable_flags(stream, FLB_IO_TCP_KA); +} + +static inline void flb_stream_disable_keepalive(struct flb_stream *stream) +{ + flb_stream_disable_flags(stream, FLB_IO_TCP_KA); +} + +static inline int flb_stream_is_keepalive(struct flb_stream *stream) +{ + return flb_stream_get_flag_status(stream, FLB_IO_TCP_KA); +} + +static inline int flb_stream_is_secure(struct flb_stream *stream) +{ + return flb_stream_get_flag_status(stream, FLB_IO_TLS); +} + +static inline int flb_stream_is_thread_safe(struct flb_stream *stream) +{ + return stream->thread_safety_flag; +} + +static inline void flb_stream_enable_thread_safety(struct flb_stream *stream) +{ + stream->thread_safety_flag = FLB_TRUE; + + pthread_mutex_init(&stream->list_mutex, NULL); + + /* We have to avoid any access to ensure that this stream context + * is only this context outside of the worker + * thread. + */ + if (mk_list_entry_orphan(&stream->_head) == 0) { + mk_list_del(&stream->_head); + } +} + +static inline int flb_stream_acquire_lock(struct flb_stream *stream, + int wait_flag) +{ + int result; + + result = 0; + + if (stream->thread_safety_flag) { + if (wait_flag) { + result = pthread_mutex_lock(&stream->list_mutex); + } + else { + result = pthread_mutex_trylock(&stream->list_mutex); + } + } + + if (result == 0) { + result = FLB_TRUE; + } + else { + result = FLB_FALSE; + } + + return result; +} + +static inline int flb_stream_release_lock(struct flb_stream *stream) +{ + int result; + + result = 0; + + if (stream->thread_safety_flag) { + result = pthread_mutex_unlock(&stream->list_mutex); + } + + if (result == 0) { + result = FLB_TRUE; + } + else { + result = FLB_FALSE; + } + + return result; +} + +#endif
\ No newline at end of file diff --git a/fluent-bit/include/fluent-bit/flb_strptime.h b/fluent-bit/include/fluent-bit/flb_strptime.h new file mode 100644 index 00000000..6212c31d --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_strptime.h @@ -0,0 +1,25 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_STRPTIME_H +#define FLB_STRPTIME_H + +char *flb_strptime(const char *s, const char *format, struct flb_tm *tm); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_task.h b/fluent-bit/include/fluent-bit/flb_task.h new file mode 100644 index 00000000..1e8de515 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_task.h @@ -0,0 +1,279 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_TASK_H +#define FLB_TASK_H + +#include <monkey/mk_core.h> +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_input.h> + +/* Task status */ +#define FLB_TASK_NEW 0 +#define FLB_TASK_RUNNING 1 + +/* + * Macro helpers to determinate return value, task_id and coro_id. When an + * output plugin returns, it must call FLB_OUTPUT_RETURN(val) where val is + * the return value, as of now defined as FLB_OK, FLB_ERROR or FLB_RETRY. + * + * The FLB_OUTPUT_RETURN macro lookup the current active 'engine coroutine' and + * it 'engine task' associated, so it emits an event to the main event loop + * indicating an output coroutine has finished. In order to specify return + * values and the proper IDs an unsigned 32 bits number is used: + * + * AAAA BBBBBBBBBBBBBB CCCCCCCCCCCCCC > 32 bit number + * ^ ^ ^ + * 4 bits 14 bits 14 bits + * return val task_id output_id + */ + +#define FLB_TASK_RET(val) (val >> 28) +#define FLB_TASK_ID(val) (uint32_t) (val & 0xfffc000) >> 14 +#define FLB_TASK_OUT(val) (val & 0x3fff) +#define FLB_TASK_SET(ret, task_id, out_id) \ + (uint32_t) ((ret << 28) | (task_id << 14) | out_id) + +/* Route status */ +#define FLB_TASK_ROUTE_INACTIVE 0 +#define FLB_TASK_ROUTE_ACTIVE 1 +#define FLB_TASK_ROUTE_DROPPED 2 + +struct flb_task_route { + int status; + struct flb_output_instance *out; + struct mk_list _head; +}; + +/* + * When a Task failed in an output instance plugin and this last one + * requested a FLB_RETRY, a flb_engine_task_retry entry is created and + * linked into the parent flb_engine_task->retries lists. + * + * This reference is used later by the scheduler to re-dispatch the + * task data to the desired output path. + */ +struct flb_task_retry { + int attempts; /* number of attempts, default 1 */ + struct flb_output_instance *o_ins; /* route that we are retrying */ + struct flb_task *parent; /* parent task reference */ + struct mk_list _head; /* link to parent task list */ +}; + +/* A task takes a buffer and sync input and output instances to handle it */ +struct flb_task { + int id; /* task id */ + uint64_t ref_id; /* external reference id */ + uint8_t status; /* new task or running ? */ + int users; /* number of users (threads) */ + struct flb_event_chunk *event_chunk; /* event chunk context */ + void *ic; /* input chunk context */ +#ifdef FLB_HAVE_METRICS + int records; /* numbers of records in 'buf' */ +#endif + struct mk_list routes; /* routes to dispatch data */ + struct mk_list retries; /* queued in-memory retries */ + struct mk_list _head; /* link to input_instance */ + struct flb_input_instance *i_ins; /* input instance */ + struct flb_config *config; /* parent flb config */ + pthread_mutex_t lock; +}; + +/* + * A queue of flb_task_enqueued tasks + * + * This structure is currently used to track pending flushes when FLB_OUTPUT_SYNCHRONOUS + * is used. + */ +struct flb_task_queue { + struct mk_list pending; + struct mk_list in_progress; +}; + +/* + * An enqueued task is a task that is not yet dispatched to a thread + * or started on the engine. + * + * There may be multiple enqueued instances of the same task on different out instances. + * + * This structure is currently used to track pending flushes when FLB_OUTPUT_SYNCHRONOUS + * is used. + */ +struct flb_task_enqueued { + struct flb_task *task; + struct flb_task_retry *retry; + struct flb_output_instance *out_instance; + struct flb_config *config; + struct mk_list _head; +}; + +int flb_task_running_count(struct flb_config *config); +int flb_task_running_print(struct flb_config *config); + +struct flb_task *flb_task_create(uint64_t ref_id, + const char *buf, + size_t size, + struct flb_input_instance *i_ins, + struct flb_input_chunk *ic, + const char *tag_buf, int tag_len, + struct flb_config *config, + int *err); + +void flb_task_add_coro(struct flb_task *task, struct flb_coro *coro); + +void flb_task_destroy(struct flb_task *task, int del); + +struct flb_task_queue* flb_task_queue_create(); +void flb_task_queue_destroy(struct flb_task_queue *queue); +struct flb_task_retry *flb_task_retry_create(struct flb_task *task, + struct flb_output_instance *ins); + +void flb_task_retry_destroy(struct flb_task_retry *retry); +int flb_task_retry_reschedule(struct flb_task_retry *retry, struct flb_config *config); +int flb_task_from_fs_storage(struct flb_task *task); +int flb_task_retry_count(struct flb_task *task, void *data); +int flb_task_retry_clean(struct flb_task *task, struct flb_output_instance *ins); + + +struct flb_task *flb_task_chunk_create(uint64_t ref_id, + const char *buf, + size_t size, + struct flb_input_instance *i_ins, + void *ic, + const char *tag_buf, int tag_len, + struct flb_config *config); + +static inline void flb_task_users_release(struct flb_task *task) +{ + if (task->users == 0 && mk_list_size(&task->retries) == 0) { + flb_task_destroy(task, FLB_TRUE); + } +} + +/* Increase the counter for users */ +static inline void flb_task_users_inc(struct flb_task *task) +{ + task->users++; +} + +/* + * Decrement the users counter from the task, and if release_check is enabled, + * it will check if the task can be destroyed. + */ +static inline void flb_task_users_dec(struct flb_task *task, int release_check) +{ + task->users--; + if (release_check == FLB_TRUE) { + flb_task_users_release(task); + } +} + +static inline void flb_task_acquire_lock(struct flb_task *task) +{ + pthread_mutex_lock(&task->lock); +} + +static inline void flb_task_release_lock(struct flb_task *task) +{ + pthread_mutex_unlock(&task->lock); +} + +static FLB_INLINE int flb_task_get_active_route_count( + struct flb_task *task) +{ + struct mk_list *iterator; + int result; + struct flb_task_route *route; + + result = 0; + + mk_list_foreach(iterator, &task->routes) { + route = mk_list_entry(iterator, struct flb_task_route, _head); + + if (route->status == FLB_TASK_ROUTE_ACTIVE) { + result++; + } + } + + return result; +} + +static FLB_INLINE size_t flb_task_get_route_status( + struct flb_task *task, + struct flb_output_instance *o_ins) +{ + struct mk_list *iterator; + size_t result; + struct flb_task_route *route; + + result = FLB_TASK_ROUTE_INACTIVE; + + mk_list_foreach(iterator, &task->routes) { + route = mk_list_entry(iterator, struct flb_task_route, _head); + + if (route->out == o_ins) { + result = route->status; + break; + } + } + + return result; +} + +static FLB_INLINE void flb_task_set_route_status( + struct flb_task *task, + struct flb_output_instance *o_ins, + int new_status) +{ + struct mk_list *iterator; + struct flb_task_route *route; + + mk_list_foreach(iterator, &task->routes) { + route = mk_list_entry(iterator, struct flb_task_route, _head); + + if (route->out == o_ins) { + route->status = new_status; + break; + } + } +} + + +static FLB_INLINE void flb_task_activate_route( + struct flb_task *task, + struct flb_output_instance *o_ins) +{ + flb_task_set_route_status(task, o_ins, FLB_TASK_ROUTE_ACTIVE); +} + +static FLB_INLINE void flb_task_deactivate_route( + struct flb_task *task, + struct flb_output_instance *o_ins) +{ + flb_task_set_route_status(task, o_ins, FLB_TASK_ROUTE_INACTIVE); +} + +static FLB_INLINE void flb_task_drop_route( + struct flb_task *task, + struct flb_output_instance *o_ins) +{ + flb_task_set_route_status(task, o_ins, FLB_TASK_ROUTE_DROPPED); +} + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_task_map.h b/fluent-bit/include/fluent-bit/flb_task_map.h new file mode 100644 index 00000000..949b3274 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_task_map.h @@ -0,0 +1,29 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_TASK_MAP_H +#define FLB_TASK_MAP_H + +#include <inttypes.h> + +struct flb_task_map { + void *task; +}; + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_thread_pool.h b/fluent-bit/include/fluent-bit/flb_thread_pool.h new file mode 100644 index 00000000..0ae9df5b --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_thread_pool.h @@ -0,0 +1,71 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_THREAD_POOL_H +#define FLB_THREAD_POOL_H + +/* Thread status */ +#define FLB_THREAD_POOL_ERROR -1 +#define FLB_THREAD_POOL_NONE 0 +#define FLB_THREAD_POOL_RUNNING 1 +#define FLB_THREAD_POOL_STOPPED 2 + +#include <fluent-bit/flb_info.h> +#ifdef FLB_SYSTEM_WINDOWS +#include <monkey/mk_core/external/winpthreads.h> +#else +#include <pthread.h> +#endif + +struct worker_params { + void (*func) (void *); + void *data; +}; + +struct flb_tp_thread { + int id; /* thread id inside the pool */ + int status; + pthread_t tid; /* OS task id */ + struct worker_params params; /* worker params before initialization */ + struct flb_worker *worker; /* worker context */ + struct mk_list _head; /* link to flb_tp->list_threads */ + struct flb_config *config; +}; + +struct flb_tp { + struct mk_list list_threads; + struct mk_list *thread_cur; + struct flb_config *config; +}; + +struct flb_tp *flb_tp_create(struct flb_config *config); +void flb_tp_destroy(struct flb_tp *tp); + +struct flb_tp_thread *flb_tp_thread_create(struct flb_tp *tp, + void (*func)(void *), void *arg, + struct flb_config *config); +struct flb_tp_thread *flb_tp_thread_get_rr(struct flb_tp *tp); +int flb_tp_thread_start(struct flb_tp *tp, struct flb_tp_thread *th); +int flb_tp_thread_start_id(struct flb_tp *tp, int id); +int flb_tp_thread_start_all(struct flb_tp *tp); +int flb_tp_thread_stop(struct flb_tp *tp, struct flb_tp_thread *th); +int flb_tp_thread_stop_all(struct flb_tp *tp); +int flb_tp_thread_destroy(); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_thread_storage.h b/fluent-bit/include/fluent-bit/flb_thread_storage.h new file mode 100644 index 00000000..8ebb436f --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_thread_storage.h @@ -0,0 +1,52 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_THREAD_STORAGE_H +#define FLB_THREAD_STORAGE_H + +#include <fluent-bit/flb_info.h> + +#ifdef FLB_SYSTEM_WINDOWS +#include <monkey/mk_core/external/winpthreads.h> +#else +#include <pthread.h> +#endif + +/* Ideal case when the compiler support direct storage through __thread */ +#ifdef FLB_HAVE_C_TLS +#define FLB_TLS_SET(key, val) key=val +#define FLB_TLS_GET(key) key +#define FLB_TLS_INIT(key) do {} while (0) +#define FLB_TLS_DEFINE(type, name) __thread type *name; + +#else + +/* Fallback mode using pthread_*() for Thread-Local-Storage usage */ +#define FLB_TLS_SET(key, val) pthread_setspecific(key, (void *) val) +#define FLB_TLS_GET(key) pthread_getspecific(key) +#define FLB_TLS_INIT(key) pthread_key_create(&key, NULL) +#define FLB_TLS_DEFINE(type, name) pthread_key_t name; +#endif + + +/* FIXME: this extern should be auto-populated from flb_thread_storage.h */ +extern FLB_TLS_DEFINE(struct flb_worker, flb_worker_ctx) + + +#endif /* !FLB_THREAD_STORAGE_H */ diff --git a/fluent-bit/include/fluent-bit/flb_time.h b/fluent-bit/include/fluent-bit/flb_time.h new file mode 100644 index 00000000..da18f227 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_time.h @@ -0,0 +1,111 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_TIME_H +#define FLB_TIME_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_time_utils.h> + +#include <time.h> +#include <msgpack.h> +#include <mpack/mpack.h> +struct flb_time { + struct timespec tm; +}; + +struct flb_tm { + struct tm tm; +#ifndef FLB_HAVE_GMTOFF + long int tm_gmtoff; +#endif +}; + +#ifndef FLB_HAVE_GMTOFF +#define flb_tm_gmtoff(x) (x)->tm_gmtoff +#else +#define flb_tm_gmtoff(x) (x)->tm.tm_gmtoff +#endif + +/* + to represent eventtime of fluentd + see also + https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v0#eventtime-ext-format + https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + */ +enum flb_time_eventtime_fmt { + FLB_TIME_ETFMT_INT = 1, /* second(integer) only */ + FLB_TIME_ETFMT_V0, /* EventTime (v0) */ + FLB_TIME_ETFMT_V1_EXT, /* EventTime (v1 ext) */ + FLB_TIME_ETFMT_V1_FIXEXT, /* EventTime (v1 fixext) */ + FLB_TIME_ETFMT_OTHER, +}; + +static inline void flb_time_zero(struct flb_time *tm) +{ + tm->tm.tv_sec = 0; + tm->tm.tv_nsec = 0; +} + +static inline void flb_time_set(struct flb_time *tm, time_t sec, long nsec) +{ + tm->tm.tv_sec = sec; + tm->tm.tv_nsec = nsec; +} + +static inline void flb_time_copy(struct flb_time *dst, struct flb_time *src) +{ + dst->tm.tv_sec = src->tm.tv_sec; + dst->tm.tv_nsec = src->tm.tv_nsec; +} + +static inline void flb_time_from_uint64(struct flb_time *dst, uint64_t value) +{ + dst->tm.tv_sec = (long) (value / 1000000000L); + dst->tm.tv_nsec = (long) (value - dst->tm.tv_sec); +} + +static inline void flb_time_from_double(struct flb_time *dst, double d) +{ + dst->tm.tv_sec = (int) d; + dst->tm.tv_nsec = (long) ((d - dst->tm.tv_sec) * 1000000000L); +} + +static inline int flb_time_equal(struct flb_time *t0, struct flb_time *t1) { + return t0->tm.tv_sec == t1->tm.tv_sec && t0->tm.tv_nsec == t1->tm.tv_nsec; +} + +int flb_time_get(struct flb_time *tm); +int flb_time_msleep(uint32_t ms); +double flb_time_to_double(struct flb_time *tm); +uint64_t flb_time_to_nanosec(struct flb_time *tm); +uint64_t flb_time_to_millisec(struct flb_time *tm); +int flb_time_add(struct flb_time *base, struct flb_time *duration, + struct flb_time *result); +int flb_time_diff(struct flb_time *time1, + struct flb_time *time0, struct flb_time *result); +int flb_time_append_to_mpack(mpack_writer_t *writer, struct flb_time *tm, int fmt); +int flb_time_append_to_msgpack(struct flb_time *tm, msgpack_packer *pk, int fmt); +int flb_time_msgpack_to_time(struct flb_time *time, msgpack_object *obj); +int flb_time_pop_from_mpack(struct flb_time *time, mpack_reader_t *reader); +int flb_time_pop_from_msgpack(struct flb_time *time, msgpack_unpacked *upk, + msgpack_object **map); +long flb_time_tz_offset_to_second(); + +#endif /* FLB_TIME_H */ diff --git a/fluent-bit/include/fluent-bit/flb_time_utils.h b/fluent-bit/include/fluent-bit/flb_time_utils.h new file mode 100644 index 00000000..cff27c16 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_time_utils.h @@ -0,0 +1,71 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_TIME_UTILS_H +#define FLB_TIME_UTILS_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_coro.h> +#include <fluent-bit/flb_scheduler.h> + +static void flb_time_thread_wakeup(struct flb_config *config, void *data) +{ + (void) config; + struct flb_coro *th; + + th = (struct flb_coro *) data; + flb_coro_resume(th); +} + +/* + * Sleep running thread for 'ms' (milliseconds). This function assume + * that's running in a co-routine. + * + * Internally it creates a timer and once the signal gets into the + * event loop after expiration time, this function resume. + * + * A context that invokes flb_time_sleep() will resume upon an + * internal call to flb_time_thread_wakeup(). + */ +static FLB_INLINE void flb_time_sleep(int ms) +{ + int ret; + struct flb_coro *coro; + struct flb_sched *sched; + + coro = flb_coro_get(); + if (!coro) { + flb_error("[thread] invalid context for thread_sleep()"); + return; + } + + /* Get the scheduler context */ + sched = flb_sched_ctx_get(); + assert(sched != NULL); + + ret = flb_sched_timer_cb_create(sched, FLB_SCHED_TIMER_CB_ONESHOT, + ms, flb_time_thread_wakeup, coro, NULL); + if (ret == -1) { + return; + } + + flb_coro_yield(coro, FLB_FALSE); +} + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_typecast.h b/fluent-bit/include/fluent-bit/flb_typecast.h new file mode 100644 index 00000000..9ae66ced --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_typecast.h @@ -0,0 +1,66 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_TYPECAST_H +#define FLB_TYPECAST_H + +#include <fluent-bit/flb_typecast.h> +#include <fluent-bit/flb_sds.h> +#include <inttypes.h> +#include <msgpack.h> + +typedef enum { + FLB_TYPECAST_TYPE_NUMBER = 0, /* from_type only */ + FLB_TYPECAST_TYPE_INT, + FLB_TYPECAST_TYPE_UINT, + FLB_TYPECAST_TYPE_FLOAT, + FLB_TYPECAST_TYPE_BOOL, + FLB_TYPECAST_TYPE_STR, + FLB_TYPECAST_TYPE_HEX, + FLB_TYPECAST_TYPE_ERROR, +} flb_typecast_type_t; + +struct flb_typecast_rule { + flb_typecast_type_t from_type; + flb_typecast_type_t to_type; +}; + +struct flb_typecast_value { + flb_typecast_type_t type; + union { + char boolean; /* bool */ + int64_t i_num; /* int */ + uint64_t ui_num; /* uint, hex */ + double d_num; /* float */ + flb_sds_t str; /* string */ + } val; +}; + +flb_typecast_type_t flb_typecast_str_to_type_t(char *type_str, int type_len); +const char * flb_typecast_type_t_to_str(flb_typecast_type_t type); +int flb_typecast_rule_destroy(struct flb_typecast_rule *rule); +struct flb_typecast_rule *flb_typecast_rule_create(char *from_type, int from_len, + char *to_type, int to_len); +int flb_typecast_value_destroy(struct flb_typecast_value* val); +struct flb_typecast_value *flb_typecast_value_create(msgpack_object input, + struct flb_typecast_rule *rule); +int flb_typecast_pack(msgpack_object input, + struct flb_typecast_rule *rule, + msgpack_packer *pck); +#endif diff --git a/fluent-bit/include/fluent-bit/flb_unescape.h b/fluent-bit/include/fluent-bit/flb_unescape.h new file mode 100644 index 00000000..f1ab6fe6 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_unescape.h @@ -0,0 +1,27 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_UNESCAPE_H +#define FLB_UNESCAPE_H + +int flb_unescape_string(const char *buf, int buf_len, char **unesc_buf); +int flb_unescape_string_utf8(const char *in_buf, int sz, char *out_buf); +int flb_mysql_unquote_string(char *buf, int buf_len, char **unesc_buf); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_upstream.h b/fluent-bit/include/fluent-bit/flb_upstream.h new file mode 100644 index 00000000..ad23238e --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_upstream.h @@ -0,0 +1,123 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_UPSTREAM_H +#define FLB_UPSTREAM_H + +#include <monkey/mk_core.h> + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_socket.h> +#include <fluent-bit/flb_network.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_io.h> +#include <fluent-bit/flb_upstream_queue.h> +#include <fluent-bit/flb_stream.h> + +#include <cmetrics/cmetrics.h> +#include <cmetrics/cmt_gauge.h> + +/* + * Upstream creation FLAGS set by Fluent Bit sub-components + * ======================================================== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * --- flb_io.h --- + * #define FLB_IO_TCP 1 + * #define FLB_IO_TLS 2 + * #define FLB_IO_ASYNC 8 + * #define FLB_IO_TCP_KA 16 + * --- + */ + +/* Upstream handler */ +struct flb_upstream { + struct flb_stream base; + + char *tcp_host; + int tcp_port; + + char *proxied_host; + int proxied_port; + char *proxy_username; + char *proxy_password; + + /* + * If an upstream context has been created in HA mode, this flag is + * set to True and the field 'ha_ctx' will reference a HA upstream + * context. + */ + int ha_mode; + void *ha_ctx; + + struct cmt_gauge *cmt_total_connections; + struct cmt_gauge *cmt_busy_connections; + const char *cmt_total_connections_label; + const char *cmt_busy_connections_label; + + /* + * If the connections will be in separate threads, this flag is + * enabled and all lists management are protected through mutexes. + */ + void *parent_upstream; + struct flb_upstream_queue queue; +}; + +static inline int flb_upstream_is_shutting_down(struct flb_upstream *u) +{ + return flb_stream_is_shutting_down(&u->base); +} + +void flb_upstream_queue_init(struct flb_upstream_queue *uq); +struct flb_upstream_queue *flb_upstream_queue_get(struct flb_upstream *u); +void flb_upstream_list_set(struct mk_list *list); +struct mk_list *flb_upstream_list_get(); + +void flb_upstream_init(); +struct flb_upstream *flb_upstream_create(struct flb_config *config, + const char *host, int port, int flags, + struct flb_tls *tls); +struct flb_upstream *flb_upstream_create_url(struct flb_config *config, + const char *url, int flags, + struct flb_tls *tls); + +int flb_upstream_destroy(struct flb_upstream *u); + +int flb_upstream_set_property(struct flb_config *config, + struct flb_net_setup *net, char *k, char *v); +int flb_upstream_is_async(struct flb_upstream *u); +void flb_upstream_thread_safe(struct flb_upstream *u); +struct mk_list *flb_upstream_get_config_map(struct flb_config *config); +int flb_upstream_needs_proxy(const char *host, const char *proxy, const char *no_proxy); + +void flb_upstream_set_total_connections_label( + struct flb_upstream *stream, + const char *label_value); +void flb_upstream_set_total_connections_gauge( + struct flb_upstream *stream, + struct cmt_gauge *gauge_instance); + +void flb_upstream_set_busy_connections_label( + struct flb_upstream *stream, + const char *label_value); +void flb_upstream_set_busy_connections_gauge( + struct flb_upstream *stream, + struct cmt_gauge *gauge_instance); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_upstream_conn.h b/fluent-bit/include/fluent-bit/flb_upstream_conn.h new file mode 100644 index 00000000..ac3063bf --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_upstream_conn.h @@ -0,0 +1,36 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_UPSTREAM_CONN_H +#define FLB_UPSTREAM_CONN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_socket.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_connection.h> + +int flb_upstream_conn_recycle(struct flb_connection *conn, int val); +struct flb_connection *flb_upstream_conn_get(struct flb_upstream *u); +int flb_upstream_conn_release(struct flb_connection *u_conn); +int flb_upstream_conn_timeouts(struct mk_list *list); +int flb_upstream_conn_pending_destroy(struct flb_upstream *u); +int flb_upstream_conn_pending_destroy_list(struct mk_list *list); +int flb_upstream_conn_active_destroy_list(struct mk_list *list); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_upstream_ha.h b/fluent-bit/include/fluent-bit/flb_upstream_ha.h new file mode 100644 index 00000000..958b3bd9 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_upstream_ha.h @@ -0,0 +1,42 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_UPSTREAM_HA_H +#define FLB_UPSTREAM_HA_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_upstream_node.h> +#include <monkey/mk_core.h> + +struct flb_upstream_ha { + flb_sds_t name; /* Upstream HA name */ + void *last_used_node; /* Last used node */ + struct mk_list nodes; /* List of available nodes */ +}; + +struct flb_upstream_ha *flb_upstream_ha_create(const char *name); +void flb_upstream_ha_destroy(struct flb_upstream_ha *ctx); +void flb_upstream_ha_node_add(struct flb_upstream_ha *ctx, + struct flb_upstream_node *node); +struct flb_upstream_node *flb_upstream_ha_node_get(struct flb_upstream_ha *ctx); +struct flb_upstream_ha *flb_upstream_ha_from_file(const char *file, + struct flb_config *config); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_upstream_node.h b/fluent-bit/include/fluent-bit/flb_upstream_node.h new file mode 100644 index 00000000..6e798371 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_upstream_node.h @@ -0,0 +1,93 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_UPSTREAM_NODE_H +#define FLB_UPSTREAM_NODE_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_hash_table.h> +#include <fluent-bit/flb_upstream.h> +#include <monkey/mk_core.h> + +struct flb_upstream_node { + flb_sds_t name; + flb_sds_t host; + flb_sds_t port; + + int tls_enabled; /* TLS enabled ? */ + +#ifdef FLB_HAVE_TLS + /* TLS: given configuration */ + int tls_verify; /* Verify certs (default: true) */ + int tls_debug; /* mbedtls debug level */ + char *tls_ca_path; /* Path to certificates */ + char *tls_ca_file; /* CA root cert */ + char *tls_crt_file; /* Certificate */ + char *tls_key_file; /* Cert Key */ + char *tls_key_passwd; /* Cert Key Password */ + + /* context with mbedTLS contexts and data */ + struct flb_tls *tls; +#else + void *tls; +#endif + + /* Hash table to store custom config key/values for plugins */ + struct flb_hash_table *ht; + + /* Upstream context */ + struct flb_upstream *u; + + void *data; + + /* Link to upstream_ha or upstream */ + struct mk_list _head; +}; + + +struct flb_upstream_node *flb_upstream_node_create(flb_sds_t name, flb_sds_t host, + flb_sds_t port, + int tls, int tls_verify, + int tls_debug, + const char *tls_vhost, + const char *tls_ca_path, + const char *tls_ca_file, + const char *tls_crt_file, + const char *tls_key_file, + const char *tls_key_passwd, + struct flb_hash_table *ht, + struct flb_config *config); +const char *flb_upstream_node_get_property(const char *prop, + struct flb_upstream_node *node); + +static inline void flb_upstream_node_set_data(void *data, + struct flb_upstream_node *node) +{ + node->data = data; +} + +static inline void *flb_upstream_node_get_data(struct flb_upstream_node *node) +{ + return node->data; +} + +void flb_upstream_node_destroy(struct flb_upstream_node *node); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_upstream_queue.h b/fluent-bit/include/fluent-bit/flb_upstream_queue.h new file mode 100644 index 00000000..996a378a --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_upstream_queue.h @@ -0,0 +1,47 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_UPSTREAM_QUEUE_H +#define FLB_UPSTREAM_QUEUE_H + +#include <fluent-bit/flb_info.h> + +struct flb_upstream_queue { + /* + * This field is a linked-list-head for upstream connections that + * are available for usage. When a connection is taken, it's moved to the + * 'busy_queue' list. + */ + struct mk_list av_queue; + + /* + * Linked list head for upstream connections that are in use by some + * plugin. When released, they are moved to the 'av_queue' list. + */ + struct mk_list busy_queue; + + /* + * Ready to destroy connections are linked here, mostly because they are + * not yet destroyed but will be destroyed in a later step of the event loop + * to avoid any race condition with a late event. + */ + struct mk_list destroy_queue; +}; + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_uri.h b/fluent-bit/include/fluent-bit/flb_uri.h new file mode 100644 index 00000000..2e6a1412 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_uri.h @@ -0,0 +1,62 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_URI_H +#define FLB_URI_H + +#include <monkey/mk_core.h> +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> + +/* By default we allow a maximum of 8 URI patterns in our map */ +#define FLB_URI_MAX 8 + +struct flb_uri_field { + size_t length; + char *value; + struct mk_list _head; +}; + +struct flb_uri { + char *full; /* Original full URI */ + uint8_t count; /* Number of entries in the list */ + struct mk_list list; /* List head for uri patterns */ + struct flb_uri_field *map; /* Map / O(1) lookup by position */ +}; + +static inline int flb_uri_to_encode(char c) +{ + if ((c >= 48 && c <= 57) || /* 0-9 */ + (c >= 65 && c <= 90) || /* A-Z */ + (c >= 97 && c <= 122) || /* a-z */ + (c == '?' || c == '&' || c == '-' || c == '_' || c == '.' || + c == '~' || c == '/' || c == '=')) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +struct flb_uri_field *flb_uri_get(struct flb_uri *uri, int pos); +struct flb_uri *flb_uri_create(const char *full_uri); +void flb_uri_destroy(struct flb_uri *uri); +void flb_uri_dump(struct flb_uri *uri); +flb_sds_t flb_uri_encode(const char *uri, size_t len); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_utf8.h b/fluent-bit/include/fluent-bit/flb_utf8.h new file mode 100644 index 00000000..617379d0 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_utf8.h @@ -0,0 +1,103 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_UTF8_H +#define FLB_UTF8_H + +#include <fluent-bit/flb_info.h> +#include <inttypes.h> + +/* is the start of a UTF-8 string ? */ +#define flb_utf8_check(c) (((c) & 0xC0) != 0x80) + +static const char trailingBytesForUTF8[256] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 +}; + +/* returns length of next utf-8 sequence */ +static inline int flb_utf8_len(const char *s) +{ + return trailingBytesForUTF8[(unsigned int)(unsigned char)s[0]] + 1; +} + +/* + * UTF-8 Decoding routines are originally written by Bjoern Hoehrmann + * <bjoern@hoehrmann.de> and taken from the following web site: + * + * http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + * + * They have been siglhy renamed to follow Fluent Bit naming requirements. + */ + +#define FLB_UTF8_ACCEPT 0 +#define FLB_UTF8_REJECT 1 + +static const uint8_t utf8d[] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df + 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef + 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff + 0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2 + 1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4 + 1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6 + 1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8 +}; + +static inline uint32_t flb_utf8_decode(uint32_t *state, uint32_t *codep, + uint32_t byte) +{ + uint32_t type = utf8d[byte]; + + *codep = (*state != FLB_UTF8_ACCEPT) ? + (byte & 0x3fu) | (*codep << 6) : + (0xff >> type) & (byte); + + *state = utf8d[256 + *state*16 + type]; + return *state; +} + + +static inline void flb_utf8_print(const uint8_t *s) { + uint32_t codepoint; + uint32_t state = 0; + + for (; *s; ++s) + if (!flb_utf8_decode(&state, &codepoint, *s)) { + printf("\\u%04x\n", codepoint); + } + + if (state != FLB_UTF8_ACCEPT) { + printf("The string is not well-formed\n"); + } +} + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_utils.h b/fluent-bit/include/fluent-bit/flb_utils.h new file mode 100644 index 00000000..5bc790ba --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_utils.h @@ -0,0 +1,77 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_UTILS_H +#define FLB_UTILS_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_pipe.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_sds.h> + +struct flb_split_entry { + char *value; + int len; + off_t last_pos; + struct mk_list _head; +}; + +void flb_utils_error(int err); +void flb_utils_error_c(const char *msg); +void flb_utils_warn_c(const char *msg); +void flb_message(int type, const char *file, int line, const char *fmt, ...); + +#ifdef FLB_HAVE_FORK +int flb_utils_set_daemon(struct flb_config *config); +#endif + +void flb_utils_print_setup(struct flb_config *config); + +struct mk_list *flb_utils_split(const char *line, int separator, int max_split); +struct mk_list *flb_utils_split_quoted(const char *line, int separator, int max_split); +void flb_utils_split_free_entry(struct flb_split_entry *entry); +void flb_utils_split_free(struct mk_list *list); +int flb_utils_timer_consume(flb_pipefd_t fd); +int64_t flb_utils_size_to_bytes(const char *size); +int64_t flb_utils_hex2int(char *hex, int len); +int flb_utils_time_to_seconds(const char *time); +int flb_utils_pipe_byte_consume(flb_pipefd_t fd); +int flb_utils_bool(const char *val); +void flb_utils_bytes_to_human_readable_size(size_t bytes, + char *out_buf, size_t size); +int flb_utils_time_split(const char *time, int *sec, long *nsec); +int flb_utils_write_str(char *buf, int *off, size_t size, + const char *str, size_t str_len); +int flb_utils_write_str_buf(const char *str, size_t str_len, + char **out, size_t *out_size); + +int flb_utils_url_split(const char *in_url, char **out_protocol, + char **out_host, char **out_port, char **out_uri); +int flb_utils_proxy_url_split(const char *in_url, char **out_protocol, + char **out_username, char **out_password, + char **out_host, char **out_port); +int flb_utils_read_file(char *path, char **out_buf, size_t *out_size); +char *flb_utils_get_os_name(); +int flb_utils_uuid_v4_gen(char *buf); +int flb_utils_get_machine_id(char **out_id, size_t *out_size); +void flb_utils_set_plugin_string_property(const char *name, + flb_sds_t *field_storage, + flb_sds_t new_value); + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_version.h.in b/fluent-bit/include/fluent-bit/flb_version.h.in new file mode 100644 index 00000000..b780bd8a --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_version.h.in @@ -0,0 +1,90 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_VERSION_H +#define FLB_VERSION_H + +#include <fluent-bit/flb_info.h> +#include <monkey/mk_core.h> + +#include <stdio.h> +#include <stdlib.h> + +/* Helpers to convert/format version string */ +#define STR_HELPER(s) #s +#define STR(s) STR_HELPER(s) + +/* Fluent Bit Version */ +#define FLB_VERSION_MAJOR @FLB_VERSION_MAJOR@ +#define FLB_VERSION_MINOR @FLB_VERSION_MINOR@ +#define FLB_VERSION_PATCH @FLB_VERSION_PATCH@ +#define FLB_VERSION (FLB_VERSION_MAJOR * 10000 \ + FLB_VERSION_MINOR * 100 \ + FLB_VERSION_PATCH) +#define FLB_VERSION_STR "@FLB_VERSION_STR@" +#define FLB_GIT_HASH "@FLB_GIT_HASH@" + +static inline void flb_version() +{ +#ifdef FLB_NIGHTLY_BUILD + printf("Fluent Bit v%s | NIGHTLY_BUILD=%s - DO NOT USE IN PRODUCTION!\n", + FLB_VERSION_STR, STR(FLB_NIGHTLY_BUILD)); +#else + printf("Fluent Bit v%s\n", FLB_VERSION_STR); +#endif + printf("Git commit: %s\n", FLB_GIT_HASH); + + exit(EXIT_SUCCESS); +} + +static inline void flb_version_banner() +{ + const char *copyright_color = ANSI_BOLD ANSI_YELLOW; + const char *bold_color = ANSI_BOLD; + const char *reset_color = ANSI_RESET; + + #ifdef FLB_LOG_NO_CONTROL_CHARS + copyright_color = ""; + bold_color = ""; + reset_color = ""; + #else + /* Only print colors to a terminal */ + if (!isatty(STDOUT_FILENO)) { + copyright_color = ""; + bold_color = ""; + reset_color = ""; + } + #endif // FLB_LOG_NO_CONTROL_CHARS + +#ifdef FLB_NIGHTLY_BUILD + fprintf(stderr, + "%sFluent Bit v%s | NIGHTLY_BUILD=%s - DO NOT USE IN PRODUCTION!" + "%s\n", bold_color, FLB_VERSION_STR, STR(FLB_NIGHTLY_BUILD), reset_color); +#else + fprintf(stderr, + "%sFluent Bit v%s%s\n", bold_color, FLB_VERSION_STR, reset_color); +#endif + fprintf(stderr, "* %sCopyright (C) 2015-2022 The Fluent Bit Authors%s\n", + copyright_color, reset_color); + fprintf(stderr, "* Fluent Bit is a CNCF sub-project under the " + "umbrella of Fluentd\n"); + fprintf(stderr, "* https://fluentbit.io\n\n"); +} + +#endif diff --git a/fluent-bit/include/fluent-bit/flb_worker.h b/fluent-bit/include/fluent-bit/flb_worker.h new file mode 100644 index 00000000..409604a4 --- /dev/null +++ b/fluent-bit/include/fluent-bit/flb_worker.h @@ -0,0 +1,63 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_WORKER_H +#define FLB_WORKER_H + +#include <fluent-bit/flb_config.h> + +struct flb_config; + +struct flb_worker { + struct mk_event event; + + /* Callback data */ + void (*func) (void *); /* function */ + void *data; /* opaque data */ + pthread_t tid; /* thread ID */ + + /* Logging */ + struct flb_log_cache *log_cache; +#ifdef _WIN32 + intptr_t log[2]; +#else + flb_pipefd_t log[2]; +#endif + + /* Runtime context */ + void *config; + void *log_ctx; + + pthread_mutex_t mutex; + struct mk_list _head; /* link to head at config->workers */ +}; + +int flb_worker_init(struct flb_config *config); +struct flb_worker *flb_worker_get(); + +struct flb_worker *flb_worker_context_create(void (*func) (void *), void *arg, + struct flb_config *config); + +int flb_worker_create(void (*func) (void *), void *arg, pthread_t *tid, + struct flb_config *config); +struct flb_worker *flb_worker_lookup(pthread_t tid, struct flb_config *config); +int flb_worker_exit(struct flb_config *config); +int flb_worker_log_level(struct flb_worker *worker); + +#endif diff --git a/fluent-bit/include/fluent-bit/http_server/flb_hs.h b/fluent-bit/include/fluent-bit/http_server/flb_hs.h new file mode 100644 index 00000000..5db4c2c5 --- /dev/null +++ b/fluent-bit/include/fluent-bit/http_server/flb_hs.h @@ -0,0 +1,66 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HS_MAIN_H +#define FLB_HS_MAIN_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_sds.h> +#include <monkey/mk_lib.h> + +/* + * HTTP buffers that contains certain cached data to be used + * by end-points. + */ +struct flb_hs_buf { + int users; + flb_sds_t data; + void *raw_data; + size_t raw_size; + struct mk_list _head; +}; + +struct flb_hs { + mk_ctx_t *ctx; /* Monkey HTTP Context */ + int vid; /* Virtual Host ID */ + int qid_metrics; /* Metrics Message Queue ID */ + int qid_metrics_v2; /* Metrics Message Queue ID for /api/v2 */ + int qid_storage; /* Storage Message Queue ID */ + int qid_health; /* health Message Queue ID */ + + pthread_t tid; /* Server Thread */ + struct flb_config *config; /* Fluent Bit context */ + + /* end-point: root */ + size_t ep_root_size; + char *ep_root_buf; +}; + +struct flb_hs *flb_hs_create(const char *listen, const char *tcp_port, + struct flb_config *config); +int flb_hs_push_health_metrics(struct flb_hs *hs, void *data, size_t size); +int flb_hs_push_pipeline_metrics(struct flb_hs *hs, void *data, size_t size); +int flb_hs_push_metrics(struct flb_hs *hs, void *data, size_t size); +int flb_hs_push_storage_metrics(struct flb_hs *hs, void *data, size_t size); + +int flb_hs_destroy(struct flb_hs *ctx); +int flb_hs_start(struct flb_hs *hs); + +#endif diff --git a/fluent-bit/include/fluent-bit/http_server/flb_hs_endpoints.h b/fluent-bit/include/fluent-bit/http_server/flb_hs_endpoints.h new file mode 100644 index 00000000..83b4405f --- /dev/null +++ b/fluent-bit/include/fluent-bit/http_server/flb_hs_endpoints.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HS_ENDPOINTS_H +#define FLB_HS_ENDPOINTS_H + +#include <fluent-bit/flb_info.h> + +int flb_hs_endpoints(struct flb_hs *hs); +int flb_hs_endpoints_free(struct flb_hs *hs); + +#endif diff --git a/fluent-bit/include/fluent-bit/http_server/flb_hs_utils.h b/fluent-bit/include/fluent-bit/http_server/flb_hs_utils.h new file mode 100644 index 00000000..a0e5c115 --- /dev/null +++ b/fluent-bit/include/fluent-bit/http_server/flb_hs_utils.h @@ -0,0 +1,42 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HS_UTIL_H +#define FLB_HS_UTIL_H + +#include <monkey/mk_lib.h> + +int flb_hs_add_content_type_to_req(mk_request_t *request, int type); + +/* Content-type */ +enum content_type { + FLB_HS_CONTENT_TYPE_JSON, + FLB_HS_CONTENT_TYPE_PROMETHEUS, + FLB_HS_CONTENT_TYPE_OTHER +}; + +#define FLB_HS_CONTENT_TYPE_KEY_STR "Content-Type" +#define FLB_HS_CONTENT_TYPE_KEY_LEN 12 + +#define FLB_HS_CONTENT_TYPE_JSON_STR "application/json" +#define FLB_HS_CONTENT_TYPE_JSON_LEN 16 +#define FLB_HS_CONTENT_TYPE_PROMETHEUS_STR "text/plain; version=0.0.4" +#define FLB_HS_CONTENT_TYPE_PROMETHEUS_LEN 25 + +#endif diff --git a/fluent-bit/include/fluent-bit/multiline/flb_ml.h b/fluent-bit/include/fluent-bit/multiline/flb_ml.h new file mode 100644 index 00000000..ffe97f86 --- /dev/null +++ b/fluent-bit/include/fluent-bit/multiline/flb_ml.h @@ -0,0 +1,359 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ML_H +#define FLB_ML_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_log.h> +#include <fluent-bit/flb_regex.h> +#include <fluent-bit/flb_mp.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_parser.h> +#include <fluent-bit/flb_log_event_decoder.h> +#include <fluent-bit/flb_log_event_encoder.h> + +/* Types available */ +#define FLB_ML_REGEX 1 /* pattern is a regular expression */ +#define FLB_ML_ENDSWITH 2 /* record key/content ends with 'abc' */ +#define FLB_ML_EQ 3 /* record key/content equaks 'abc' */ + +/* Rule types */ +#define FLB_ML_RULE_STATE_START 0 +#define FLB_ML_RULE_STATE_PART 1 +#define FLB_ML_RULE_STATE_END 2 + +/* Payload types */ +#define FLB_ML_TYPE_TEXT 0 /* raw text */ +#define FLB_ML_TYPE_RECORD 1 /* Fluent Bit msgpack record */ +#define FLB_ML_TYPE_MAP 2 /* msgpack object/map (k/v pairs) */ +#define FLB_ML_TYPE_EVENT 3 /* Fluent Bit decoded event */ + +#define FLB_ML_FLUSH_TIMEOUT 4000 /* Flush timeout default (milliseconds) */ + +/* Default multiline buffer size: 4Kb */ +#define FLB_ML_BUF_SIZE 1024*4 + +/* Maximum number of groups per stream */ +#define FLB_ML_MAX_GROUPS 6 + +struct flb_ml; +struct flb_ml_parser; +struct flb_ml_stream; + +struct flb_ml_rule { + /* If the rule contains a 'start_state' this flag is turned on */ + int start_state; + + /* Name of the rule (state name) */ + struct mk_list from_states; + + /* + * Defined 'to_state' in the rule. Note that this work only at definition + * since we do a mapping to multiple possible rules having the same + * name. + */ + flb_sds_t to_state; + + struct mk_list to_state_map; + + /* regex content pattern */ + struct flb_regex *regex; + + /* regex end pattern */ + struct flb_regex *regex_end; + + struct mk_list _head; +}; + +struct flb_ml_stream_group { + flb_sds_t name; /* group name, unique identifier */ + + uint64_t last_flush; /* last flush/check time */ + + int counter_lines; /* counter for the number of lines */ + + /* Multiline content buffer */ + flb_sds_t buf; + + /* internal state */ + int first_line; /* first line of multiline message ? */ + + struct flb_ml_rule *rule_to_state; + + /* packaging buffers */ + msgpack_sbuffer mp_md_sbuf; /* temporary msgpack buffer */ + msgpack_packer mp_md_pck; /* temporary msgpack packer */ + + msgpack_sbuffer mp_sbuf; /* temporary msgpack buffer */ + msgpack_packer mp_pck; /* temporary msgpack packer */ + struct flb_time mp_time; /* multiline time parsed from first line */ + + struct mk_list _head; +}; + +struct flb_ml_stream { + uint64_t id; + flb_sds_t name; /* name of the stream, mostly for debugging purposes */ + struct mk_list groups; /* groups inside a stream */ + + /* Flush callback and opaque data type */ + int (*cb_flush) (struct flb_ml_parser *, + struct flb_ml_stream *, + void *cb_data, + char *buf_data, + size_t buf_size); + void *cb_data; + + struct flb_ml_stream_group *last_stream_group; + + /* runtime flags */ + int forced_flush; + + /* parent context */ + struct flb_ml *ml; + + /* reference to parent instance */ + struct flb_ml_parser_ins *parser; + + struct mk_list _head; +}; + +/* Multiline Parser definition (no running state, just definition) */ +struct flb_ml_parser { + int type; /* type: REGEX, ENDSWITH or EQ */ + int negate; /* negate start pattern ? */ + int flush_ms; /* default flush timeout in milliseconds */ + + flb_sds_t name; + + /* + * If 'multiline type' is ENDSWITH or EQ, a 'match_str' string is passed + * so we can compare against it. We don't use a regex pattern for efficiency. + */ + flb_sds_t match_str; + + /* + * The 'key' name that contains the multiline message. For REGEX, ENDSWITH and + * EQ types, the conditions are applied to 'key_content' unless 'key_pattern' + * is set, on that case 'key_content' is used as a raw buffer and appended + * as part of the multiline message. + */ + flb_sds_t key_content; + + /* + * Optional: define a 'key' name that matches the pattern to decide if the + * line is complete or a continuation. + * + * This is not mandatory, most of the rules works directly on 'key_content' + * but other use-cases like 'CRI' uses a different 'key' to define if the + * line is complete or not. + */ + flb_sds_t key_pattern; + + /* + * Optional: define a 'key' name that specify a specific group of logs. + * As an example, consider containerized logs coming from Docker or CRI + * where the logs must be group by stream 'stdout' and 'stderr'. On that + * case key_group = 'stream'. + * + * If the origin stream will not have groups, this can be null and the + * multiline context creator will only use a default group for everything + * under the same stream. + */ + flb_sds_t key_group; + + /* internal */ + struct flb_parser *parser; /* parser context */ + flb_sds_t parser_name; /* parser name for delayed init */ + + /* + * If multiline type is REGEX, it needs a set of pre-defined rules to deal + * with messages. + */ + struct mk_list regex_rules; + + /* Fluent Bit parent context */ + struct flb_config *config; + + /* Link node to struct flb_ml_group_parser->parsers */ + struct mk_list _head; +}; + +/* Multiline parser instance with running state */ +struct flb_ml_parser_ins { + struct flb_ml_parser *ml_parser; /* multiline parser */ + + /* flush callback */ + int (*cb_flush)(struct flb_ml_parser *, /* multiline context */ + struct flb_ml_stream *, /* stream context */ + void *, /* opaque data */ + char *, /* text buffer */ + size_t); /* buffer length */ + + void *cb_data; /* opaque data */ + + + /* + * Duplicate original parser definition properties for this instance. There + * are cases where the caller wants an instance of a certain multiline + * parser but with a custom value for key_content, key_pattern or key_group. + */ + flb_sds_t key_content; + flb_sds_t key_pattern; + flb_sds_t key_group; + + /* + * last stream_id and last_stream_group: keeping a reference of the last + * insertion path is important to determinate when should we flush our + * stream/stream_group buffer. + */ + uint64_t last_stream_id; + struct flb_ml_stream_group *last_stream_group; + + /* + * Every multiline parser context has N streams, a stream represent a source + * of data. + */ + struct mk_list streams; + + /* Link to struct flb_ml_group->parsers */ + struct mk_list _head; +}; + +struct flb_ml_group { + int id; /* group id (auto assigned) */ + struct mk_list parsers; /* list head for parser instances */ + + /* keep context of the previous match */ + struct flb_ml_parser_ins *lru_parser; + + /* + * Flush callback for parsers instances on this group + * -------------------------------------------------- + */ + int flush_ms; /* flush interval */ + + /* flush callback */ + int (*cb_flush)(struct flb_ml_parser *, /* multiline context */ + struct flb_ml_stream *, /* stream context */ + void *, /* opaque data */ + char *, /* text buffer */ + size_t); /* buffer length */ + + void *cb_data; /* opaque data */ + + /* Parent multiline context */ + struct flb_ml *ml; + + struct mk_list _head; /* link to struct flb_ml->groups list */ +}; + +struct flb_ml { + flb_sds_t name; /* name of this multiline setup */ + int flush_ms; /* max flush interval found in groups/parsers */ + uint64_t last_flush; /* last flush time (involving groups) */ + struct mk_list groups; /* list head for flb_ml_group(s) */ + struct flb_log_event_encoder log_event_encoder; + struct flb_log_event_decoder log_event_decoder; + struct flb_config *config; /* Fluent Bit context */ +}; + +struct flb_ml *flb_ml_create(struct flb_config *ctx, char *name); +int flb_ml_destroy(struct flb_ml *ml); + +int flb_ml_register_context(struct flb_ml_stream_group *group, + struct flb_time *tm, msgpack_object *map); + +int flb_ml_append_text(struct flb_ml *ml, + uint64_t stream_id, + struct flb_time *tm, + void *buf, + size_t size); + +int flb_ml_append_object(struct flb_ml *ml, + uint64_t stream_id, + struct flb_time *tm, + msgpack_object *metadata, + msgpack_object *obj); + +int flb_ml_append_event(struct flb_ml *ml, + uint64_t stream_id, + struct flb_log_event *event); + + +// int flb_ml_append_object(struct flb_ml *ml, uint64_t stream_id, +// struct flb_time *tm, msgpack_object *obj); + +// int flb_ml_append_log_event(struct flb_ml *ml, uint64_t stream_id, +// struct flb_log_event *event); + +int flb_ml_parsers_init(struct flb_config *ctx); + +void flb_ml_flush_pending_now(struct flb_ml *ml); + +void flb_ml_flush_parser_instance(struct flb_ml *ml, + struct flb_ml_parser_ins *parser_i, + uint64_t stream_id, + int forced_flush); + +int flb_ml_auto_flush_init(struct flb_ml *ml); + +int flb_ml_flush_stream_group(struct flb_ml_parser *ml_parser, + struct flb_ml_stream *mst, + struct flb_ml_stream_group *group, + int forced_flush); + +/* Multiline streams */ +int flb_ml_stream_create(struct flb_ml *ml, + char *name, + int name_len, + int (*cb_flush) (struct flb_ml_parser *, + struct flb_ml_stream *, + void *cb_data, + char *buf_data, + size_t buf_size), + void *cb_data, + uint64_t *stream_id); + +int flb_ml_stream_destroy(struct flb_ml_stream *mst); + +void flb_ml_stream_id_destroy_all(struct flb_ml *ml, uint64_t stream_id); + +struct flb_ml_stream *flb_ml_stream_get(struct flb_ml_parser_ins *parser, + uint64_t stream_id); + +struct flb_ml_stream_group *flb_ml_stream_group_get(struct flb_ml_parser_ins *ins, + struct flb_ml_stream *mst, + msgpack_object *group_name); + +int flb_ml_init(struct flb_config *config); +int flb_ml_exit(struct flb_config *config); + +int flb_ml_type_lookup(char *str); + +int flb_ml_flush_stdout(struct flb_ml_parser *parser, + struct flb_ml_stream *mst, + void *data, char *buf_data, size_t buf_size); + +#include "flb_ml_mode.h" + +#endif diff --git a/fluent-bit/include/fluent-bit/multiline/flb_ml_group.h b/fluent-bit/include/fluent-bit/multiline/flb_ml_group.h new file mode 100644 index 00000000..1dd62137 --- /dev/null +++ b/fluent-bit/include/fluent-bit/multiline/flb_ml_group.h @@ -0,0 +1,31 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ML_GROUP_H +#define FLB_ML_GROUP_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/multiline/flb_ml.h> +#include <fluent-bit/multiline/flb_ml_parser.h> + +struct flb_ml_group *flb_ml_group_create(struct flb_ml *ml); +void flb_ml_group_destroy(struct flb_ml_group *group); +int flb_ml_group_add_parser(struct flb_ml *ctx, struct flb_ml_parser_ins *p); + +#endif diff --git a/fluent-bit/include/fluent-bit/multiline/flb_ml_mode.h b/fluent-bit/include/fluent-bit/multiline/flb_ml_mode.h new file mode 100644 index 00000000..3545994c --- /dev/null +++ b/fluent-bit/include/fluent-bit/multiline/flb_ml_mode.h @@ -0,0 +1,38 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ML_MODE_H +#define FLB_ML_MODE_H + +struct flb_ml *flb_ml_mode_create(struct flb_config *config, char *mode, + int flush_ms, char *key); + +/* Python language mode */ +struct flb_ml *flb_ml_mode_python(struct flb_config *config, + int flush_ms, char *key); + +/* Java language mode */ +struct flb_ml *flb_ml_mode_java(struct flb_config *config, + int flush_ms, char *key); + +/* Go language mode */ +struct flb_ml *flb_ml_mode_go(struct flb_config *config, + int flush_ms, char *key); + +#endif diff --git a/fluent-bit/include/fluent-bit/multiline/flb_ml_parser.h b/fluent-bit/include/fluent-bit/multiline/flb_ml_parser.h new file mode 100644 index 00000000..aad5fb02 --- /dev/null +++ b/fluent-bit/include/fluent-bit/multiline/flb_ml_parser.h @@ -0,0 +1,59 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ML_PARSER_H +#define FLB_ML_PARSER_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_parser.h> + +int flb_ml_parser_init(struct flb_ml_parser *ml_parser); + +int flb_ml_parser_builtin_create(struct flb_config *config); + +struct flb_ml_parser *flb_ml_parser_create(struct flb_config *ctx, + char *name, + int type, char *match_str, int negate, + int flush_ms, + char *key_content, + char *key_group, + char *key_pattern, + struct flb_parser *parser_ctx, + char *parser_name); +int flb_ml_parser_destroy(struct flb_ml_parser *ml_parser); +void flb_ml_parser_destroy_all(struct mk_list *list); + +struct flb_ml_parser *flb_ml_parser_get(struct flb_config *ctx, char *name); + +struct flb_ml_parser_ins *flb_ml_parser_instance_create(struct flb_ml *ml, + char *name); +int flb_ml_parser_instance_set(struct flb_ml_parser_ins *p, char *prop, char *val); + +int flb_ml_parser_instance_destroy(struct flb_ml_parser_ins *ins); +int flb_ml_parser_instance_has_data(struct flb_ml_parser_ins *ins); + +/* Built-in multiline parsers */ +struct flb_ml_parser *flb_ml_parser_docker(struct flb_config *config); +struct flb_ml_parser *flb_ml_parser_cri(struct flb_config *config); +struct flb_ml_parser *flb_ml_parser_java(struct flb_config *config, char *key); +struct flb_ml_parser *flb_ml_parser_go(struct flb_config *config, char *key); +struct flb_ml_parser *flb_ml_parser_ruby(struct flb_config *config, char *key); +struct flb_ml_parser *flb_ml_parser_python(struct flb_config *config, char *key); + +#endif diff --git a/fluent-bit/include/fluent-bit/multiline/flb_ml_rule.h b/fluent-bit/include/fluent-bit/multiline/flb_ml_rule.h new file mode 100644 index 00000000..ed46fa77 --- /dev/null +++ b/fluent-bit/include/fluent-bit/multiline/flb_ml_rule.h @@ -0,0 +1,43 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_ML_RULE_H +#define FLB_ML_RULE_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/multiline/flb_ml.h> + +int flb_ml_rule_create(struct flb_ml_parser *ml_parser, + flb_sds_t from_states, + char *regex_pattern, + flb_sds_t to_state, + char *end_pattern); +void flb_ml_rule_destroy(struct flb_ml_rule *rule); +void flb_ml_rule_destroy_all(struct flb_ml_parser *ml_parser); +int flb_ml_rule_process(struct flb_ml_parser *ml_parser, + struct flb_ml_stream *mst, + struct flb_ml_stream_group *group, + msgpack_object *full_map, + void *buf, size_t size, struct flb_time *tm, + msgpack_object *val_content, + msgpack_object *val_pattern); +int flb_ml_rule_init(struct flb_ml_parser *ml_parser); + +#endif diff --git a/fluent-bit/include/fluent-bit/record_accessor/flb_ra_parser.h b/fluent-bit/include/fluent-bit/record_accessor/flb_ra_parser.h new file mode 100644 index 00000000..e646c33a --- /dev/null +++ b/fluent-bit/include/fluent-bit/record_accessor/flb_ra_parser.h @@ -0,0 +1,76 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_RA_PARSER_H +#define FLB_RA_PARSER_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <monkey/mk_core.h> + +#define FLB_RA_PARSER_STRING 0 /* fixed string */ +#define FLB_RA_PARSER_KEYMAP 1 /* record map key */ +#define FLB_RA_PARSER_ARRAY_ID 2 /* fixed string */ +#define FLB_RA_PARSER_FUNC 3 /* record function: tag, time... */ +#define FLB_RA_PARSER_REGEX_ID 4 /* regex id / capture position */ +#define FLB_RA_PARSER_TAG 5 /* full tag */ +#define FLB_RA_PARSER_TAG_PART 6 /* tag part */ + +struct flb_ra_subentry { + int type; /* string = FLB_RA_PARSER_STRING | array id = FLB_RA_PARSER_ARRAY_ID */ + union { + int array_id; + flb_sds_t str; + }; + struct mk_list _head; +}; + +struct flb_ra_array { + int index; + struct mk_list *subkeys; +}; + +struct flb_ra_key { + flb_sds_t name; + struct mk_list *subkeys; +}; + +struct flb_ra_parser { + int type; /* token type */ + int id; /* used by PARSER_REGEX_ID & PARSER_TAG_PART */ + struct flb_ra_key *key; /* context of data type */ + struct mk_list *slist; /* temporary list for subkeys parsing */ + struct mk_list _head; /* link to parent flb_record_accessor->list */ +}; + +struct flb_ra_key *flb_ra_parser_key_add(struct flb_ra_parser *ra, char *key); + +int flb_ra_parser_subentry_add_string(struct flb_ra_parser *rp, char *key); +int flb_ra_parser_subentry_add_array_id(struct flb_ra_parser *rp, int id); + +int flb_ra_parser_subkey_count(struct flb_ra_parser *rp); +void flb_ra_parser_dump(struct flb_ra_parser *rp); +struct flb_ra_parser *flb_ra_parser_string_create(char *str, int len); +struct flb_ra_parser *flb_ra_parser_regex_id_create(int id); +struct flb_ra_parser *flb_ra_parser_meta_create(char *str, int len); +struct flb_ra_parser *flb_ra_parser_tag_create(); +struct flb_ra_parser *flb_ra_parser_tag_part_create(int id); +void flb_ra_parser_destroy(struct flb_ra_parser *rp); + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp.h new file mode 100644 index 00000000..455909ab --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp.h @@ -0,0 +1,190 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_H +#define FLB_SP_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/flb_input.h> +#include <monkey/mk_core.h> +#include <rbtree.h> + +/* Aggregate num type */ +#define FLB_SP_NUM_I64 0 +#define FLB_SP_NUM_F64 1 +#define FLB_SP_BOOLEAN 2 +#define FLB_SP_STRING 3 + +struct sp_buffer { + char* buffer; + size_t size; +}; + +struct aggregate_num { + int type; + int ops; + int64_t i64; + double f64; + bool boolean; + flb_sds_t string; +}; + +struct aggregate_data { + struct aggregate_num *nums; + struct mk_list _head; +}; + +struct timeseries_forecast { + struct aggregate_num *nums; + struct mk_list _head; + + // future time to forecast + double future_time; + + // time offset (the first time value captured) + double offset; + double latest_x; + + double sigma_x; + double sigma_y; + + double sigma_xy; + double sigma_x2; +}; + +struct aggregate_node { + int groupby_keys; + int records; + int nums_size; + struct aggregate_num *nums; + struct aggregate_num *groupby_nums; + + /* Aggregate data */ + struct aggregate_data **aggregate_data; + + /* To keep track of the aggregation nodes */ + struct rb_tree_node _rb_head; + struct mk_list _head; +}; + +struct flb_sp_window_data { + char *buf_data; + size_t buf_size; + struct mk_list _head; +}; + +struct flb_sp_hopping_slot { + struct rb_tree aggregate_tree; + struct mk_list aggregate_list; + int records; + struct mk_list _head; +}; + +struct flb_sp_task_window { + int type; + + int fd; + struct mk_event event; + struct mk_event event_hop; + + struct rb_tree aggregate_tree; + struct mk_list aggregate_list; + + /* Hopping window parameters */ + /* + * first hopping window. Timer event is set to window size for the first, + * and will change to the advance_by time thereafter + */ + bool first_hop; + int fd_hop; + int advance_by; + struct mk_list hopping_slot; + + int records; + + struct mk_list data; +}; + +struct flb_sp_task { + flb_sds_t name; /* task name */ + flb_sds_t query; /* SQL text query */ + + /* + * if the command source is an existent stream (input plugin instance), we + * map the instance address here to perform a fast lookup once the data + * comes in. + */ + void *source_instance; + + /* + * If the command created a new stream, this field keeps a reference to + * the initialized stream context. + */ + void *stream; + + int aggregate_keys; /* do commands contains aggregate keys? */ + struct flb_sp *sp; /* parent context */ + struct flb_sp_cmd *cmd; /* (SQL) commands */ + + struct flb_sp_task_window window; /* task window */ + + void *snapshot; /* snapshot pages for SNAPSHOT sream type */ + + struct mk_list _head; /* link to parent list flb_sp->tasks */ +}; + +struct flb_sp { + struct mk_list tasks; /* processor tasks */ + struct flb_config *config; /* reference to Fluent Bit context */ +}; + +struct flb_sp *flb_sp_create(struct flb_config *config); +void flb_sp_destroy(struct flb_sp *sp); + +int flb_sp_do(struct flb_sp *sp, struct flb_input_instance *in, + const char *tag, int tag_len, + const char *buf_data, size_t buf_size); +int sp_process_data(const char *tag, int tag_len, + const char *buf_data, size_t buf_size, + char **out_buf, size_t *out_size, + struct flb_sp_task *task, + struct flb_sp *sp); +int sp_process_data_aggr(const char *buf_data, size_t buf_size, + const char *tag, int tag_len, + struct flb_sp_task *task, + struct flb_sp *sp, int convert_str_to_num); +void package_results(const char *tag, int tag_len, + char **out_buf, size_t *out_size, + struct flb_sp_task *task); +int sp_process_hopping_slot(const char *tag, int tag_len, + struct flb_sp_task *task); + +int flb_sp_snapshot_create(struct flb_sp_task *task); +struct flb_sp_task *flb_sp_task_create(struct flb_sp *sp, const char *name, + const char *query); +int flb_sp_fd_event(int fd, struct flb_sp *sp); +void flb_sp_task_destroy(struct flb_sp_task *task); +void groupby_nums_destroy(struct aggregate_num *groupby_nums, int size); +void flb_sp_aggregate_node_destroy(struct flb_sp_cmd *cmd, + struct aggregate_node *aggregate_node); + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_aggregate_func.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_aggregate_func.h new file mode 100644 index 00000000..9a097101 --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_aggregate_func.h @@ -0,0 +1,55 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_AGGREGATE_FUNC_H +#define FLB_SP_AGGREGATE_FUNC_H + + +typedef void (*aggregate_function_destroy)(struct aggregate_node *, + int); + +typedef int (*aggregate_function_clone)(struct aggregate_node *, + struct aggregate_node *, + struct flb_sp_cmd_key *, + int); + +typedef void (*aggregate_function_add)(struct aggregate_node *, + struct flb_sp_cmd_key *, + int, + struct flb_time *, + int64_t, double); + +typedef void (*aggregate_function_calc)(struct aggregate_node *, + struct flb_sp_cmd_key *, + msgpack_packer *, + int); + +typedef void (*aggregate_function_remove)(struct aggregate_node *, + struct aggregate_node *, + int); + +extern char aggregate_func_string[AGGREGATE_FUNCTIONS][sizeof("TIMESERIES_FORECAST") + 1]; + +extern aggregate_function_clone aggregate_func_clone[AGGREGATE_FUNCTIONS]; +extern aggregate_function_add aggregate_func_add[AGGREGATE_FUNCTIONS]; +extern aggregate_function_calc aggregate_func_calc[AGGREGATE_FUNCTIONS]; +extern aggregate_function_remove aggregate_func_remove[AGGREGATE_FUNCTIONS]; +extern aggregate_function_destroy aggregate_func_destroy[AGGREGATE_FUNCTIONS]; + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_func_record.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_func_record.h new file mode 100644 index 00000000..90a5348c --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_func_record.h @@ -0,0 +1,30 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_FUNC_RECORD_H +#define FLB_SP_FUNC_RECORD_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/stream_processor/flb_sp_parser.h> + +int flb_sp_func_record(const char *tag, int tag_len, struct flb_time *tms, + msgpack_packer *mp_pck, struct flb_sp_cmd_key *cmd_key); + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_func_time.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_func_time.h new file mode 100644 index 00000000..7ac5aae8 --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_func_time.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_FUNC_TIME_H +#define FLB_SP_FUNC_TIME_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/stream_processor/flb_sp_parser.h> + +int flb_sp_func_time(msgpack_packer *mp_pck, struct flb_sp_cmd_key *cmd_key); + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_groupby.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_groupby.h new file mode 100644 index 00000000..abcc6a2d --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_groupby.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_GROUPBY_H +#define FLB_SP_GROUPBY_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/stream_processor/flb_sp.h> + +int flb_sp_groupby_compare(const void *lhs, const void *rhs); + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_key.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_key.h new file mode 100644 index 00000000..8b6c3303 --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_key.h @@ -0,0 +1,33 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_KEY_H +#define FLB_SP_KEY_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <msgpack.h> + +struct flb_sp_value *flb_sp_key_to_value(flb_sds_t ckey, + msgpack_object map, + struct mk_list *subkeys); +void flb_sp_key_value_destroy(struct flb_sp_value *v); +void flb_sp_key_value_print(struct flb_sp_value *v); + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_parser.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_parser.h new file mode 100644 index 00000000..b5006dca --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_parser.h @@ -0,0 +1,288 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_PARSER_H +#define FLB_SP_PARSER_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/stream_processor/flb_sp.h> +#include <fluent-bit/stream_processor/flb_sp_window.h> + +/* Aggregation functions */ +#define FLB_SP_NOP 0 +#define FLB_SP_AVG 1 +#define FLB_SP_SUM 2 +#define FLB_SP_COUNT 3 +#define FLB_SP_MIN 4 +#define FLB_SP_MAX 5 +#define FLB_SP_FORECAST 6 + +/* Update this whenever a new aggregate function is added */ +#define AGGREGATE_FUNCTIONS 6 + +/* Date time functions */ +#define FLB_SP_NOW 10 +#define FLB_SP_UNIX_TIMESTAMP 11 + +/* Record functions */ +#define FLB_SP_RECORD_TAG 20 +#define FLB_SP_RECORD_TIME 21 + +/* Status */ +#define FLB_SP_OK 0 +#define FLB_SP_ERROR -1 + +/* Command type */ +#define FLB_SP_SELECT 0 +#define FLB_SP_CREATE_STREAM 1 +#define FLB_SP_CREATE_SNAPSHOT 2 +#define FLB_SP_FLUSH_SNAPSHOT 3 + +/* Source type */ +#define FLB_SP_STREAM 0 +#define FLB_SP_TAG 1 + +/* Parameter type */ +#define FLB_SP_KEY 0 +#define FLB_SP_VAL 1 + +/* Expression type */ +enum Expressions { + FLB_LOGICAL_OP = 0, + FLB_EXP_KEY, + FLB_EXP_BOOL, + FLB_EXP_INT, + FLB_EXP_FLOAT, + FLB_EXP_STRING, + FLB_EXP_NULL, + FLB_EXP_FUNC, + FLB_EXP_PARAM, +}; + +/* Logical operation */ +enum Operations { + FLB_EXP_PAR = 0, + + FLB_EXP_NOT, + FLB_EXP_AND, + FLB_EXP_OR, + + FLB_EXP_EQ, + FLB_EXP_LT, + FLB_EXP_LTE, + FLB_EXP_GT, + FLB_EXP_GTE, + + FLB_EXP_IS_NULL, + FLB_EXP_IS_NOT_NULL +}; + +#define FLB_SP_TIME_SECOND 0 +#define FLB_SP_TIME_MINUTE 1 +#define FLB_SP_TIME_HOUR 2 + +/* Groupby key */ +struct flb_sp_cmd_gb_key { + flb_sds_t name; /* key name */ + struct mk_list _head; /* Link to flb_sp_cmd->gb_keys */ + int id; /* Position */ + void *gb_nums; + struct mk_list *subkeys; /* sub-keys selection */ +}; + +/* Property (key/value) */ +struct flb_sp_cmd_prop { + flb_sds_t key; /* key name */ + flb_sds_t val; /* value name */ + struct mk_list _head; /* Link to flb_sp_cmd->stream_props */ +}; + +/* Key selection */ +struct flb_sp_cmd_key { + int aggr_func; /* Aggregation function */ + int time_func; /* Time function */ + int record_func; /* Record function */ + flb_sds_t name; /* Parent Key name */ + flb_sds_t alias; /* Key output alias (key AS alias) */ + void *gb_key; /* Key name reference to gb_key */ + // TODO: make it a general union type (or array of values) + int constant; /* constant parameter value + (used specifically for timeseries_forecast) */ + struct mk_list *subkeys; /* sub-keys selection */ + struct mk_list _head; /* Link to flb_sp_cmd->keys */ +}; + +struct flb_sp_window { + int type; + time_t size; + time_t advance_by; +}; + +struct flb_sp_cmd { + int status; + int type; /* FLB_SP_CREATE_STREAM or FLB_SP_SELECT */ + + /* Stream creation */ + flb_sds_t stream_name; /* Name for created stream */ + struct mk_list stream_props; /* Stream properties: WITH(a='b',..) */ + + /* Selection */ + struct mk_list keys; /* list head of record fields */ + + + struct flb_exp *condition; /* WHERE condition in select statement */ + struct mk_list cond_list; + + struct flb_sp_window window; /* WINDOW window in select statement */ + + struct mk_list gb_keys; /* list head of group-by record fields */ + char *alias; + + /* + * When parsing a SQL statement that have references to keys with sub-keys + * like record['a']['b']['c'], the following 'tmp_subkeys' list will hold + * a list of the discovered sub-keys (linked list). + * + * When the parser gets into the parent field name (record), the list is + * moved to the proper struct flb_sp_key->subkeys list pointer and this + * field is re-created again as an empty list. + */ + struct mk_list *tmp_subkeys; + + /* Limit on the number of records returning */ + int limit; + + /* Source of data */ + int source_type; /* FLB_SP_STREAM or FLB_SP_TAG */ + flb_sds_t source_name; /* Name after stream: or tag: */ +}; + +/* condition value types */ +typedef union { + bool boolean; + int64_t i64; + double f64; + flb_sds_t string; +} sp_val; + +struct flb_exp { + int type; + struct mk_list _head; + struct flb_exp *left; + struct flb_exp *right; +}; + +struct flb_exp_op { + int type; + struct mk_list _head; + struct flb_exp *left; + struct flb_exp *right; + int operation; +}; + +struct flb_exp_key { + int type; + struct mk_list _head; + flb_sds_t name; + struct mk_list *subkeys; + int func; +}; + +struct flb_exp_func { + int type; + struct mk_list _head; + flb_sds_t name; + struct flb_exp_val *(*cb_func) (const char *, int, + struct flb_time *, struct flb_exp_val *); + struct flb_exp *param; +}; + +struct flb_exp_val { + int type; + struct mk_list _head; + sp_val val; +}; + +/* Represent any value object */ +struct flb_sp_value { + int type; + msgpack_object o; + sp_val val; +}; + +struct flb_exp_param { + int type; + struct mk_list _head; + struct flb_exp *param; +}; + +struct flb_sp_cmd *flb_sp_cmd_create(const char *sql); +void flb_sp_cmd_destroy(struct flb_sp_cmd *cmd); + +/* Stream */ +int flb_sp_cmd_stream_new(struct flb_sp_cmd *cmd, const char *stream_name); +int flb_sp_cmd_snapshot_new(struct flb_sp_cmd *cmd, const char *snapshot_name); +int flb_sp_cmd_snapshot_flush_new(struct flb_sp_cmd *cmd, const char *snapshot_name); +int flb_sp_cmd_stream_prop_add(struct flb_sp_cmd *cmd, const char *key, const char *val); +void flb_sp_cmd_stream_prop_del(struct flb_sp_cmd_prop *prop); +const char *flb_sp_cmd_stream_prop_get(struct flb_sp_cmd *cmd, const char *key); + +/* Selection keys */ +int flb_sp_cmd_key_add(struct flb_sp_cmd *cmd, int func, const char *key_name); +void flb_sp_cmd_key_del(struct flb_sp_cmd_key *key); +void flb_sp_cmd_alias_add(struct flb_sp_cmd *cmd, const char *key_alias); +int flb_sp_cmd_source(struct flb_sp_cmd *cmd, int type, const char *source); +void flb_sp_cmd_dump(struct flb_sp_cmd *cmd); + +int flb_sp_cmd_window(struct flb_sp_cmd *cmd, int window_type, + int size, int time_unit, + int advance_by_size, int advance_by_time_unit); + +void flb_sp_cmd_condition_add(struct flb_sp_cmd *cmd, struct flb_exp *e); +struct flb_exp *flb_sp_cmd_operation(struct flb_sp_cmd *cmd, + struct flb_exp *e1, struct flb_exp *e2, + int operation); +struct flb_exp *flb_sp_cmd_comparison(struct flb_sp_cmd *cmd, + struct flb_exp *key, struct flb_exp *val, + int operation); +struct flb_exp *flb_sp_cmd_condition_key(struct flb_sp_cmd *cmd, const char *key); +struct flb_exp *flb_sp_cmd_condition_integer(struct flb_sp_cmd *cmd, + int integer); +struct flb_exp *flb_sp_cmd_condition_float(struct flb_sp_cmd *cmd, float fval); +struct flb_exp *flb_sp_cmd_condition_string(struct flb_sp_cmd *cmd, + const char *string); +struct flb_exp *flb_sp_cmd_condition_boolean(struct flb_sp_cmd *cmd, + bool boolean); +struct flb_exp *flb_sp_cmd_condition_null(struct flb_sp_cmd *cmd); +struct flb_exp *flb_sp_record_function_add(struct flb_sp_cmd *cmd, + char *name, struct flb_exp *param); + +void flb_sp_cmd_condition_del(struct flb_sp_cmd *cmd); + +int flb_sp_cmd_gb_key_add(struct flb_sp_cmd *cmd, const char *key); +void flb_sp_cmd_gb_key_del(struct flb_sp_cmd_gb_key *key); + +void flb_sp_cmd_limit_add(struct flb_sp_cmd *cmd, int limit); + +int flb_sp_cmd_timeseries_forecast(struct flb_sp_cmd *cmd, int func, + const char *key_name, int seconds); + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_record_func.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_record_func.h new file mode 100644 index 00000000..66bfc2ad --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_record_func.h @@ -0,0 +1,85 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_RECORD_FUNC_H +#define FLB_SP_RECORD_FUNC_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_time.h> +#include <fluent-bit/stream_processor/flb_sp_parser.h> + +#define RECORD_FUNCTIONS_SIZE 3 + +typedef struct flb_exp_val *(*record_function_typ) (const char *, int, + struct flb_time *, + struct flb_exp_val *); + +struct flb_exp_val *cb_contains(const char *tag, int tag_len, + struct flb_time *tms, + struct flb_exp_val *param) +{ + struct flb_exp_val *result; + + if (param == NULL) { + return NULL; + } + + result = flb_calloc(1, sizeof(struct flb_exp_val)); + if (!result) { + flb_errno(); + return NULL; + } + + result->type = FLB_EXP_BOOL; + result->val.boolean = true; + + return result; +} + +/* Return the record timestamp */ +struct flb_exp_val *cb_time(const char *tag, int tag_len, + struct flb_time *tms, + struct flb_exp_val *param) +{ + struct flb_exp_val *result; + (void) param; + + result = flb_calloc(1, sizeof(struct flb_exp_val)); + if (!result) { + flb_errno(); + return NULL; + } + + result->type = FLB_EXP_FLOAT; + result->val.f64 = flb_time_to_double(tms); + + return result; +} + +char *record_functions[RECORD_FUNCTIONS_SIZE] = { + "contains", + "time" +}; + +record_function_typ record_functions_ptr[RECORD_FUNCTIONS_SIZE] = { + cb_contains, + cb_time +}; + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_snapshot.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_snapshot.h new file mode 100644 index 00000000..181a95b0 --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_snapshot.h @@ -0,0 +1,49 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_SNAPSHOT_H +#define FLB_SP_SNAPSHOT_H + +#define SNAPSHOT_PAGE_SIZE 1024 + +struct flb_sp_snapshot_page { + int records; + int start_pos; /* Start position of the valid data */ + int end_pos; /* End position of the valid data */ + char *snapshot_page; + struct mk_list _head; +}; + +struct flb_sp_snapshot { + int time_limit; + int record_limit; + int records; + size_t size; + struct mk_list pages; +}; + +int flb_sp_snapshot_update(struct flb_sp_task *task, const char *buf_data, + size_t buf_size, struct flb_time *tms); + +int flb_sp_snapshot_flush(struct flb_sp *sp, struct flb_sp_task *task, + char **out_buf_data, size_t *out_buf_size); + +void flb_sp_snapshot_destroy(struct flb_sp_snapshot *snapshot); + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_stream.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_stream.h new file mode 100644 index 00000000..0cd9981c --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_stream.h @@ -0,0 +1,40 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_STREAM_H +#define FLB_SP_STREAM_H + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_sds.h> + +struct flb_sp_stream { + flb_sds_t name; /* stream name */ + flb_sds_t tag; /* tag specified through properties */ + int routable; /* is it routable ? */ + void *in; /* input instance context */ +}; + +int flb_sp_stream_create(const char *name, struct flb_sp_task *task, + struct flb_sp *sp); +void flb_sp_stream_destroy(struct flb_sp_stream *stream, struct flb_sp *sp); + +int flb_sp_stream_append_data(const char *buf_data, size_t buf_size, + struct flb_sp_stream *stream); + +#endif diff --git a/fluent-bit/include/fluent-bit/stream_processor/flb_sp_window.h b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_window.h new file mode 100644 index 00000000..c38b5d43 --- /dev/null +++ b/fluent-bit/include/fluent-bit/stream_processor/flb_sp_window.h @@ -0,0 +1,33 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SP_WINDOW +#define FLB_SP_WINDOW + +#include <fluent-bit/stream_processor/flb_sp.h> + +#define FLB_SP_WINDOW_DEFAULT 0 +#define FLB_SP_WINDOW_TUMBLING 1 +#define FLB_SP_WINDOW_HOPPING 2 + +void flb_sp_window_prune(struct flb_sp_task *task); +int flb_sp_window_populate(struct flb_sp_task *task, const char *buf_data, + size_t buf_size); + +#endif diff --git a/fluent-bit/include/fluent-bit/tls/flb_tls.h b/fluent-bit/include/fluent-bit/tls/flb_tls.h new file mode 100644 index 00000000..93601a18 --- /dev/null +++ b/fluent-bit/include/fluent-bit/tls/flb_tls.h @@ -0,0 +1,136 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_TLS_H +#define FLB_TLS_H + +#ifdef FLB_HAVE_TLS + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_coro.h> +#include <stddef.h> + +#define FLB_TLS_CLIENT "Fluent Bit" + +/* TLS backend return status on read/write */ +#define FLB_TLS_WANT_READ -0x7e4 +#define FLB_TLS_WANT_WRITE -0x7e6 + +/* Cert Flags */ +#define FLB_TLS_CA_ROOT 1 +#define FLB_TLS_CERT 2 +#define FLB_TLS_PRIV_KEY 4 + +#define FLB_TLS_UNKNOWN_CONNECTION 0 +#define FLB_TLS_UPSTREAM_CONNECTION 1 +#define FLB_TLS_DOWNSTREAM_CONNECTION 2 + +#define FLB_TLS_CLIENT_MODE 0 +#define FLB_TLS_SERVER_MODE 1 + +struct flb_tls; +struct flb_connection; + +struct flb_tls_session { + /* opaque data type for backend session context */ + void *ptr; + struct flb_tls *tls; + struct flb_connection *connection; +}; + +/* + * Structure to connect a backend API library: every backend provided by + * mbedtls.c or openssl.c use this structure to register it self. + */ +struct flb_tls_backend { + /* library backend name */ + char *name; + + /* create backend context */ + void *(*context_create) (int, int, int, + const char *, const char *, + const char *, const char *, + const char *, const char *); + + /* destroy backend context */ + void (*context_destroy) (void *); + + /* Session management */ + void *(*session_create) (struct flb_tls *, int); + int (*session_destroy) (void *); + + /* I/O */ + int (*net_read) (struct flb_tls_session *, void *, size_t); + int (*net_write) (struct flb_tls_session *, const void *data, + size_t len); + int (*net_handshake) (struct flb_tls *, char *, void *); +}; + +/* Main TLS context */ +struct flb_tls { + int verify; /* FLB_TRUE | FLB_FALSE */ + int debug; /* Debug level */ + char *vhost; /* Virtual hostname for SNI */ + int mode; /* Client or Server */ + + /* Bakend library for TLS */ + void *ctx; /* TLS context created */ + struct flb_tls_backend *api; /* backend API */ +}; + +int flb_tls_init(); + +struct flb_tls *flb_tls_create(int mode, + int verify, + int debug, + const char *vhost, + const char *ca_path, + const char *ca_file, const char *crt_file, + const char *key_file, const char *key_passwd); + +int flb_tls_destroy(struct flb_tls *tls); +int flb_tls_load_system_certificates(struct flb_tls *tls); + +struct mk_list *flb_tls_get_config_map(struct flb_config *config); + +int flb_tls_session_destroy(struct flb_tls_session *session); + +int flb_tls_session_create(struct flb_tls *tls, + struct flb_connection *connection, + struct flb_coro *co); + +int flb_tls_net_read(struct flb_tls_session *session, + void *buf, + size_t len); + +int flb_tls_net_read_async(struct flb_coro *th, + struct flb_tls_session *session, + void *buf, size_t len); + +int flb_tls_net_write(struct flb_tls_session *session, + const void *data, size_t len, size_t *out_len); + +int flb_tls_net_write_async(struct flb_coro *th, + struct flb_tls_session *session, + const void *data, size_t len, size_t *out_len); + + +#endif +#endif diff --git a/fluent-bit/include/fluent-bit/tls/flb_tls_info.h.in b/fluent-bit/include/fluent-bit/tls/flb_tls_info.h.in new file mode 100644 index 00000000..56386cf0 --- /dev/null +++ b/fluent-bit/include/fluent-bit/tls/flb_tls_info.h.in @@ -0,0 +1,48 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_TLS_INFO_H +#define FLB_TLS_INFO_H + +#ifdef FLB_HAVE_TLS + +#ifdef FLB_SYSTEM_MACOS +/* On macOS, default PEM format certificates are not provided by + * default. Instead, we'll generate them into: + * "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_SYSCONFDIR@/certs/rootcert.pem" + * during the building process. + */ +#define FLB_DEFAULT_SEARCH_CA_BUNDLE "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_SYSCONFDIR@/certs/rootcert.pem" +#define FLB_DEFAULT_CA_DIR "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_SYSCONFDIR@/certs/" + +#else +/* + * RHEL-family distrbutions do not provide system certificates in + * a format that OpenSSL's CAPath can read, but do provide a single + * packed cert in /etc/certs. + * + * Use the bundled cert as the default trusted CA. + */ +#define FLB_DEFAULT_SEARCH_CA_BUNDLE "/etc/ssl/certs/ca-bundle.crt" +#define FLB_DEFAULT_CA_DIR "/etc/ssl/certs/" + +#endif /* FLB_SYSTEM_MACOS */ + +#endif +#endif diff --git a/fluent-bit/include/fluent-bit/wasm/flb_wasm.h b/fluent-bit/include/fluent-bit/wasm/flb_wasm.h new file mode 100644 index 00000000..7e683527 --- /dev/null +++ b/fluent-bit/include/fluent-bit/wasm/flb_wasm.h @@ -0,0 +1,56 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_WASM_H +#define FLB_WASM_H + +#include "wasm_export.h" + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_time.h> + +/* WASM Context */ +struct flb_wasm { + wasm_module_t module; + wasm_module_inst_t module_inst; + wasm_function_inst_t func; + wasm_exec_env_t exec_env; + uint32_t tag_buffer; + uint32_t record_buffer; + char *buffer; + void *config; /* Fluent Bit context */ + struct mk_list _head; /* Link to flb_config->wasm */ +}; + +void flb_wasm_init(struct flb_config *config); +struct flb_wasm *flb_wasm_instantiate(struct flb_config *config, const char *wasm_path, + struct mk_list *acessible_dir_list, + int stdinfd, int stdoutfd, int stderrfd); + +char *flb_wasm_call_function_format_json(struct flb_wasm *fw, const char *function_name, + const char* tag_data, size_t tag_len, + struct flb_time t, + const char* record_data, size_t record_len); +int flb_wasm_call_wasi_main(struct flb_wasm *fw); +void flb_wasm_buffer_free(struct flb_wasm *fw); +void flb_wasm_destroy(struct flb_wasm *fw); +int flb_wasm_destroy_all(struct flb_config *ctx); + +#endif |