From 5da14042f70711ea5cf66e034699730335462f66 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 5 May 2024 14:08:03 +0200 Subject: Merging upstream version 1.45.3+dfsg. Signed-off-by: Daniel Baumann --- src/fluent-bit/plugins/filter_ecs/ecs.h | 152 ++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 src/fluent-bit/plugins/filter_ecs/ecs.h (limited to 'src/fluent-bit/plugins/filter_ecs/ecs.h') diff --git a/src/fluent-bit/plugins/filter_ecs/ecs.h b/src/fluent-bit/plugins/filter_ecs/ecs.h new file mode 100644 index 000000000..71d0248fa --- /dev/null +++ b/src/fluent-bit/plugins/filter_ecs/ecs.h @@ -0,0 +1,152 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FILTER_ECS_H +#define FLB_FILTER_ECS_H + +#include +#include +#include +#include +#include + +#define FLB_ECS_FILTER_HOST "127.0.0.1" +#define FLB_ECS_FILTER_PORT "51678" +#define FLB_ECS_FILTER_CLUSTER_PATH "/v1/metadata" +#define FLB_ECS_FILTER_TASK_PATH_FORMAT "/v1/tasks?dockerid=%s" +#define FLB_ECS_FILTER_METADATA_RETRIES "2" + +/* + * Kubernetes recommends not running more than 110 pods per node + * In ECS, the number of tasks per instance will vary considerably + * But this should be a very safe starting size for the table + * Since we use the TTL hash table there is no max size. + */ +#define FLB_ECS_FILTER_HASH_TABLE_SIZE 100 + + +struct flb_ecs_metadata_key { + flb_sds_t key; + flb_sds_t template; + struct flb_record_accessor *ra; + + struct mk_list _head; +}; + +struct flb_ecs_metadata_buffer { + /* msgpack_sbuffer */ + char *buf; + size_t size; + + /* unpacked object to use with flb_ra_translate */ + msgpack_unpacked unpacked; + msgpack_object obj; + int free_packer; + + /* the hash table only stores a pointer- we need the list to track and free these */ + struct mk_list _head; + /* we clean up the memory for these once ecs_meta_cache_ttl has expired */ + time_t last_used_time; + + /* + * To remove from the hash table on TTL expiration, we need the ID + * While we use a TTL hash, it won't clean up the memory, so we have a separate routine for that + * and it needs to ensure that the list and hash table has the same contents + */ + flb_sds_t id; +}; + +struct flb_ecs_cluster_metadata { + flb_sds_t cluster_name; + flb_sds_t container_instance_arn; + flb_sds_t container_instance_id; + flb_sds_t ecs_agent_version; +}; + +/* + * The ECS Agent task response gives us both task & container at the same time + * We need a temporary structure to organize the task metadata + * Before we create the final flb_ecs_metadata_buffer objects with all metadata + * So this struct just stores tmp pointers to the deserialized msgpack + */ +struct flb_ecs_task_metadata { + const char* task_arn; + int task_arn_len; + const char *task_id; + int task_id_len; + const char *task_def_family; + int task_def_family_len; + const char *task_def_version; + int task_def_version_len; +}; + +struct flb_filter_ecs { + /* upstream connection to ECS Agent */ + struct flb_upstream *ecs_upstream; + + /* Filter plugin instance reference */ + struct flb_filter_instance *ins; + + struct mk_list metadata_keys; + int metadata_keys_len; + + flb_sds_t ecs_host; + int ecs_port; + + int agent_endpoint_retries; + + /* + * This field is used when we build new container metadata objects + */ + struct flb_ecs_cluster_metadata cluster_metadata; + int has_cluster_metadata; + /* + * If looking up the container fails, we should still always be able to + * attach cluster metadata. So we have a fallback metadata buffer for that. + * For example, users may want to attach cluster name to Docker Daemon logs, + * even though Docker is not an AWS ECS Task/container. + */ + struct flb_ecs_metadata_buffer cluster_meta_buf; + + /* + * Maps 12 char container short ID to metadata buffer + */ + struct flb_hash_table *container_hash_table; + + /* + * The hash table only stores pointers, so we keep a list of meta objects + * that need to be freed + */ + struct mk_list metadata_buffers; + + /* + * Fluent Bit may pick up logs for containers that were not scheduled by ECS + * These will lead to continuous error messages. Therefore, we store + * a hash table of tags for which we could not get metadata so we can stop + * retrying on them. + */ + struct flb_hash_table *failed_metadata_request_tags; + + int ecs_meta_cache_ttl; + char *ecs_tag_prefix; + int ecs_tag_prefix_len; + int cluster_metadata_only; +}; + +#endif -- cgit v1.2.3