diff options
Diffstat (limited to 'src/fluent-bit/plugins/in_http')
-rw-r--r-- | src/fluent-bit/plugins/in_http/CMakeLists.txt | 12 | ||||
-rw-r--r-- | src/fluent-bit/plugins/in_http/http.c | 204 | ||||
-rw-r--r-- | src/fluent-bit/plugins/in_http/http.h | 58 | ||||
-rw-r--r-- | src/fluent-bit/plugins/in_http/http_config.c | 157 | ||||
-rw-r--r-- | src/fluent-bit/plugins/in_http/http_config.h | 29 | ||||
-rw-r--r-- | src/fluent-bit/plugins/in_http/http_conn.c | 306 | ||||
-rw-r--r-- | src/fluent-bit/plugins/in_http/http_conn.h | 54 | ||||
-rw-r--r-- | src/fluent-bit/plugins/in_http/http_prot.c | 665 | ||||
-rw-r--r-- | src/fluent-bit/plugins/in_http/http_prot.h | 31 |
9 files changed, 1516 insertions, 0 deletions
diff --git a/src/fluent-bit/plugins/in_http/CMakeLists.txt b/src/fluent-bit/plugins/in_http/CMakeLists.txt new file mode 100644 index 000000000..69ebeab71 --- /dev/null +++ b/src/fluent-bit/plugins/in_http/CMakeLists.txt @@ -0,0 +1,12 @@ +if(NOT FLB_METRICS) + message(FATAL_ERROR "HTTP input plugin requires FLB_HTTP_SERVER=On.") +endif() + +set(src + http.c + http_conn.c + http_prot.c + http_config.c + ) + +FLB_PLUGIN(in_http "${src}" "monkey-core-static") diff --git a/src/fluent-bit/plugins/in_http/http.c b/src/fluent-bit/plugins/in_http/http.c new file mode 100644 index 000000000..7bbe6e5f4 --- /dev/null +++ b/src/fluent-bit/plugins/in_http/http.c @@ -0,0 +1,204 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include <fluent-bit/flb_input_plugin.h> +#include <fluent-bit/flb_network.h> +#include <fluent-bit/flb_config.h> + +#include "http.h" +#include "http_conn.h" +#include "http_config.h" + +/* + * For a server event, the collection event means a new client have arrived, we + * accept the connection and create a new TCP instance which will wait for + * JSON map messages. + */ +static int in_http_collect(struct flb_input_instance *ins, + struct flb_config *config, void *in_context) +{ + struct flb_connection *connection; + struct http_conn *conn; + struct flb_http *ctx; + + ctx = in_context; + + connection = flb_downstream_conn_get(ctx->downstream); + + if (connection == NULL) { + flb_plg_error(ctx->ins, "could not accept new connection"); + + return -1; + } + + flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", + connection->fd); + + conn = http_conn_add(connection, ctx); + + if (conn == NULL) { + flb_downstream_conn_release(connection); + + return -1; + } + + return 0; +} + +static int in_http_init(struct flb_input_instance *ins, + struct flb_config *config, void *data) +{ + unsigned short int port; + int ret; + struct flb_http *ctx; + + (void) data; + + /* Create context and basic conf */ + ctx = http_config_create(ins); + if (!ctx) { + return -1; + } + + ctx->collector_id = -1; + + /* Populate context with config map defaults and incoming properties */ + ret = flb_input_config_map_set(ins, (void *) ctx); + if (ret == -1) { + flb_plg_error(ctx->ins, "configuration error"); + http_config_destroy(ctx); + return -1; + } + + /* Set the context */ + flb_input_set_context(ins, ctx); + + port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); + + ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, + ins->flags, + ctx->listen, + port, + ins->tls, + config, + &ins->net_setup); + + if (ctx->downstream == NULL) { + flb_plg_error(ctx->ins, + "could not initialize downstream on %s:%s. Aborting", + ctx->listen, ctx->tcp_port); + + http_config_destroy(ctx); + + return -1; + } + + flb_input_downstream_set(ctx->downstream, ctx->ins); + + if (ctx->successful_response_code != 200 && + ctx->successful_response_code != 201 && + ctx->successful_response_code != 204) { + flb_plg_error(ctx->ins, "%d is not supported response code. Use default 201", + ctx->successful_response_code); + ctx->successful_response_code = 201; + } + + /* Collect upon data available on the standard input */ + ret = flb_input_set_collector_socket(ins, + in_http_collect, + ctx->downstream->server_fd, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); + http_config_destroy(ctx); + + return -1; + } + + ctx->collector_id = ret; + + return 0; +} + +static int in_http_exit(void *data, struct flb_config *config) +{ + struct flb_http *ctx; + + (void) config; + + ctx = data; + + if (ctx != NULL) { + http_config_destroy(ctx); + } + + return 0; +} + +/* Configuration properties map */ +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, + 0, FLB_TRUE, offsetof(struct flb_http, buffer_max_size), + "" + }, + + { + FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, + 0, FLB_TRUE, offsetof(struct flb_http, buffer_chunk_size), + "" + }, + + { + FLB_CONFIG_MAP_SLIST_1, "success_header", NULL, + FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_http, success_headers), + "Add an HTTP header key/value pair on success. Multiple headers can be set" + }, + + { + FLB_CONFIG_MAP_STR, "tag_key", NULL, + 0, FLB_TRUE, offsetof(struct flb_http, tag_key), + "" + }, + { + FLB_CONFIG_MAP_INT, "successful_response_code", "201", + 0, FLB_TRUE, offsetof(struct flb_http, successful_response_code), + "Set successful response code. 200, 201 and 204 are supported." + }, + + + /* EOF */ + {0} +}; + +/* Plugin reference */ +struct flb_input_plugin in_http_plugin = { + .name = "http", + .description = "HTTP", + .cb_init = in_http_init, + .cb_pre_run = NULL, + .cb_collect = in_http_collect, + .cb_flush_buf = NULL, + .cb_pause = NULL, + .cb_resume = NULL, + .cb_exit = in_http_exit, + .config_map = config_map, + .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS +}; diff --git a/src/fluent-bit/plugins/in_http/http.h b/src/fluent-bit/plugins/in_http/http.h new file mode 100644 index 000000000..f9832e9b2 --- /dev/null +++ b/src/fluent-bit/plugins/in_http/http.h @@ -0,0 +1,58 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_HTTP_H +#define FLB_IN_HTTP_H + +#include <fluent-bit/flb_downstream.h> +#include <fluent-bit/flb_config.h> +#include <fluent-bit/flb_input.h> +#include <fluent-bit/flb_utils.h> +#include <fluent-bit/flb_log_event_encoder.h> + +#include <monkey/monkey.h> + +#define HTTP_BUFFER_MAX_SIZE "4M" +#define HTTP_BUFFER_CHUNK_SIZE "512K" + +struct flb_http { + int successful_response_code; + flb_sds_t listen; + flb_sds_t tcp_port; + const char *tag_key; + + int collector_id; + + /* Success HTTP headers */ + struct mk_list *success_headers; + flb_sds_t success_headers_str; + + size_t buffer_max_size; /* Maximum buffer size */ + size_t buffer_chunk_size; /* Chunk allocation size */ + + struct flb_log_event_encoder log_encoder; + struct flb_downstream *downstream; /* Client manager */ + struct mk_list connections; /* linked list of connections */ + + struct mk_server *server; + struct flb_input_instance *ins; +}; + + +#endif diff --git a/src/fluent-bit/plugins/in_http/http_config.c b/src/fluent-bit/plugins/in_http/http_config.c new file mode 100644 index 000000000..f23759a66 --- /dev/null +++ b/src/fluent-bit/plugins/in_http/http_config.c @@ -0,0 +1,157 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit/flb_input_plugin.h> + +#include "http.h" +#include "http_config.h" +#include "http_conn.h" +#include "http_config.h" + +struct flb_http *http_config_create(struct flb_input_instance *ins) +{ + struct mk_list *header_iterator; + struct flb_slist_entry *header_value; + struct flb_slist_entry *header_name; + struct flb_config_map_val *header_pair; + char port[8]; + int ret; + struct flb_http *ctx; + + ctx = flb_calloc(1, sizeof(struct flb_http)); + if (!ctx) { + flb_errno(); + return NULL; + } + ctx->ins = ins; + mk_list_init(&ctx->connections); + + /* Load the config map */ + ret = flb_input_config_map_set(ins, (void *) ctx); + if (ret == -1) { + flb_free(ctx); + return NULL; + } + + /* Listen interface (if not set, defaults to 0.0.0.0:9880) */ + flb_input_net_default_listener("0.0.0.0", 9880, ins); + + ctx->listen = flb_strdup(ins->host.listen); + snprintf(port, sizeof(port) - 1, "%d", ins->host.port); + ctx->tcp_port = flb_strdup(port); + + /* HTTP Server specifics */ + ctx->server = flb_calloc(1, sizeof(struct mk_server)); + ctx->server->keep_alive = MK_TRUE; + + /* monkey detects server->workers == 0 as the server not being initialized at the + * moment so we want to make sure that it stays that way! + */ + + ret = flb_log_event_encoder_init(&ctx->log_encoder, + FLB_LOG_EVENT_FORMAT_DEFAULT); + + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_plg_error(ctx->ins, "error initializing event encoder : %d", ret); + + http_config_destroy(ctx); + + return NULL; + } + + ctx->success_headers_str = flb_sds_create_size(1); + + if (ctx->success_headers_str == NULL) { + http_config_destroy(ctx); + + return NULL; + } + + flb_config_map_foreach(header_iterator, header_pair, ctx->success_headers) { + header_name = mk_list_entry_first(header_pair->val.list, + struct flb_slist_entry, + _head); + + header_value = mk_list_entry_last(header_pair->val.list, + struct flb_slist_entry, + _head); + + ret = flb_sds_cat_safe(&ctx->success_headers_str, + header_name->str, + flb_sds_len(header_name->str)); + + if (ret == 0) { + ret = flb_sds_cat_safe(&ctx->success_headers_str, + ": ", + 2); + } + + if (ret == 0) { + ret = flb_sds_cat_safe(&ctx->success_headers_str, + header_value->str, + flb_sds_len(header_value->str)); + } + + if (ret == 0) { + ret = flb_sds_cat_safe(&ctx->success_headers_str, + "\r\n", + 2); + } + + if (ret != 0) { + http_config_destroy(ctx); + + return NULL; + } + } + + return ctx; +} + +int http_config_destroy(struct flb_http *ctx) +{ + /* release all connections */ + http_conn_release_all(ctx); + + flb_log_event_encoder_destroy(&ctx->log_encoder); + + if (ctx->collector_id != -1) { + flb_input_collector_delete(ctx->collector_id, ctx->ins); + + ctx->collector_id = -1; + } + + if (ctx->downstream != NULL) { + flb_downstream_destroy(ctx->downstream); + } + + if (ctx->server) { + flb_free(ctx->server); + } + + if (ctx->success_headers_str != NULL) { + flb_sds_destroy(ctx->success_headers_str); + } + + + flb_free(ctx->listen); + flb_free(ctx->tcp_port); + flb_free(ctx); + return 0; +} diff --git a/src/fluent-bit/plugins/in_http/http_config.h b/src/fluent-bit/plugins/in_http/http_config.h new file mode 100644 index 000000000..8a9611116 --- /dev/null +++ b/src/fluent-bit/plugins/in_http/http_config.h @@ -0,0 +1,29 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_HTTP_CONFIG_H +#define FLB_IN_HTTP_CONFIG_H + +#include <fluent-bit/flb_input_plugin.h> +#include "http.h" + +struct flb_http *http_config_create(struct flb_input_instance *ins); +int http_config_destroy(struct flb_http *ctx); + +#endif diff --git a/src/fluent-bit/plugins/in_http/http_conn.c b/src/fluent-bit/plugins/in_http/http_conn.c new file mode 100644 index 000000000..a5d9efa98 --- /dev/null +++ b/src/fluent-bit/plugins/in_http/http_conn.c @@ -0,0 +1,306 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit/flb_input_plugin.h> +#include <fluent-bit/flb_engine.h> + +#include "http.h" +#include "http_conn.h" +#include "http_prot.h" + +static void http_conn_request_init(struct mk_http_session *session, + struct mk_http_request *request); + +static int http_conn_event(void *data) +{ + int status; + size_t size; + ssize_t available; + ssize_t bytes; + char *tmp; + char *request_end; + size_t request_len; + struct flb_connection *connection; + struct http_conn *conn; + struct mk_event *event; + struct flb_http *ctx; + + connection = (struct flb_connection *) data; + + conn = connection->user_data; + + ctx = conn->ctx; + + event = &connection->event; + + if (event->mask & MK_EVENT_READ) { + available = (conn->buf_size - conn->buf_len) - 1; + if (available < 1) { + if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { + flb_plg_trace(ctx->ins, + "fd=%i incoming data exceed limit (%zu KB)", + event->fd, (ctx->buffer_max_size / 1024)); + http_conn_del(conn); + return -1; + } + + size = conn->buf_size + ctx->buffer_chunk_size; + tmp = flb_realloc(conn->buf_data, size); + if (!tmp) { + flb_errno(); + return -1; + } + flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %zu", + event->fd, conn->buf_size, size); + + conn->buf_data = tmp; + conn->buf_size = size; + available = (conn->buf_size - conn->buf_len) - 1; + } + + /* Read data */ + bytes = flb_io_net_read(connection, + (void *) &conn->buf_data[conn->buf_len], + available); + + if (bytes <= 0) { + flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); + http_conn_del(conn); + return -1; + } + + flb_plg_trace(ctx->ins, "read()=%zi pre_len=%i now_len=%zi", + bytes, conn->buf_len, conn->buf_len + bytes); + conn->buf_len += bytes; + conn->buf_data[conn->buf_len] = '\0'; + + status = mk_http_parser(&conn->request, &conn->session.parser, + conn->buf_data, conn->buf_len, conn->session.server); + + if (status == MK_HTTP_PARSER_OK) { + /* Do more logic parsing and checks for this request */ + http_prot_handle(ctx, conn, &conn->session, &conn->request); + + /* Evict the processed request from the connection buffer and reinitialize + * the HTTP parser. + */ + + request_end = NULL; + + if (NULL != conn->request.data.data) { + request_end = &conn->request.data.data[conn->request.data.len]; + } + else { + request_end = strstr(conn->buf_data, "\r\n\r\n"); + + if(NULL != request_end) { + request_end = &request_end[4]; + } + } + + if (NULL != request_end) { + request_len = (size_t)(request_end - conn->buf_data); + + if (0 < (conn->buf_len - request_len)) { + memmove(conn->buf_data, &conn->buf_data[request_len], + conn->buf_len - request_len); + + conn->buf_data[conn->buf_len - request_len] = '\0'; + conn->buf_len -= request_len; + } + else { + memset(conn->buf_data, 0, request_len); + + conn->buf_len = 0; + } + + /* Reinitialize the parser so the next request is properly + * handled, the additional memset intends to wipe any left over data + * from the headers parsed in the previous request. + */ + memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); + mk_http_parser_init(&conn->session.parser); + http_conn_request_init(&conn->session, &conn->request); + } + } + else if (status == MK_HTTP_PARSER_ERROR) { + http_prot_handle_error(ctx, conn, &conn->session, &conn->request); + + /* Reinitialize the parser so the next request is properly + * handled, the additional memset intends to wipe any left over data + * from the headers parsed in the previous request. + */ + memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); + mk_http_parser_init(&conn->session.parser); + http_conn_request_init(&conn->session, &conn->request); + } + + /* FIXME: add Protocol handler here */ + return bytes; + } + + if (event->mask & MK_EVENT_CLOSE) { + flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); + http_conn_del(conn); + return -1; + } + + return 0; + +} + +static void http_conn_session_init(struct mk_http_session *session, + struct mk_server *server, + int client_fd) +{ + /* Alloc memory for node */ + session->_sched_init = MK_TRUE; + session->pipelined = MK_FALSE; + session->counter_connections = 0; + session->close_now = MK_FALSE; + session->status = MK_REQUEST_STATUS_INCOMPLETE; + session->server = server; + session->socket = client_fd; + + /* creation time in unix time */ + session->init_time = time(NULL); + + session->channel = mk_channel_new(MK_CHANNEL_SOCKET, session->socket); + session->channel->io = session->server->network; + + /* Init session request list */ + mk_list_init(&session->request_list); + + /* Initialize the parser */ + mk_http_parser_init(&session->parser); +} + +static void http_conn_request_init(struct mk_http_session *session, + struct mk_http_request *request) +{ + memset(request, 0, sizeof(struct mk_http_request)); + + mk_http_request_init(session, request, session->server); + + request->in_headers.type = MK_STREAM_IOV; + request->in_headers.dynamic = MK_FALSE; + request->in_headers.cb_consumed = NULL; + request->in_headers.cb_finished = NULL; + request->in_headers.stream = &request->stream; + + mk_list_add(&request->in_headers._head, &request->stream.inputs); + + request->session = session; +} + +struct http_conn *http_conn_add(struct flb_connection *connection, + struct flb_http *ctx) +{ + struct http_conn *conn; + int ret; + + conn = flb_calloc(1, sizeof(struct http_conn)); + if (!conn) { + flb_errno(); + return NULL; + } + + conn->connection = connection; + + /* Set data for the event-loop */ + MK_EVENT_NEW(&connection->event); + + connection->user_data = conn; + connection->event.type = FLB_ENGINE_EV_CUSTOM; + connection->event.handler = http_conn_event; + + /* Connection info */ + conn->ctx = ctx; + conn->buf_len = 0; + + conn->buf_data = flb_malloc(ctx->buffer_chunk_size); + if (!conn->buf_data) { + flb_errno(); + + flb_plg_error(ctx->ins, "could not allocate new connection"); + flb_free(conn); + + return NULL; + } + conn->buf_size = ctx->buffer_chunk_size; + + /* Register instance into the event loop */ + ret = mk_event_add(flb_engine_evl_get(), + connection->fd, + FLB_ENGINE_EV_CUSTOM, + MK_EVENT_READ, + &connection->event); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not register new connection"); + + flb_free(conn->buf_data); + flb_free(conn); + + return NULL; + } + + /* Initialize HTTP Session: this is a custom context for Monkey HTTP */ + http_conn_session_init(&conn->session, ctx->server, conn->connection->fd); + + /* Initialize HTTP Request: this is the initial request and it will be reinitialized + * automatically after the request is handled so it can be used for the next one. + */ + http_conn_request_init(&conn->session, &conn->request); + + /* Link connection node to parent context list */ + mk_list_add(&conn->_head, &ctx->connections); + + return conn; +} + +int http_conn_del(struct http_conn *conn) +{ + if (conn->session.channel != NULL) { + mk_channel_release(conn->session.channel); + } + + /* The downstream unregisters the file descriptor from the event-loop + * so there's nothing to be done by the plugin + */ + flb_downstream_conn_release(conn->connection); + + mk_list_del(&conn->_head); + + flb_free(conn->buf_data); + flb_free(conn); + + return 0; +} + +void http_conn_release_all(struct flb_http *ctx) +{ + struct mk_list *tmp; + struct mk_list *head; + struct http_conn *conn; + + mk_list_foreach_safe(head, tmp, &ctx->connections) { + conn = mk_list_entry(head, struct http_conn, _head); + http_conn_del(conn); + } +} diff --git a/src/fluent-bit/plugins/in_http/http_conn.h b/src/fluent-bit/plugins/in_http/http_conn.h new file mode 100644 index 000000000..8e1078982 --- /dev/null +++ b/src/fluent-bit/plugins/in_http/http_conn.h @@ -0,0 +1,54 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_HTTP_CONN +#define FLB_IN_HTTP_CONN + +#include <fluent-bit/flb_input_plugin.h> +#include <fluent-bit/flb_connection.h> + +#include <monkey/mk_http.h> +#include <monkey/mk_http_parser.h> +#include <monkey/mk_utils.h> + +struct http_conn { + /* Buffer */ + char *buf_data; /* Buffer data */ + int buf_len; /* Data length */ + int buf_size; /* Buffer size */ + + /* + * Parser context: we only held one parser per connection + * which is re-used everytime we have a new request. + */ + struct mk_http_parser parser; + struct mk_http_request request; + struct mk_http_session session; + struct flb_connection *connection; + + void *ctx; /* Plugin parent context */ + struct mk_list _head; /* link to flb_http->connections */ +}; + +struct http_conn *http_conn_add(struct flb_connection *connection, struct flb_http *ctx); +int http_conn_del(struct http_conn *conn); +void http_conn_release_all(struct flb_http *ctx); + + +#endif diff --git a/src/fluent-bit/plugins/in_http/http_prot.c b/src/fluent-bit/plugins/in_http/http_prot.c new file mode 100644 index 000000000..ab16eb328 --- /dev/null +++ b/src/fluent-bit/plugins/in_http/http_prot.c @@ -0,0 +1,665 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fluent-bit/flb_input_plugin.h> +#include <fluent-bit/flb_version.h> +#include <fluent-bit/flb_error.h> +#include <fluent-bit/flb_pack.h> + +#include <monkey/monkey.h> +#include <monkey/mk_core.h> + +#include "http.h" +#include "http_conn.h" + +#define HTTP_CONTENT_JSON 0 +#define HTTP_CONTENT_URLENCODED 1 + +static inline char hex2nibble(char c) +{ + if ((c >= 0x30) && (c <= '9')) { + return c - 0x30; + } + // 0x30-0x39 are digits, 0x41-0x46 A-F, + // so there is a gap at 0x40 + if ((c >= 'A') && (c <= 'F')) { + return (c - 'A') + 10; + } + if ((c >= 'a') && (c <= 'f')) { + return (c - 'a') + 10; + } + return 0; +} + +static int sds_uri_decode(flb_sds_t s) +{ + char buf[1024]; + char *optr; + char *iptr; + + + for (optr = buf, iptr = s; iptr < s + flb_sds_len(s) && optr-buf < sizeof(buf); iptr++) { + if (*iptr == '%') { + if (iptr+2 > (s + flb_sds_len(s))) { + return -1; + } + *optr++ = hex2nibble(*(iptr+1)) << 4 | hex2nibble(*(iptr+2)); + iptr+=2; + } else if (*iptr == '+') { + *optr++ = ' '; + } else { + *optr++ = *iptr; + } + } + + memcpy(s, buf, optr-buf); + s[optr-buf] = '\0'; + flb_sds_len_set(s, (optr-buf)); + + return 0; +} + +static int send_response(struct http_conn *conn, int http_status, char *message) +{ + struct flb_http *context; + size_t sent; + int len; + flb_sds_t out; + + context = (struct flb_http *) conn->ctx; + + out = flb_sds_create_size(256); + if (!out) { + return -1; + } + + if (message) { + len = strlen(message); + } + else { + len = 0; + } + + if (http_status == 201) { + flb_sds_printf(&out, + "HTTP/1.1 201 Created \r\n" + "Server: Fluent Bit v%s\r\n" + "%s" + "Content-Length: 0\r\n\r\n", + FLB_VERSION_STR, + context->success_headers_str); + } + else if (http_status == 200) { + flb_sds_printf(&out, + "HTTP/1.1 200 OK\r\n" + "Server: Fluent Bit v%s\r\n" + "%s" + "Content-Length: 0\r\n\r\n", + FLB_VERSION_STR, + context->success_headers_str); + } + else if (http_status == 204) { + flb_sds_printf(&out, + "HTTP/1.1 204 No Content\r\n" + "Server: Fluent Bit v%s\r\n" + "%s" + "\r\n\r\n", + FLB_VERSION_STR, + context->success_headers_str); + } + else if (http_status == 400) { + flb_sds_printf(&out, + "HTTP/1.1 400 Forbidden\r\n" + "Server: Fluent Bit v%s\r\n" + "Content-Length: %i\r\n\r\n%s", + FLB_VERSION_STR, + len, message); + } + + /* We should check this operations result */ + flb_io_net_write(conn->connection, + (void *) out, + flb_sds_len(out), + &sent); + + flb_sds_destroy(out); + + return 0; +} + +/* implements functionality to get tag from key in record */ +static flb_sds_t tag_key(struct flb_http *ctx, msgpack_object *map) +{ + size_t map_size = map->via.map.size; + msgpack_object_kv *kv; + msgpack_object key; + msgpack_object val; + char *key_str = NULL; + char *val_str = NULL; + size_t key_str_size = 0; + size_t val_str_size = 0; + int j; + int check = FLB_FALSE; + int found = FLB_FALSE; + flb_sds_t tag; + + kv = map->via.map.ptr; + + for(j=0; j < map_size; j++) { + check = FLB_FALSE; + found = FLB_FALSE; + key = (kv+j)->key; + if (key.type == MSGPACK_OBJECT_BIN) { + key_str = (char *) key.via.bin.ptr; + key_str_size = key.via.bin.size; + check = FLB_TRUE; + } + if (key.type == MSGPACK_OBJECT_STR) { + key_str = (char *) key.via.str.ptr; + key_str_size = key.via.str.size; + check = FLB_TRUE; + } + + if (check == FLB_TRUE) { + if (strncmp(ctx->tag_key, key_str, key_str_size) == 0) { + val = (kv+j)->val; + if (val.type == MSGPACK_OBJECT_BIN) { + val_str = (char *) val.via.bin.ptr; + val_str_size = val.via.str.size; + found = FLB_TRUE; + break; + } + if (val.type == MSGPACK_OBJECT_STR) { + val_str = (char *) val.via.str.ptr; + val_str_size = val.via.str.size; + found = FLB_TRUE; + break; + } + } + } + } + + if (found == FLB_TRUE) { + tag = flb_sds_create_len(val_str, val_str_size); + if (!tag) { + flb_errno(); + return NULL; + } + return tag; + } + + + flb_plg_error(ctx->ins, "Could not find tag_key %s in record", ctx->tag_key); + return NULL; +} + +int process_pack(struct flb_http *ctx, flb_sds_t tag, char *buf, size_t size) +{ + int ret; + size_t off = 0; + msgpack_unpacked result; + struct flb_time tm; + int i = 0; + msgpack_object *obj; + msgpack_object record; + flb_sds_t tag_from_record = NULL; + + flb_time_get(&tm); + + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, buf, size, &off) == MSGPACK_UNPACK_SUCCESS) { + if (result.data.type == MSGPACK_OBJECT_MAP) { + tag_from_record = NULL; + if (ctx->tag_key) { + obj = &result.data; + tag_from_record = tag_key(ctx, obj); + } + + ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_set_timestamp( + &ctx->log_encoder, + &tm); + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_set_body_from_msgpack_object( + &ctx->log_encoder, + &result.data); + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + if (tag_from_record) { + flb_input_log_append(ctx->ins, + tag_from_record, + flb_sds_len(tag_from_record), + ctx->log_encoder.output_buffer, + ctx->log_encoder.output_length); + + flb_sds_destroy(tag_from_record); + } + else if (tag) { + flb_input_log_append(ctx->ins, tag, flb_sds_len(tag), + ctx->log_encoder.output_buffer, + ctx->log_encoder.output_length); + } + else { + /* use default plugin Tag (it internal name, e.g: http.0 */ + flb_input_log_append(ctx->ins, NULL, 0, + ctx->log_encoder.output_buffer, + ctx->log_encoder.output_length); + } + } + else { + flb_plg_error(ctx->ins, "Error encoding record : %d", ret); + } + + flb_log_event_encoder_reset(&ctx->log_encoder); + } + else if (result.data.type == MSGPACK_OBJECT_ARRAY) { + obj = &result.data; + for (i = 0; i < obj->via.array.size; i++) + { + record = obj->via.array.ptr[i]; + + tag_from_record = NULL; + if (ctx->tag_key) { + tag_from_record = tag_key(ctx, &record); + } + + ret = flb_log_event_encoder_begin_record(&ctx->log_encoder); + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_set_timestamp( + &ctx->log_encoder, + &tm); + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_set_body_from_msgpack_object( + &ctx->log_encoder, + &record); + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + if (tag_from_record) { + flb_input_log_append(ctx->ins, + tag_from_record, + flb_sds_len(tag_from_record), + ctx->log_encoder.output_buffer, + ctx->log_encoder.output_length); + + flb_sds_destroy(tag_from_record); + } + else if (tag) { + flb_input_log_append(ctx->ins, tag, flb_sds_len(tag), + ctx->log_encoder.output_buffer, + ctx->log_encoder.output_length); + } + else { + /* use default plugin Tag (it internal name, e.g: http.0 */ + flb_input_log_append(ctx->ins, NULL, 0, + ctx->log_encoder.output_buffer, + ctx->log_encoder.output_length); + } + } + else { + flb_plg_error(ctx->ins, "Error encoding record : %d", ret); + } + + /* TODO : Optimize this + * + * This is wasteful, considering that we are emitting a series + * of records we should start and commit each one and then + * emit them all at once after the loop. + */ + + flb_log_event_encoder_reset(&ctx->log_encoder); + } + + break; + } + else { + flb_plg_error(ctx->ins, "skip record from invalid type: %i", + result.data.type); + + msgpack_unpacked_destroy(&result); + + return -1; + } + } + + msgpack_unpacked_destroy(&result); + + return 0; +} + +static ssize_t parse_payload_json(struct flb_http *ctx, flb_sds_t tag, + char *payload, size_t size) +{ + int ret; + int out_size; + char *pack; + struct flb_pack_state pack_state; + + /* Initialize packer */ + flb_pack_state_init(&pack_state); + + /* Pack JSON as msgpack */ + ret = flb_pack_json_state(payload, size, + &pack, &out_size, &pack_state); + flb_pack_state_reset(&pack_state); + + /* Handle exceptions */ + if (ret == FLB_ERR_JSON_PART) { + flb_plg_warn(ctx->ins, "JSON data is incomplete, skipping"); + return -1; + } + else if (ret == FLB_ERR_JSON_INVAL) { + flb_plg_warn(ctx->ins, "invalid JSON message, skipping"); + return -1; + } + else if (ret == -1) { + return -1; + } + + /* Process the packaged JSON and return the last byte used */ + process_pack(ctx, tag, pack, out_size); + flb_free(pack); + + return 0; +} + +static ssize_t parse_payload_urlencoded(struct flb_http *ctx, flb_sds_t tag, + char *payload, size_t size) +{ + struct mk_list *kvs; + struct mk_list *head = NULL; + struct flb_split_entry *cur = NULL; + char **keys = NULL; + char **vals = NULL; + char *sep; + char *start; + int idx = 0; + int ret = -1; + msgpack_packer pck; + msgpack_sbuffer sbuf; + + + /* initialize buffers */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + kvs = flb_utils_split(payload, '&', -1 ); + if (kvs == NULL) { + goto split_error; + } + + keys = flb_calloc(mk_list_size(kvs), sizeof(char *)); + if (keys == NULL) { + goto keys_calloc_error; + } + + vals = flb_calloc(mk_list_size(kvs), sizeof(char *)); + if (vals == NULL) { + goto vals_calloc_error; + } + + mk_list_foreach(head, kvs) { + cur = mk_list_entry(head, struct flb_split_entry, _head); + if (cur->value[0] == '\n') { + start = &cur->value[1]; + } else { + start = cur->value; + } + sep = strchr(start, '='); + if (sep == NULL) { + vals[idx] = NULL; + continue; + } + *sep++ = '\0'; + + keys[idx] = flb_sds_create_len(start, strlen(start)); + vals[idx] = flb_sds_create_len(sep, strlen(sep)); + + flb_sds_trim(keys[idx]); + flb_sds_trim(vals[idx]); + idx++; + } + + msgpack_pack_map(&pck, mk_list_size(kvs)); + for (idx = 0; idx < mk_list_size(kvs); idx++) { + msgpack_pack_str(&pck, flb_sds_len(keys[idx])); + msgpack_pack_str_body(&pck, keys[idx], flb_sds_len(keys[idx])); + + if (sds_uri_decode(vals[idx]) != 0) { + goto decode_error; + } else { + msgpack_pack_str(&pck, flb_sds_len(vals[idx])); + msgpack_pack_str_body(&pck, vals[idx], strlen(vals[idx])); + } + } + + ret = process_pack(ctx, tag, sbuf.data, sbuf.size); + +decode_error: + for (idx = 0; idx < mk_list_size(kvs); idx++) { + if (keys[idx]) { + flb_sds_destroy(keys[idx]); + } + if (vals[idx]) { + flb_sds_destroy(vals[idx]); + } + } + flb_free(vals); +vals_calloc_error: + flb_free(keys); +keys_calloc_error: + flb_utils_split_free(kvs); +split_error: + msgpack_sbuffer_destroy(&sbuf); + return ret; +} + +static int process_payload(struct flb_http *ctx, struct http_conn *conn, + flb_sds_t tag, + struct mk_http_session *session, + struct mk_http_request *request) +{ + int type = -1; + struct mk_http_header *header; + + header = &session->parser.headers[MK_HEADER_CONTENT_TYPE]; + if (header->key.data == NULL) { + send_response(conn, 400, "error: header 'Content-Type' is not set\n"); + return -1; + } + + if (header->val.len == 16 && + strncasecmp(header->val.data, "application/json", 16) == 0) { + type = HTTP_CONTENT_JSON; + } + + if (header->val.len == 33 && + strncasecmp(header->val.data, "application/x-www-form-urlencoded", 33) == 0) { + type = HTTP_CONTENT_URLENCODED; + } + + if (type == -1) { + send_response(conn, 400, "error: invalid 'Content-Type'\n"); + return -1; + } + + if (request->data.len <= 0) { + send_response(conn, 400, "error: no payload found\n"); + return -1; + } + + if (type == HTTP_CONTENT_JSON) { + parse_payload_json(ctx, tag, request->data.data, request->data.len); + } else if (type == HTTP_CONTENT_URLENCODED) { + parse_payload_urlencoded(ctx, tag, request->data.data, request->data.len); + } + + return 0; +} + +static inline int mk_http_point_header(mk_ptr_t *h, + struct mk_http_parser *parser, int key) +{ + struct mk_http_header *header; + + header = &parser->headers[key]; + if (header->type == key) { + h->data = header->val.data; + h->len = header->val.len; + return 0; + } + else { + h->data = NULL; + h->len = -1; + } + + return -1; +} + +/* + * Handle an incoming request. It perform extra checks over the request, if + * everything is OK, it enqueue the incoming payload. + */ +int http_prot_handle(struct flb_http *ctx, struct http_conn *conn, + struct mk_http_session *session, + struct mk_http_request *request) +{ + int i; + int ret; + int len; + char *uri; + char *qs; + off_t diff; + flb_sds_t tag; + struct mk_http_header *header; + + if (request->uri.data[0] != '/') { + send_response(conn, 400, "error: invalid request\n"); + return -1; + } + + /* Decode URI */ + uri = mk_utils_url_decode(request->uri); + if (!uri) { + uri = mk_mem_alloc_z(request->uri.len + 1); + if (!uri) { + return -1; + } + memcpy(uri, request->uri.data, request->uri.len); + uri[request->uri.len] = '\0'; + } + + /* Try to match a query string so we can remove it */ + qs = strchr(uri, '?'); + if (qs) { + /* remove the query string part */ + diff = qs - uri; + uri[diff] = '\0'; + } + + /* Compose the query string using the URI */ + len = strlen(uri); + + if (len == 1) { + tag = NULL; /* use default tag */ + } + else { + tag = flb_sds_create_size(len); + if (!tag) { + mk_mem_free(uri); + return -1; + } + + /* New tag skipping the URI '/' */ + flb_sds_cat(tag, uri + 1, len - 1); + + /* Sanitize, only allow alphanum chars */ + for (i = 0; i < flb_sds_len(tag); i++) { + if (!isalnum(tag[i]) && tag[i] != '_' && tag[i] != '.') { + tag[i] = '_'; + } + } + } + + mk_mem_free(uri); + + /* Check if we have a Host header: Hostname ; port */ + mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); + + /* Header: Connection */ + mk_http_point_header(&request->connection, &session->parser, + MK_HEADER_CONNECTION); + + /* HTTP/1.1 needs Host header */ + if (!request->host.data && request->protocol == MK_HTTP_PROTOCOL_11) { + flb_sds_destroy(tag); + return -1; + } + + /* Should we close the session after this request ? */ + mk_http_keepalive_check(session, request, ctx->server); + + /* Content Length */ + header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; + if (header->type == MK_HEADER_CONTENT_LENGTH) { + request->_content_length.data = header->val.data; + request->_content_length.len = header->val.len; + } + else { + request->_content_length.data = NULL; + } + + if (request->method != MK_METHOD_POST) { + flb_sds_destroy(tag); + send_response(conn, 400, "error: invalid HTTP method\n"); + return -1; + } + + ret = process_payload(ctx, conn, tag, session, request); + flb_sds_destroy(tag); + send_response(conn, ctx->successful_response_code, NULL); + return ret; +} + +/* + * Handle an incoming request which has resulted in an http parser error. + */ +int http_prot_handle_error(struct flb_http *ctx, struct http_conn *conn, + struct mk_http_session *session, + struct mk_http_request *request) +{ + send_response(conn, 400, "error: invalid request\n"); + return -1; +} diff --git a/src/fluent-bit/plugins/in_http/http_prot.h b/src/fluent-bit/plugins/in_http/http_prot.h new file mode 100644 index 000000000..1cb603089 --- /dev/null +++ b/src/fluent-bit/plugins/in_http/http_prot.h @@ -0,0 +1,31 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2022 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_HTTP_PROT +#define FLB_IN_HTTP_PROT + +int http_prot_handle(struct flb_http *ctx, struct http_conn *conn, + struct mk_http_session *session, + struct mk_http_request *request); + +int http_prot_handle_error(struct flb_http *ctx, struct http_conn *conn, + struct mk_http_session *session, + struct mk_http_request *request); + +#endif |