Merging upstream version 1.44.3.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

7 files changed, 545 insertions, 0 deletions

diff --git a/fluent-bit/src/http_server/api/v2/CMakeLists.txt b/fluent-bit/src/http_server/api/v2/CMakeLists.txt
new file mode 100644
index 000000000..a9d590fbd
--- /dev/null
+++ b/fluent-bit/src/http_server/api/v2/CMakeLists.txt
@@ -0,0 +1,10 @@
+# api/v2
+set(src
+  metrics.c
+  reload.c
+  register.c
+  )
+
+include_directories(${MONKEY_INCLUDE_DIR})
+add_library(api-v2 STATIC ${src})
+target_link_libraries(api-v2 monkey-core-static fluent-bit-static)
diff --git a/fluent-bit/src/http_server/api/v2/metrics.c b/fluent-bit/src/http_server/api/v2/metrics.c
new file mode 100644
index 000000000..27513b7a4
--- /dev/null
+++ b/fluent-bit/src/http_server/api/v2/metrics.c
@@ -0,0 +1,259 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2022 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_pack.h>
+#include <fluent-bit/flb_input.h>
+#include <fluent-bit/flb_filter.h>
+#include <fluent-bit/flb_output.h>
+#include <fluent-bit/flb_sds.h>
+#include <fluent-bit/flb_version.h>
+#include <fluent-bit/flb_time.h>
+#include "metrics.h"
+
+#include <fluent-bit/flb_http_server.h>
+
+#define null_check(x) do { if (!x) { goto error; } else {sds = x;} } while (0)
+
+pthread_key_t hs_metrics_v2_key;
+
+static struct mk_list *hs_metrics_v2_key_create()
+{
+    struct mk_list *metrics_list = NULL;
+
+    metrics_list = flb_malloc(sizeof(struct mk_list));
+    if (metrics_list == NULL) {
+        flb_errno();
+        return NULL;
+    }
+    mk_list_init(metrics_list);
+    pthread_setspecific(hs_metrics_v2_key, metrics_list);
+
+    return metrics_list;
+}
+
+static void hs_metrics_v2_key_destroy(void *data)
+{
+    struct mk_list *metrics_list = (struct mk_list*) data;
+    struct mk_list *tmp;
+    struct mk_list *head;
+    struct flb_hs_buf *entry;
+
+    if (metrics_list == NULL) {
+        return;
+    }
+    mk_list_foreach_safe(head, tmp, metrics_list) {
+        entry = mk_list_entry(head, struct flb_hs_buf, _head);
+        if (entry != NULL) {
+            if (entry->raw_data != NULL) {
+                cmt_destroy(entry->raw_data);
+                entry->raw_data = NULL;
+            }
+            mk_list_del(&entry->_head);
+            flb_free(entry);
+        }
+    }
+
+    flb_free(metrics_list);
+}
+
+/* Return the newest metrics buffer */
+static struct flb_hs_buf *metrics_get_latest()
+{
+    struct flb_hs_buf *buf;
+    struct mk_list *metrics_list;
+
+    metrics_list = pthread_getspecific(hs_metrics_v2_key);
+    if (!metrics_list) {
+        return NULL;
+    }
+
+    if (mk_list_size(metrics_list) == 0) {
+        return NULL;
+    }
+
+    buf = mk_list_entry_last(metrics_list, struct flb_hs_buf, _head);
+    return buf;
+}
+
+/* Delete unused metrics, note that we only care about the latest node */
+static int cleanup_metrics()
+{
+    int c = 0;
+    struct mk_list *tmp;
+    struct mk_list *head;
+    struct mk_list *metrics_list;
+    struct flb_hs_buf *last;
+    struct flb_hs_buf *entry;
+
+    metrics_list = pthread_getspecific(hs_metrics_v2_key);
+    if (!metrics_list) {
+        return -1;
+    }
+
+    last = metrics_get_latest();
+    if (!last) {
+        return -1;
+    }
+
+    mk_list_foreach_safe(head, tmp, metrics_list) {
+        entry = mk_list_entry(head, struct flb_hs_buf, _head);
+        if (entry != last && entry->users == 0) {
+            mk_list_del(&entry->_head);
+            cmt_destroy(entry->raw_data);
+            flb_free(entry);
+            c++;
+        }
+    }
+
+    return c;
+}
+
+/*
+ * Callback invoked every time some metrics are received through a message queue channel.
+ * This function runs in a Monkey HTTP thread worker and it purpose is to take the metrics
+ * data and store it somewhere so then it can be available by the end-points upon
+ * HTTP client requests.
+ */
+static void cb_mq_metrics(mk_mq_t *queue, void *data, size_t size)
+{
+    int ret;
+    size_t off = 0;
+    struct cmt *cmt;
+    struct flb_hs_buf *buf;
+    struct mk_list *metrics_list = NULL;
+
+    metrics_list = pthread_getspecific(hs_metrics_v2_key);
+    if (!metrics_list) {
+        metrics_list = hs_metrics_v2_key_create();
+        if (metrics_list == NULL) {
+            return;
+        }
+    }
+
+    /* decode cmetrics */
+    ret = cmt_decode_msgpack_create(&cmt, data, size, &off);
+    if (ret != 0) {
+        return;
+    }
+
+    buf = flb_malloc(sizeof(struct flb_hs_buf));
+    if (!buf) {
+        flb_errno();
+        return;
+    }
+    buf->users = 0;
+    buf->data = NULL;
+
+    /* Store CMetrics context as the raw_data */
+    buf->raw_data = cmt;
+    buf->raw_size = 0;
+
+    mk_list_add(&buf->_head, metrics_list);
+    cleanup_metrics();
+}
+
+/* API: expose metrics in Prometheus format /api/v2/metrics/prometheus */
+static void cb_metrics_prometheus(mk_request_t *request, void *data)
+{
+    struct cmt *cmt;
+    struct flb_hs_buf *buf;
+    cfl_sds_t payload;
+
+    buf = metrics_get_latest();
+    if (!buf) {
+        mk_http_status(request, 404);
+        mk_http_done(request);
+        return;
+    }
+
+    cmt = (struct cmt *) buf->raw_data;
+
+    /* convert CMetrics to text */
+    payload = cmt_encode_prometheus_create(cmt, CMT_FALSE);
+    if (!payload) {
+        mk_http_status(request, 500);
+        mk_http_done(request);
+        return;
+    }
+
+    buf->users++;
+
+    mk_http_status(request, 200);
+    flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_PROMETHEUS);
+    mk_http_send(request, payload, cfl_sds_len(payload), NULL);
+    mk_http_done(request);
+
+    cmt_encode_prometheus_destroy(payload);
+
+    buf->users--;
+}
+
+/* API: expose built-in metrics /api/v1/metrics (JSON format) */
+static void cb_metrics(mk_request_t *request, void *data)
+{
+    struct cmt *cmt;
+    struct flb_hs_buf *buf;
+    cfl_sds_t payload;
+
+    buf = metrics_get_latest();
+    if (!buf) {
+        mk_http_status(request, 404);
+        mk_http_done(request);
+        return;
+    }
+
+    cmt = (struct cmt *) buf->raw_data;
+
+    /* convert CMetrics to text */
+    payload = cmt_encode_text_create(cmt);
+    if (!payload) {
+        mk_http_status(request, 500);
+        mk_http_done(request);
+        return;
+    }
+
+    buf->users++;
+
+    mk_http_status(request, 200);
+    mk_http_send(request, payload, cfl_sds_len(payload), NULL);
+    mk_http_done(request);
+
+    cmt_encode_text_destroy(payload);
+
+    buf->users--;
+}
+
+/* Perform registration */
+int api_v2_metrics(struct flb_hs *hs)
+{
+
+    pthread_key_create(&hs_metrics_v2_key, hs_metrics_v2_key_destroy);
+
+    /* Create a message queue */
+    hs->qid_metrics_v2 = mk_mq_create(hs->ctx, "/metrics_v2",
+                                      cb_mq_metrics, NULL);
+    /* HTTP end-points */
+    mk_vhost_handler(hs->ctx, hs->vid, "/api/v2/metrics/prometheus",
+                     cb_metrics_prometheus, hs);
+
+    mk_vhost_handler(hs->ctx, hs->vid, "/api/v2/metrics", cb_metrics, hs);
+
+    return 0;
+}
diff --git a/fluent-bit/src/http_server/api/v2/metrics.h b/fluent-bit/src/http_server/api/v2/metrics.h
new file mode 100644
index 000000000..5336865dd
--- /dev/null
+++ b/fluent-bit/src/http_server/api/v2/metrics.h
@@ -0,0 +1,28 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2022 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_HS_API_V2_METRICS_H
+#define FLB_HS_API_V2_METRICS_H
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_http_server.h>
+
+int api_v2_metrics(struct flb_hs *hs);
+
+#endif
diff --git a/fluent-bit/src/http_server/api/v2/register.c b/fluent-bit/src/http_server/api/v2/register.c
new file mode 100644
index 000000000..7a0956fbf
--- /dev/null
+++ b/fluent-bit/src/http_server/api/v2/register.c
@@ -0,0 +1,31 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2023 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_http_server.h>
+
+#include "metrics.h"
+#include "reload.h"
+
+int api_v2_registration(struct flb_hs *hs)
+{
+    api_v2_reload(hs);
+    api_v2_metrics(hs);
+    return 0;
+}
diff --git a/fluent-bit/src/http_server/api/v2/register.h b/fluent-bit/src/http_server/api/v2/register.h
new file mode 100644
index 000000000..da6d78f3a
--- /dev/null
+++ b/fluent-bit/src/http_server/api/v2/register.h
@@ -0,0 +1,28 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2023 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_API_V2_REG_H
+#define FLB_API_V2_REG_H
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_http_server.h>
+
+int api_v2_registration(struct flb_hs *hs);
+
+#endif
diff --git a/fluent-bit/src/http_server/api/v2/reload.c b/fluent-bit/src/http_server/api/v2/reload.c
new file mode 100644
index 000000000..3bb5159fa
--- /dev/null
+++ b/fluent-bit/src/http_server/api/v2/reload.c
@@ -0,0 +1,161 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2023 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_pack.h>
+#include <fluent-bit/flb_input.h>
+#include <fluent-bit/flb_filter.h>
+#include <fluent-bit/flb_output.h>
+#include <fluent-bit/flb_sds.h>
+#include <fluent-bit/flb_version.h>
+#include <fluent-bit/flb_time.h>
+#include <fluent-bit/flb_lib.h>
+#include <fluent-bit/flb_reload.h>
+#include "reload.h"
+
+#include <signal.h>
+
+#include <fluent-bit/flb_http_server.h>
+
+static void handle_reload_request(mk_request_t *request, struct flb_config *config)
+{
+    int ret;
+    flb_sds_t out_buf;
+    size_t out_size;
+    msgpack_packer mp_pck;
+    msgpack_sbuffer mp_sbuf;
+
+    /* initialize buffers */
+    msgpack_sbuffer_init(&mp_sbuf);
+    msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write);
+
+    msgpack_pack_map(&mp_pck, 2);
+    msgpack_pack_str(&mp_pck, 6);
+    msgpack_pack_str_body(&mp_pck, "reload", 6);
+
+#ifdef FLB_SYSTEM_WINDOWS
+    ret = -1;
+
+    msgpack_pack_str(&mp_pck, 11);
+    msgpack_pack_str_body(&mp_pck, "unsupported", 11);
+    msgpack_pack_str(&mp_pck, 6);
+    msgpack_pack_str_body(&mp_pck, "status", 6);
+    msgpack_pack_int64(&mp_pck, ret);
+#else
+    if (config->enable_hot_reload != FLB_TRUE) {
+        msgpack_pack_str(&mp_pck, 11);
+        msgpack_pack_str_body(&mp_pck, "not enabled", 11);
+        msgpack_pack_str(&mp_pck, 6);
+        msgpack_pack_str_body(&mp_pck, "status", 6);
+        msgpack_pack_int64(&mp_pck, -1);
+    }
+    else {
+        ret = kill(getpid(), SIGHUP);
+        if (ret != 0) {
+            mk_http_status(request, 500);
+            mk_http_done(request);
+            return;
+        }
+
+        msgpack_pack_str(&mp_pck, 4);
+        msgpack_pack_str_body(&mp_pck, "done", 4);
+        msgpack_pack_str(&mp_pck, 6);
+        msgpack_pack_str_body(&mp_pck, "status", 6);
+        msgpack_pack_int64(&mp_pck, ret);
+    }
+
+#endif
+
+    /* Export to JSON */
+    out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size);
+    msgpack_sbuffer_destroy(&mp_sbuf);
+    if (!out_buf) {
+        mk_http_status(request, 400);
+        mk_http_done(request);
+        return;
+    }
+    out_size = flb_sds_len(out_buf);
+
+    mk_http_status(request, 200);
+    flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON);
+    mk_http_send(request, out_buf, out_size, NULL);
+    mk_http_done(request);
+
+    flb_sds_destroy(out_buf);
+}
+
+static void handle_get_reload_status(mk_request_t *request, struct flb_config *config)
+{
+    flb_sds_t out_buf;
+    size_t out_size;
+    msgpack_packer mp_pck;
+    msgpack_sbuffer mp_sbuf;
+
+    /* initialize buffers */
+    msgpack_sbuffer_init(&mp_sbuf);
+    msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write);
+
+    msgpack_pack_map(&mp_pck, 1);
+    msgpack_pack_str(&mp_pck, 16);
+    msgpack_pack_str_body(&mp_pck, "hot_reload_count", 16);
+    msgpack_pack_int64(&mp_pck, config->hot_reloaded_count);
+
+    /* Export to JSON */
+    out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size);
+    msgpack_sbuffer_destroy(&mp_sbuf);
+    if (!out_buf) {
+        mk_http_status(request, 400);
+        mk_http_done(request);
+        return;
+    }
+    out_size = flb_sds_len(out_buf);
+
+    mk_http_status(request, 200);
+    flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON);
+    mk_http_send(request, out_buf, out_size, NULL);
+    mk_http_done(request);
+
+    flb_sds_destroy(out_buf);
+}
+
+static void cb_reload(mk_request_t *request, void *data)
+{
+    struct flb_hs *hs = data;
+    struct flb_config *config = hs->config;
+
+    if (request->method == MK_METHOD_POST ||
+        request->method == MK_METHOD_PUT) {
+        handle_reload_request(request, config);
+    }
+    else if (request->method == MK_METHOD_GET) {
+        handle_get_reload_status(request, config);
+    }
+    else {
+        mk_http_status(request, 400);
+        mk_http_done(request);
+    }
+}
+
+/* Perform registration */
+int api_v2_reload(struct flb_hs *hs)
+{
+    mk_vhost_handler(hs->ctx, hs->vid, "/api/v2/reload", cb_reload, hs);
+
+    return 0;
+}
diff --git a/fluent-bit/src/http_server/api/v2/reload.h b/fluent-bit/src/http_server/api/v2/reload.h
new file mode 100644
index 000000000..e64e867d6
--- /dev/null
+++ b/fluent-bit/src/http_server/api/v2/reload.h
@@ -0,0 +1,28 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2023 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_HS_API_V2_RELOAD_H
+#define FLB_HS_API_V2_RELOAD_H
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_http_server.h>
+
+int api_v2_reload(struct flb_hs *hs);
+
+#endif