Adding upstream version 1.44.3.upstream/1.44.3upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 491 insertions, 0 deletions

diff --git a/fluent-bit/tests/internal/parser_regex.c b/fluent-bit/tests/internal/parser_regex.c
new file mode 100644
index 00000000..2283231c
--- /dev/null
+++ b/fluent-bit/tests/internal/parser_regex.c
@@ -0,0 +1,491 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <fluent-bit/flb_config.h>
+#include <fluent-bit/flb_config_format.h>
+#include <fluent-bit/flb_parser.h>
+#include <fluent-bit/flb_parser_decoder.h>
+#include <msgpack.h>
+#include <float.h>
+#include <math.h>
+#include "flb_tests_internal.h"
+
+static int msgpack_strncmp(char* str, size_t str_len, msgpack_object obj)
+{
+    int ret = -1;
+
+    if (str == NULL) {
+        flb_error("str is NULL");
+        return -1;
+    }
+
+    switch (obj.type)  {
+    case MSGPACK_OBJECT_STR:
+        if (obj.via.str.size != str_len) {
+            return -1;
+        }
+        ret = strncmp(str, obj.via.str.ptr, str_len);
+        break;
+    case MSGPACK_OBJECT_POSITIVE_INTEGER:
+        {
+            unsigned long val = strtoul(str, NULL, 10);
+            if (val == (unsigned long)obj.via.u64) {
+                ret = 0;
+            }
+        }
+        break;
+    case MSGPACK_OBJECT_NEGATIVE_INTEGER:
+        {
+            long long val = strtoll(str, NULL, 10);
+            if (val == (unsigned long)obj.via.i64) {
+                ret = 0;
+            }
+        }
+        break;
+    case MSGPACK_OBJECT_FLOAT32:
+    case MSGPACK_OBJECT_FLOAT64:
+        {
+            double val = strtod(str, NULL);
+            if (fabs(val - obj.via.f64) < DBL_EPSILON) {
+                ret = 0;
+            }
+        }
+        break;
+    case MSGPACK_OBJECT_BOOLEAN:
+        if (obj.via.boolean) {
+            if (str_len != 4 /*true*/) {
+                return -1;
+            }
+            ret = strncasecmp(str, "true", 4);
+        }
+        else {
+            if (str_len != 5 /*false*/) {
+                return -1;
+            }
+            ret = strncasecmp(str, "false", 5);
+        }
+        break;
+    default:
+        flb_error("not supported");
+    }
+
+    return ret;
+}
+
+struct str_list {
+    size_t size;
+    char **lists;
+};
+
+static int compare_msgpack(void *msgpack_data, size_t msgpack_size, struct str_list *l)
+{
+    msgpack_unpacked result;
+    msgpack_object obj;
+    size_t off = 0;
+    int map_size;
+    int i_map;
+    int i_list;
+    int num = 0;
+
+    if (!TEST_CHECK(msgpack_data != NULL)) {
+        TEST_MSG("msgpack_data is NULL");
+        return -1;
+    }
+    else if (!TEST_CHECK(msgpack_size > 0)) {
+        TEST_MSG("msgpack_size is 0");
+        return -1;
+    }
+
+    msgpack_unpacked_init(&result);
+    while (msgpack_unpack_next(&result, msgpack_data, msgpack_size, &off) == MSGPACK_UNPACK_SUCCESS) {
+        obj = result.data;
+        /*
+        msgpack_object_print(stdout, obj);
+        */
+        if (!TEST_CHECK(obj.type == MSGPACK_OBJECT_MAP)) {
+            TEST_MSG("map error. type = %d", obj.type);
+            continue;
+        }
+        map_size = obj.via.map.size;
+        for (i_map=0; i_map<map_size; i_map++) {
+            if (!TEST_CHECK(obj.via.map.ptr[i_map].key.type == MSGPACK_OBJECT_STR)) {
+                TEST_MSG("key is not string. type =%d", obj.via.map.ptr[i_map].key.type);
+                continue;
+            }
+            for (i_list=0; i_list< l->size/2; i_list++)  {
+                if (msgpack_strncmp(l->lists[i_list*2], strlen(l->lists[i_list*2]),
+                                    obj.via.map.ptr[i_map].key) == 0 &&
+                    msgpack_strncmp(l->lists[i_list*2+1], strlen(l->lists[i_list*2+1]),
+                                    obj.via.map.ptr[i_map].val) == 0) {
+                    num++;
+                }
+            }
+        }
+    }
+    msgpack_unpacked_destroy(&result);
+    if (!TEST_CHECK(num == l->size/2)) {
+        msgpack_object_print(stdout, obj);
+        putchar('\n');
+        TEST_MSG("compare failed. matched_num=%d expect=%lu", num, l->size/2);
+        return -1;
+    }
+    return 0;
+}
+
+void test_basic()
+{
+    struct flb_parser *parser = NULL;
+    struct flb_config *config = NULL;
+    int ret = 0;
+    char *input = "text 100 1.23 true";
+    char *regex = "(?<str>[a-z]+) (?<int>\\d+) (?<double>\\d+\\.\\d+) (?<bool>true)";
+    void *out_buf = NULL;
+    size_t out_size = 0;
+    struct flb_time out_time;
+    char *expected_strs[] = {"str", "text", "int", "100", "double","1.23", "bool", "true"};
+    struct str_list expected = {
+                                .size = sizeof(expected_strs)/sizeof(char*),
+                                .lists = &expected_strs[0],
+    };
+
+    config = flb_config_init();
+    if(!TEST_CHECK(config != NULL)) {
+        TEST_MSG("flb_config_init failed");
+        exit(1);
+    }
+
+    parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, NULL, NULL, NULL,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               NULL, 0, NULL, config);
+    if (!TEST_CHECK(parser != NULL)) {
+        TEST_MSG("flb_parser_create failed");
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time);
+    if (!TEST_CHECK(ret != -1)) {
+        TEST_MSG("flb_parser_do failed");
+        flb_parser_destroy(parser);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = compare_msgpack(out_buf, out_size, &expected);
+    if (!TEST_CHECK(ret == 0)) {
+        TEST_MSG("compare failed");
+    }
+
+    flb_free(out_buf);
+    flb_parser_destroy(parser);
+    flb_config_exit(config);
+}
+
+void test_time_key()
+{
+    struct flb_parser *parser = NULL;
+    struct flb_config *config = NULL;
+    int ret = 0;
+    char *input = "text 100 1.23 true 2022-10-31T12:00:01.123";
+    char *regex = "(?<str>[a-z]+) (?<int>\\d+) (?<double>\\d+\\.\\d+) (?<bool>true) (?<time>.+)";
+    void *out_buf = NULL;
+    size_t out_size = 0;
+    struct flb_time out_time;
+    char *expected_strs[] = {"str", "text", "int", "100", "double","1.23", "bool", "true"};
+    struct str_list expected = {
+                                .size = sizeof(expected_strs)/sizeof(char*),
+                                .lists = &expected_strs[0],
+    };
+
+    out_time.tm.tv_sec = 0;
+    out_time.tm.tv_nsec = 0;
+
+
+    config = flb_config_init();
+    if(!TEST_CHECK(config != NULL)) {
+        TEST_MSG("flb_config_init failed");
+        exit(1);
+    }
+
+    parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               NULL, 0, NULL, config);
+    if (!TEST_CHECK(parser != NULL)) {
+        TEST_MSG("flb_parser_create failed");
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time);
+    if (!TEST_CHECK(ret != -1)) {
+        TEST_MSG("flb_parser_do failed");
+        flb_parser_destroy(parser);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = compare_msgpack(out_buf, out_size, &expected);
+    if (!TEST_CHECK(ret == 0)) {
+        TEST_MSG("compare failed");
+        flb_free(out_buf);
+        flb_parser_destroy(parser);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    if (!TEST_CHECK(out_time.tm.tv_sec == 1667217601 && out_time.tm.tv_nsec == 123000000)) {
+        TEST_MSG("timestamp error. sec  Got=%ld Expect=1667217601", out_time.tm.tv_sec);
+        TEST_MSG("timestamp error. nsec Got=%ld Expect=123000000", out_time.tm.tv_nsec);
+    }
+
+    flb_free(out_buf);
+    flb_parser_destroy(parser);
+    flb_config_exit(config);
+}
+
+void test_time_keep()
+{
+    struct flb_parser *parser = NULL;
+    struct flb_config *config = NULL;
+    int ret = 0;
+    char *input = "text 100 1.23 true 2022-10-31T12:00:01.123";
+    char *regex = "(?<str>[a-z]+) (?<int>\\d+) (?<double>\\d+\\.\\d+) (?<bool>true) (?<time>.+)";
+    void *out_buf = NULL;
+    size_t out_size = 0;
+    struct flb_time out_time;
+    char *expected_strs[] = {"str", "text", "int", "100", "double","1.23", "bool", "true", "time", "2022-10-31T12:00:01.123"};
+    struct str_list expected = {
+                                .size = sizeof(expected_strs)/sizeof(char*),
+                                .lists = &expected_strs[0],
+    };
+
+    out_time.tm.tv_sec = 0;
+    out_time.tm.tv_nsec = 0;
+
+
+    config = flb_config_init();
+    if(!TEST_CHECK(config != NULL)) {
+        TEST_MSG("flb_config_init failed");
+        exit(1);
+    }
+
+    parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
+                               FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE,
+                               NULL, 0, NULL, config);
+    if (!TEST_CHECK(parser != NULL)) {
+        TEST_MSG("flb_parser_create failed");
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time);
+    if (!TEST_CHECK(ret != -1)) {
+        TEST_MSG("flb_parser_do failed");
+        flb_parser_destroy(parser);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = compare_msgpack(out_buf, out_size, &expected);
+    if (!TEST_CHECK(ret == 0)) {
+        TEST_MSG("compare failed");
+        flb_free(out_buf);
+        flb_parser_destroy(parser);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    if (!TEST_CHECK(out_time.tm.tv_sec == 1667217601 && out_time.tm.tv_nsec == 123000000)) {
+        TEST_MSG("timestamp error. sec  Got=%ld Expect=1667217601", out_time.tm.tv_sec);
+        TEST_MSG("timestamp error. nsec Got=%ld Expect=123000000", out_time.tm.tv_nsec);
+    }
+
+    flb_free(out_buf);
+    flb_parser_destroy(parser);
+    flb_config_exit(config);
+}
+
+void test_types()
+{
+    struct flb_parser *parser = NULL;
+    struct flb_config *config = NULL;
+    int ret = 0;
+    char *input = "text 100 1.23 true";
+    char *regex = "(?<str>[a-z]+) (?<int>\\d+) (?<double>\\d+\\.\\d+) (?<bool>true)";
+    struct flb_parser_types *types = NULL;
+
+    void *out_buf = NULL;
+    size_t out_size = 0;
+    struct flb_time out_time;
+    char *expected_strs[] = {"str", "text", "int", "256" /*= 0x100 */, "double","1.23", "bool", "true"};
+    struct str_list expected = {
+                                .size = sizeof(expected_strs)/sizeof(char*),
+                                .lists = &expected_strs[0],
+    };
+
+    config = flb_config_init();
+    if(!TEST_CHECK(config != NULL)) {
+        TEST_MSG("flb_config_init failed");
+        exit(1);
+    }
+
+    /* Note: types will be released by flb_parser_destroy */
+    types = flb_malloc(sizeof(struct flb_parser_types));
+    if (!TEST_CHECK(types != NULL)) {
+        TEST_MSG("flb_malloc failed");
+        flb_config_exit(config);
+        exit(1);
+    }
+    types->key = flb_malloc(strlen("int")+1);
+    if (!TEST_CHECK(types->key != NULL)) {
+        TEST_MSG("flb_malloc failed");
+        flb_free(types);
+        flb_config_exit(config);
+        exit(1);
+    }
+    strcpy(types->key, "int");
+    types->key_len = 3;
+    types->type = FLB_PARSER_TYPE_HEX;
+
+    parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, NULL, NULL, NULL,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               types, 1, NULL, config);
+    if (!TEST_CHECK(parser != NULL)) {
+        TEST_MSG("flb_parser_create failed");
+        flb_free(types->key);
+        flb_free(types);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time);
+    if (!TEST_CHECK(ret != -1)) {
+        TEST_MSG("flb_parser_do failed");
+        flb_parser_destroy(parser);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = compare_msgpack(out_buf, out_size, &expected);
+    if (!TEST_CHECK(ret == 0)) {
+        TEST_MSG("compare failed");
+    }
+
+    flb_free(out_buf);
+    flb_parser_destroy(parser);
+    flb_config_exit(config);
+}
+
+void test_decode_field_json()
+{
+    struct flb_parser *parser = NULL;
+    struct flb_config *config = NULL;
+    struct cfl_variant *var = NULL;
+    int ret = 0;
+    char *input = "{\"str\":\"text\", \"int\":100, \"double\":1.23, \"bool\":true}";
+    char *regex = "(?<json_str>.+)";
+    struct flb_cf *cf = NULL;
+    struct flb_cf_section *section = NULL;
+    struct mk_list *decoder = NULL;
+
+    void *out_buf = NULL;
+    size_t out_size = 0;
+    struct flb_time out_time;
+    char *expected_strs[] = {"str", "text", "int", "100", "double","1.23", "bool", "true"};
+    struct str_list expected = {
+                                .size = sizeof(expected_strs)/sizeof(char*),
+                                .lists = &expected_strs[0],
+    };
+
+    config = flb_config_init();
+    if(!TEST_CHECK(config != NULL)) {
+        TEST_MSG("flb_config_init failed");
+        exit(1);
+    }
+    cf = flb_cf_create();
+    if (!TEST_CHECK(cf != NULL)) {
+        TEST_MSG("flb_cf_create failed");
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    section = flb_cf_section_create(cf, "TEST", 4);
+    if (!TEST_CHECK(section != NULL)) {
+        TEST_MSG("flb_cf_section_create failed");
+        flb_cf_destroy(cf);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+	var = flb_cf_section_property_add(cf, section->properties, "decode_field", 12, "json json_str", 13);
+	if(!TEST_CHECK(var != NULL)) {
+        TEST_MSG("flb_cf_section_property_add failed");
+        flb_cf_destroy(cf);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    decoder = flb_parser_decoder_list_create(section);
+    if (!TEST_CHECK(decoder != NULL)) {
+        TEST_MSG("flb_parser_decoder_list_create failed");
+        flb_cf_destroy(cf);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, NULL, NULL, NULL,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               NULL, 0, decoder, config);
+    if (!TEST_CHECK(parser != NULL)) {
+        TEST_MSG("flb_parser_create failed");
+        flb_cf_destroy(cf);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = flb_parser_do(parser, input, strlen(input), &out_buf, &out_size, &out_time);
+    if (!TEST_CHECK(ret != -1)) {
+        TEST_MSG("flb_parser_do failed");
+        flb_parser_destroy(parser);
+        flb_cf_destroy(cf);
+        flb_config_exit(config);
+        exit(1);
+    }
+
+    ret = compare_msgpack(out_buf, out_size, &expected);
+    if (!TEST_CHECK(ret == 0)) {
+        TEST_MSG("compare failed");
+    }
+
+    flb_free(out_buf);
+    flb_parser_destroy(parser);
+    flb_cf_destroy(cf);
+    flb_config_exit(config);
+}
+
+
+TEST_LIST = {
+    { "basic", test_basic},
+    { "time_key", test_time_key},
+    { "time_keep", test_time_keep},
+    { "types", test_types},
+    { "decode_field_json", test_decode_field_json},
+    { 0 }
+};