summaryrefslogtreecommitdiffstats
path: root/fluent-bit/tests/internal/parser.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--fluent-bit/tests/internal/parser.c528
1 files changed, 528 insertions, 0 deletions
diff --git a/fluent-bit/tests/internal/parser.c b/fluent-bit/tests/internal/parser.c
new file mode 100644
index 00000000..6a560db7
--- /dev/null
+++ b/fluent-bit/tests/internal/parser.c
@@ -0,0 +1,528 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_mem.h>
+#include <fluent-bit/flb_parser.h>
+#include <fluent-bit/flb_error.h>
+#include <fluent-bit/flb_str.h>
+#include <fluent-bit/flb_sds.h>
+
+#include <time.h>
+#include <string.h>
+#include "flb_tests_internal.h"
+
+/* Parsers configuration */
+#define JSON_PARSERS FLB_TESTS_DATA_PATH "/data/parser/json.conf"
+#define REGEX_PARSERS FLB_TESTS_DATA_PATH "/data/parser/regex.conf"
+
+/* Templates */
+#define JSON_FMT_01 "{\"key001\": 12345, \"key002\": 0.99, \"time\": \"%s\"}"
+#define REGEX_FMT_01 "12345 0.99 %s"
+
+#define isleap(y) ((y) % 4 == 0 && ((y) % 400 == 0 || (y) % 100 != 0))
+#define year2sec(y) (isleap(y) ? 31622400 : 31536000)
+
+/* Timezone */
+struct tz_check {
+ char *val;
+ int diff;
+};
+
+
+struct tz_check tz_entries_ok[] = {
+ {"+0000", 0},
+ {"+00:00", 0},
+ {"+00:59", 3540},
+ {"-0600", -21000},
+ {"-06:00", -21000},
+};
+
+struct tz_check tz_entries_error[] = {
+ {"0000", 0},
+ {"+00:90", 0},
+ {"--600", 0},
+ {"-06:00", -21000},
+};
+
+/* Time Lookup */
+struct time_check {
+ char *parser_name;
+ char *time_string;
+ time_t epoch;
+ double frac_seconds;
+
+ /*
+ * Some tests requires to set the UTC offset to the given time,
+ * when this flag is enabled the parser is adjusted for each test.
+ */
+ int utc_offset;
+};
+
+struct time_check time_entries[] = {
+ /*
+ * Samples generated by scripts/dates.sh
+ * =====================================
+ * UTC => 07/17/2017 20:17:03 +0000, 1500322623
+ * IST => 07/18/2017 01:47:03 +0530, 1500322623
+ * JP => 07/18/2017 05:17:03 +0900, 1500322623
+ * ZW => 07/17/2017 22:17:03 +0200, 1500322623
+ */
+
+ /*
+ * No year tests (old Syslog)
+ * ==========================
+ */
+
+ /* Fixed UTC Offset = -0600 (-21600) */
+ {"no_year" , "Feb 16 04:06:58" , 1487239618, 0 , -21600},
+ {"no_year_N" , "Feb 16 04:06:58.1234" , 1487239618, 0.1234, -21600},
+ {"no_year_NC" , "Feb 16 04:06:58,1234" , 1487239618, 0.1234, -21600},
+
+ /* No year with imezone specified */
+ {"no_year_TZ" , "Feb 16 04:06:58 -0600" , 1487239618, 0 , 0},
+ {"no_year_N_TZ", "Feb 16 04:06:58.1234 -0600", 1487239618, 0.1234, 0},
+ {"no_year_NC_TZ","Feb 16 04:06:58,1234 -0600", 1487239618, 0.1234, 0},
+
+ /* Same date for different timezones, same timestamp */
+ {"generic_TZ" , "07/17/2017 20:17:03 +0000" , 1500322623, 0, 0},
+ {"generic_TZ" , "07/18/2017 01:47:03 +0530" , 1500322623, 0, 0},
+ {"generic_TZ" , "07/18/2017 05:17:03 +0900" , 1500322623, 0, 0},
+ {"generic_TZ" , "07/17/2017 22:17:03 +0200" , 1500322623, 0, 0},
+ {"generic_N_TZ" , "07/17/2017 22:17:03.1 +0200", 1500322623, 0.1, 0},
+ {"generic_NC_TZ", "07/17/2017 22:17:03,1 +0200", 1500322623, 0.1, 0},
+#ifndef __APPLE__
+ {"generic_TZ" , "07/18/2017 01:47:03 +05:30" , 1500322623, 0, 0},
+ {"generic_N_TZ" , "07/17/2017 22:17:03.1 +02:00", 1500322623, 0.1, 0},
+ {"generic_NC_TZ", "07/17/2017 22:17:03,1 +02:00", 1500322623, 0.1, 0},
+ {"generic_NL_TZ", "07/17/2017 22:17:03:1 +02:00", 1500322623, 0.1, 0},
+#endif
+ /* Same date for different timezones, same timestamp w/ fixed UTC offset */
+ {"generic" , "07/18/2017 01:47:03" , 1500322623, 0, 19800},
+ {"generic" , "07/18/2017 05:17:03" , 1500322623, 0, 32400},
+ {"generic" , "07/17/2017 22:17:03" , 1500322623, 0, 7200},
+ {"generic_N" , "07/17/2017 22:17:03.1" , 1500322623, 0.1, 7200},
+ {"generic_NC", "07/17/2017 22:17:03,1" , 1500322623, 0.1, 7200},
+
+ /* default UTC: the following timings 'are' in UTC already */
+ {"default_UTC" , "07/17/2017 20:17:03" , 1500322623, 0 , 0},
+ {"default_UTC_Z" , "07/17/2017 20:17:03Z" , 1500322623, 0 , 0},
+ {"default_UTC_N_Z", "07/17/2017 20:17:03.1234Z", 1500322623, 0.1234, 0},
+ {"default_UTC_NC_Z","07/17/2017 20:17:03,1234Z", 1500322623, 0.1234, 0},
+
+ {"apache_error", "Fri Jul 17 20:17:03.1234 2017", 1500322623, 0.1234, 0}
+};
+
+
+int flb_parser_json_do(struct flb_parser *parser,
+ char *buf, size_t length,
+ void **out_buf, size_t *out_size,
+ struct flb_time *out_time);
+
+int flb_parser_regex_do(struct flb_parser *parser,
+ char *buf, size_t length,
+ void **out_buf, size_t *out_size,
+ struct flb_time *out_time);
+
+/* Parse timezone string and get the offset */
+void test_parser_tzone_offset()
+{
+ int i;
+ int len;
+ int ret;
+ int diff;
+ struct tz_check *t;
+
+ /* Valid offsets */
+ for (i = 0; i < sizeof(tz_entries_ok) / sizeof(struct tz_check); i++) {
+ t = &tz_entries_ok[0];
+ len = strlen(t->val);
+
+ ret = flb_parser_tzone_offset(t->val, len, &diff);
+ TEST_CHECK(ret == 0 && diff == t->diff);
+ }
+
+ /* Invalid offsets */
+ for (i = 0; i < sizeof(tz_entries_error) / sizeof(struct tz_check); i++) {
+ t = &tz_entries_error[0];
+ len = strlen(t->val);
+
+ ret = flb_parser_tzone_offset(t->val, len, &diff);
+ TEST_CHECK(ret != 0);
+ }
+}
+
+static void load_json_parsers(struct flb_config *config)
+{
+ int ret;
+
+ ret = flb_parser_conf_file(JSON_PARSERS, config);
+ TEST_CHECK(ret == 0);
+}
+
+static void load_regex_parsers(struct flb_config *config)
+{
+ int ret;
+
+ ret = flb_parser_conf_file(REGEX_PARSERS, config);
+ TEST_CHECK(ret == 0);
+}
+
+void test_parser_time_lookup()
+{
+ int i;
+ int j;
+ int len;
+ int ret;
+ int toff;
+ int year_diff = 0;
+ double ns;
+ time_t now;
+ time_t epoch;
+ struct flb_parser *p;
+ struct flb_config *config;
+ struct time_check *t;
+ struct flb_tm tm;
+
+ config = flb_config_init();
+
+ load_json_parsers(config);
+
+ /* Iterate tests */
+ now = time(NULL);
+ for (i = 0; i < sizeof(time_entries) / sizeof(struct time_check); i++) {
+ t = &time_entries[i];
+ p = flb_parser_get(t->parser_name, config);
+ TEST_CHECK(p != NULL);
+
+ if (p == NULL) {
+ continue;
+ }
+
+ /* Alter time offset if set */
+ toff = 0;
+ if (t->utc_offset != 0) {
+ toff = p->time_offset;
+ p->time_offset = t->utc_offset;
+ }
+
+ /* Adjust timestamp for parsers using no-year */
+ if (p->time_with_year == FLB_FALSE) {
+ time_t time_test = t->epoch;
+ struct tm tm_now;
+ struct tm tm_test;
+
+ gmtime_r(&now, &tm_now);
+ gmtime_r(&time_test, &tm_test);
+
+ year_diff = 0;
+ for (j = tm_test.tm_year; j < tm_now.tm_year; j++) {
+ year_diff += year2sec(tm_test.tm_mon < 2 ? j : j + 1);
+ }
+ }
+ else {
+ year_diff = 0;
+ }
+
+ /* Lookup time */
+ len = strlen(t->time_string);
+ ret = flb_parser_time_lookup(t->time_string, len, now, p, &tm, &ns);
+ if(!(TEST_CHECK(ret == 0))) {
+ TEST_MSG("time lookup error: parser:'%s' timestr:'%s'", t->parser_name, t->time_string);
+ continue;
+ }
+
+ epoch = flb_parser_tm2time(&tm);
+ epoch -= year_diff;
+ TEST_CHECK(t->epoch == epoch);
+ TEST_CHECK(t->frac_seconds == ns);
+
+ if (t->utc_offset != 0) {
+ p->time_offset = toff;
+ }
+ }
+
+ flb_parser_exit(config);
+ flb_config_exit(config);
+}
+
+/* Do time lookup using the JSON parser backend*/
+void test_json_parser_time_lookup()
+{
+ int i;
+ int j;
+ int ret;
+ int len;
+ int toff;
+ int year_diff = 0;
+ time_t epoch;
+ long nsec;
+ char buf[512];
+ void *out_buf;
+ size_t out_size;
+ struct flb_time out_time;
+ struct flb_parser *p;
+ struct flb_config *config;
+ struct time_check *t;
+
+ config = flb_config_init();
+
+ /* Load parsers */
+ load_json_parsers(config);
+
+ for (i = 0; i < sizeof(time_entries) / sizeof(struct time_check); i++) {
+ t = &time_entries[i];
+ p = flb_parser_get(t->parser_name, config);
+ TEST_CHECK(p != NULL);
+
+ if (p == NULL) {
+ continue;
+ }
+
+ /* Alter time offset if set */
+ toff = 0;
+ if (t->utc_offset != 0) {
+ toff = p->time_offset;
+ p->time_offset = t->utc_offset;
+ }
+
+ /* Adjust timestamp for parsers using no-year */
+ if (p->time_with_year == FLB_FALSE) {
+ time_t time_now = time(NULL);
+ time_t time_test = t->epoch;
+ struct tm tm_now;
+ struct tm tm_test;
+
+ gmtime_r(&time_now, &tm_now);
+ gmtime_r(&time_test, &tm_test);
+
+ year_diff = 0;
+ for (j = tm_test.tm_year; j < tm_now.tm_year; j++) {
+ year_diff += year2sec(tm_test.tm_mon < 2 ? j : j + 1);
+ }
+ }
+ else {
+ year_diff = 0;
+ }
+
+ /* Compose the string */
+ len = snprintf(buf, sizeof(buf) - 1, JSON_FMT_01, t->time_string);
+
+ /* Invoke the JSON parser backend */
+ ret = flb_parser_json_do(p, buf, len, &out_buf, &out_size, &out_time);
+ TEST_CHECK(ret != -1);
+ TEST_CHECK(out_buf != NULL);
+
+ /* Check time */
+ epoch = t->epoch + year_diff;
+
+ TEST_CHECK(out_time.tm.tv_sec == epoch);
+ nsec = t->frac_seconds * 1000000000;
+ TEST_CHECK(out_time.tm.tv_nsec == nsec);
+
+ if (t->utc_offset != 0) {
+ p->time_offset = toff;
+ }
+
+ flb_free(out_buf);
+ }
+
+ flb_parser_exit(config);
+ flb_config_exit(config);
+}
+
+/* Do time lookup using the Regex parser backend*/
+void test_regex_parser_time_lookup()
+{
+ int i;
+ int j;
+ int ret;
+ int len;
+ int toff;
+ int year_diff = 0;
+ time_t epoch;
+ long nsec;
+ char buf[512];
+ void *out_buf;
+ size_t out_size;
+ struct flb_time out_time;
+ struct flb_parser *p;
+ struct flb_config *config;
+ struct time_check *t;
+
+ config = flb_config_init();
+
+ /* Load parsers */
+ load_regex_parsers(config);
+
+ for (i = 0; i < sizeof(time_entries) / sizeof(struct time_check); i++) {
+ t = &time_entries[i];
+ p = flb_parser_get(t->parser_name, config);
+ TEST_CHECK(p != NULL);
+
+ if (p == NULL) {
+ continue;
+ }
+
+ /* Alter time offset if set */
+ toff = 0;
+ if (t->utc_offset != 0) {
+ toff = p->time_offset;
+ p->time_offset = t->utc_offset;
+ }
+
+ /* Adjust timestamp for parsers using no-year */
+ if (p->time_with_year == FLB_FALSE) {
+ time_t time_now = time(NULL);
+ time_t time_test = t->epoch;
+ struct tm tm_now;
+ struct tm tm_test;
+
+ gmtime_r(&time_now, &tm_now);
+ gmtime_r(&time_test, &tm_test);
+
+ year_diff = 0;
+ for (j = tm_test.tm_year; j < tm_now.tm_year; j++) {
+ year_diff += year2sec(tm_test.tm_mon < 2 ? j : j + 1);
+ }
+ }
+ else {
+ year_diff = 0;
+ }
+
+ /* Compose the string */
+ len = snprintf(buf, sizeof(buf) - 1, REGEX_FMT_01, t->time_string);
+
+ /* Invoke the JSON parser backend */
+ ret = flb_parser_regex_do(p, buf, len, &out_buf, &out_size, &out_time);
+ TEST_CHECK(ret != -1);
+ TEST_CHECK(out_buf != NULL);
+
+ /* Adjust time without year */
+ epoch = t->epoch + year_diff;
+
+ /* Check time */
+ TEST_CHECK(out_time.tm.tv_sec == epoch);
+ nsec = t->frac_seconds * 1000000000;
+ TEST_CHECK(out_time.tm.tv_nsec == nsec);
+
+ if (t->utc_offset != 0) {
+ p->time_offset = toff;
+ }
+
+ flb_free(out_buf);
+ }
+
+ flb_parser_exit(config);
+ flb_config_exit(config);
+}
+
+static char *get_msgpack_map_key(void *buf, size_t buf_size, char *key) {
+ int i;
+ size_t off = 0;
+ int key_size;
+ char *ptr = NULL;
+ msgpack_unpacked result;
+ msgpack_object map;
+ msgpack_object k;
+ msgpack_object v;
+
+ msgpack_unpacked_init(&result);
+ msgpack_unpack_next(&result, buf, buf_size, &off);
+
+ map = result.data;
+
+ if (map.type != MSGPACK_OBJECT_MAP) {
+ msgpack_unpacked_destroy(&result);
+ return NULL;
+ }
+
+ key_size = strlen(key);
+
+
+ /* printf("map_size: %d\n", map.via.map.size); */
+
+ for (i = 0; i < map.via.map.size; i++) {
+ k = map.via.map.ptr[i].key;
+ v = map.via.map.ptr[i].val;
+ if (k.type != MSGPACK_OBJECT_STR) {
+ continue;
+ }
+ /* printf("key(%.*s)(%d) == (%s)(%d)\n", k.via.str.size, k.via.str.ptr, k.via.str.size, key, key_size); */
+ if (k.via.str.size == key_size && strncmp(key, (char *) k.via.str.ptr, k.via.str.size) == 0) {
+ ptr = flb_strndup(v.via.str.ptr, v.via.str.size);
+ break;
+ }
+ }
+
+ msgpack_unpacked_destroy(&result);
+
+ return ptr;
+}
+
+static int a_mysql_unquote_test(struct flb_parser *p, char *source, char *expected) {
+
+ int ret;
+ void *out_buf;
+ size_t out_size;
+ struct flb_time out_time;
+ char *val001;
+
+
+ ret = flb_parser_regex_do(p, source, strlen(source), &out_buf, &out_size, &out_time);
+
+ TEST_CHECK(ret != -1);
+ TEST_CHECK(out_buf != NULL);
+ if(ret < 0 || out_buf == NULL) return -1;
+
+ val001 = get_msgpack_map_key(out_buf, out_size, "key001");
+ if(!TEST_CHECK(val001 != NULL)) {
+ flb_free(out_buf);
+ return -1;
+ }
+
+ TEST_CHECK_(strcmp(val001,expected) == 0, "source(%s) expected(%s) got(%s)", source, expected, val001);
+ flb_free(val001);
+ flb_free(out_buf);
+
+ return 1;
+}
+
+
+void test_mysql_unquoted()
+{
+ struct flb_parser *p;
+ struct flb_config *config;
+
+ config = flb_config_init();
+
+ /* Load parsers */
+ load_regex_parsers(config);
+
+ p = flb_parser_get("mysql_quoted_stuff", config);
+ TEST_CHECK(p != NULL);
+
+ a_mysql_unquote_test(p,"2010-01-01 02:10:22,plain", "plain");
+ a_mysql_unquote_test(p,"2010-01-01 02:10:22,''", "");
+ a_mysql_unquote_test(p,"2010-01-01 02:10:22,'333'", "333");
+ a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\n'", "\n");
+ a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\r'", "\r");
+ a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\''", "'");
+ a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\\"'", "\"");
+ a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\\\'", "\\");
+ a_mysql_unquote_test(p,"2010-01-01 02:10:22,'abc\\nE\\\\'", "abc\nE\\");
+
+ flb_parser_exit(config);
+ flb_config_exit(config);
+
+
+}
+
+
+TEST_LIST = {
+ { "tzone_offset", test_parser_tzone_offset},
+ { "time_lookup", test_parser_time_lookup},
+ { "json_time_lookup", test_json_parser_time_lookup},
+ { "regex_time_lookup", test_regex_parser_time_lookup},
+ { "mysql_unquoted" , test_mysql_unquoted },
+ { 0 }
+};
generated by cgit v1.2.3 (git 2.39.1) at 2024-09-12 08:16:00 +0000