diff options
Diffstat (limited to '')
-rw-r--r-- | fluent-bit/tests/internal/parser.c | 528 |
1 files changed, 528 insertions, 0 deletions
diff --git a/fluent-bit/tests/internal/parser.c b/fluent-bit/tests/internal/parser.c new file mode 100644 index 00000000..6a560db7 --- /dev/null +++ b/fluent-bit/tests/internal/parser.c @@ -0,0 +1,528 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_mem.h> +#include <fluent-bit/flb_parser.h> +#include <fluent-bit/flb_error.h> +#include <fluent-bit/flb_str.h> +#include <fluent-bit/flb_sds.h> + +#include <time.h> +#include <string.h> +#include "flb_tests_internal.h" + +/* Parsers configuration */ +#define JSON_PARSERS FLB_TESTS_DATA_PATH "/data/parser/json.conf" +#define REGEX_PARSERS FLB_TESTS_DATA_PATH "/data/parser/regex.conf" + +/* Templates */ +#define JSON_FMT_01 "{\"key001\": 12345, \"key002\": 0.99, \"time\": \"%s\"}" +#define REGEX_FMT_01 "12345 0.99 %s" + +#define isleap(y) ((y) % 4 == 0 && ((y) % 400 == 0 || (y) % 100 != 0)) +#define year2sec(y) (isleap(y) ? 31622400 : 31536000) + +/* Timezone */ +struct tz_check { + char *val; + int diff; +}; + + +struct tz_check tz_entries_ok[] = { + {"+0000", 0}, + {"+00:00", 0}, + {"+00:59", 3540}, + {"-0600", -21000}, + {"-06:00", -21000}, +}; + +struct tz_check tz_entries_error[] = { + {"0000", 0}, + {"+00:90", 0}, + {"--600", 0}, + {"-06:00", -21000}, +}; + +/* Time Lookup */ +struct time_check { + char *parser_name; + char *time_string; + time_t epoch; + double frac_seconds; + + /* + * Some tests requires to set the UTC offset to the given time, + * when this flag is enabled the parser is adjusted for each test. + */ + int utc_offset; +}; + +struct time_check time_entries[] = { + /* + * Samples generated by scripts/dates.sh + * ===================================== + * UTC => 07/17/2017 20:17:03 +0000, 1500322623 + * IST => 07/18/2017 01:47:03 +0530, 1500322623 + * JP => 07/18/2017 05:17:03 +0900, 1500322623 + * ZW => 07/17/2017 22:17:03 +0200, 1500322623 + */ + + /* + * No year tests (old Syslog) + * ========================== + */ + + /* Fixed UTC Offset = -0600 (-21600) */ + {"no_year" , "Feb 16 04:06:58" , 1487239618, 0 , -21600}, + {"no_year_N" , "Feb 16 04:06:58.1234" , 1487239618, 0.1234, -21600}, + {"no_year_NC" , "Feb 16 04:06:58,1234" , 1487239618, 0.1234, -21600}, + + /* No year with imezone specified */ + {"no_year_TZ" , "Feb 16 04:06:58 -0600" , 1487239618, 0 , 0}, + {"no_year_N_TZ", "Feb 16 04:06:58.1234 -0600", 1487239618, 0.1234, 0}, + {"no_year_NC_TZ","Feb 16 04:06:58,1234 -0600", 1487239618, 0.1234, 0}, + + /* Same date for different timezones, same timestamp */ + {"generic_TZ" , "07/17/2017 20:17:03 +0000" , 1500322623, 0, 0}, + {"generic_TZ" , "07/18/2017 01:47:03 +0530" , 1500322623, 0, 0}, + {"generic_TZ" , "07/18/2017 05:17:03 +0900" , 1500322623, 0, 0}, + {"generic_TZ" , "07/17/2017 22:17:03 +0200" , 1500322623, 0, 0}, + {"generic_N_TZ" , "07/17/2017 22:17:03.1 +0200", 1500322623, 0.1, 0}, + {"generic_NC_TZ", "07/17/2017 22:17:03,1 +0200", 1500322623, 0.1, 0}, +#ifndef __APPLE__ + {"generic_TZ" , "07/18/2017 01:47:03 +05:30" , 1500322623, 0, 0}, + {"generic_N_TZ" , "07/17/2017 22:17:03.1 +02:00", 1500322623, 0.1, 0}, + {"generic_NC_TZ", "07/17/2017 22:17:03,1 +02:00", 1500322623, 0.1, 0}, + {"generic_NL_TZ", "07/17/2017 22:17:03:1 +02:00", 1500322623, 0.1, 0}, +#endif + /* Same date for different timezones, same timestamp w/ fixed UTC offset */ + {"generic" , "07/18/2017 01:47:03" , 1500322623, 0, 19800}, + {"generic" , "07/18/2017 05:17:03" , 1500322623, 0, 32400}, + {"generic" , "07/17/2017 22:17:03" , 1500322623, 0, 7200}, + {"generic_N" , "07/17/2017 22:17:03.1" , 1500322623, 0.1, 7200}, + {"generic_NC", "07/17/2017 22:17:03,1" , 1500322623, 0.1, 7200}, + + /* default UTC: the following timings 'are' in UTC already */ + {"default_UTC" , "07/17/2017 20:17:03" , 1500322623, 0 , 0}, + {"default_UTC_Z" , "07/17/2017 20:17:03Z" , 1500322623, 0 , 0}, + {"default_UTC_N_Z", "07/17/2017 20:17:03.1234Z", 1500322623, 0.1234, 0}, + {"default_UTC_NC_Z","07/17/2017 20:17:03,1234Z", 1500322623, 0.1234, 0}, + + {"apache_error", "Fri Jul 17 20:17:03.1234 2017", 1500322623, 0.1234, 0} +}; + + +int flb_parser_json_do(struct flb_parser *parser, + char *buf, size_t length, + void **out_buf, size_t *out_size, + struct flb_time *out_time); + +int flb_parser_regex_do(struct flb_parser *parser, + char *buf, size_t length, + void **out_buf, size_t *out_size, + struct flb_time *out_time); + +/* Parse timezone string and get the offset */ +void test_parser_tzone_offset() +{ + int i; + int len; + int ret; + int diff; + struct tz_check *t; + + /* Valid offsets */ + for (i = 0; i < sizeof(tz_entries_ok) / sizeof(struct tz_check); i++) { + t = &tz_entries_ok[0]; + len = strlen(t->val); + + ret = flb_parser_tzone_offset(t->val, len, &diff); + TEST_CHECK(ret == 0 && diff == t->diff); + } + + /* Invalid offsets */ + for (i = 0; i < sizeof(tz_entries_error) / sizeof(struct tz_check); i++) { + t = &tz_entries_error[0]; + len = strlen(t->val); + + ret = flb_parser_tzone_offset(t->val, len, &diff); + TEST_CHECK(ret != 0); + } +} + +static void load_json_parsers(struct flb_config *config) +{ + int ret; + + ret = flb_parser_conf_file(JSON_PARSERS, config); + TEST_CHECK(ret == 0); +} + +static void load_regex_parsers(struct flb_config *config) +{ + int ret; + + ret = flb_parser_conf_file(REGEX_PARSERS, config); + TEST_CHECK(ret == 0); +} + +void test_parser_time_lookup() +{ + int i; + int j; + int len; + int ret; + int toff; + int year_diff = 0; + double ns; + time_t now; + time_t epoch; + struct flb_parser *p; + struct flb_config *config; + struct time_check *t; + struct flb_tm tm; + + config = flb_config_init(); + + load_json_parsers(config); + + /* Iterate tests */ + now = time(NULL); + for (i = 0; i < sizeof(time_entries) / sizeof(struct time_check); i++) { + t = &time_entries[i]; + p = flb_parser_get(t->parser_name, config); + TEST_CHECK(p != NULL); + + if (p == NULL) { + continue; + } + + /* Alter time offset if set */ + toff = 0; + if (t->utc_offset != 0) { + toff = p->time_offset; + p->time_offset = t->utc_offset; + } + + /* Adjust timestamp for parsers using no-year */ + if (p->time_with_year == FLB_FALSE) { + time_t time_test = t->epoch; + struct tm tm_now; + struct tm tm_test; + + gmtime_r(&now, &tm_now); + gmtime_r(&time_test, &tm_test); + + year_diff = 0; + for (j = tm_test.tm_year; j < tm_now.tm_year; j++) { + year_diff += year2sec(tm_test.tm_mon < 2 ? j : j + 1); + } + } + else { + year_diff = 0; + } + + /* Lookup time */ + len = strlen(t->time_string); + ret = flb_parser_time_lookup(t->time_string, len, now, p, &tm, &ns); + if(!(TEST_CHECK(ret == 0))) { + TEST_MSG("time lookup error: parser:'%s' timestr:'%s'", t->parser_name, t->time_string); + continue; + } + + epoch = flb_parser_tm2time(&tm); + epoch -= year_diff; + TEST_CHECK(t->epoch == epoch); + TEST_CHECK(t->frac_seconds == ns); + + if (t->utc_offset != 0) { + p->time_offset = toff; + } + } + + flb_parser_exit(config); + flb_config_exit(config); +} + +/* Do time lookup using the JSON parser backend*/ +void test_json_parser_time_lookup() +{ + int i; + int j; + int ret; + int len; + int toff; + int year_diff = 0; + time_t epoch; + long nsec; + char buf[512]; + void *out_buf; + size_t out_size; + struct flb_time out_time; + struct flb_parser *p; + struct flb_config *config; + struct time_check *t; + + config = flb_config_init(); + + /* Load parsers */ + load_json_parsers(config); + + for (i = 0; i < sizeof(time_entries) / sizeof(struct time_check); i++) { + t = &time_entries[i]; + p = flb_parser_get(t->parser_name, config); + TEST_CHECK(p != NULL); + + if (p == NULL) { + continue; + } + + /* Alter time offset if set */ + toff = 0; + if (t->utc_offset != 0) { + toff = p->time_offset; + p->time_offset = t->utc_offset; + } + + /* Adjust timestamp for parsers using no-year */ + if (p->time_with_year == FLB_FALSE) { + time_t time_now = time(NULL); + time_t time_test = t->epoch; + struct tm tm_now; + struct tm tm_test; + + gmtime_r(&time_now, &tm_now); + gmtime_r(&time_test, &tm_test); + + year_diff = 0; + for (j = tm_test.tm_year; j < tm_now.tm_year; j++) { + year_diff += year2sec(tm_test.tm_mon < 2 ? j : j + 1); + } + } + else { + year_diff = 0; + } + + /* Compose the string */ + len = snprintf(buf, sizeof(buf) - 1, JSON_FMT_01, t->time_string); + + /* Invoke the JSON parser backend */ + ret = flb_parser_json_do(p, buf, len, &out_buf, &out_size, &out_time); + TEST_CHECK(ret != -1); + TEST_CHECK(out_buf != NULL); + + /* Check time */ + epoch = t->epoch + year_diff; + + TEST_CHECK(out_time.tm.tv_sec == epoch); + nsec = t->frac_seconds * 1000000000; + TEST_CHECK(out_time.tm.tv_nsec == nsec); + + if (t->utc_offset != 0) { + p->time_offset = toff; + } + + flb_free(out_buf); + } + + flb_parser_exit(config); + flb_config_exit(config); +} + +/* Do time lookup using the Regex parser backend*/ +void test_regex_parser_time_lookup() +{ + int i; + int j; + int ret; + int len; + int toff; + int year_diff = 0; + time_t epoch; + long nsec; + char buf[512]; + void *out_buf; + size_t out_size; + struct flb_time out_time; + struct flb_parser *p; + struct flb_config *config; + struct time_check *t; + + config = flb_config_init(); + + /* Load parsers */ + load_regex_parsers(config); + + for (i = 0; i < sizeof(time_entries) / sizeof(struct time_check); i++) { + t = &time_entries[i]; + p = flb_parser_get(t->parser_name, config); + TEST_CHECK(p != NULL); + + if (p == NULL) { + continue; + } + + /* Alter time offset if set */ + toff = 0; + if (t->utc_offset != 0) { + toff = p->time_offset; + p->time_offset = t->utc_offset; + } + + /* Adjust timestamp for parsers using no-year */ + if (p->time_with_year == FLB_FALSE) { + time_t time_now = time(NULL); + time_t time_test = t->epoch; + struct tm tm_now; + struct tm tm_test; + + gmtime_r(&time_now, &tm_now); + gmtime_r(&time_test, &tm_test); + + year_diff = 0; + for (j = tm_test.tm_year; j < tm_now.tm_year; j++) { + year_diff += year2sec(tm_test.tm_mon < 2 ? j : j + 1); + } + } + else { + year_diff = 0; + } + + /* Compose the string */ + len = snprintf(buf, sizeof(buf) - 1, REGEX_FMT_01, t->time_string); + + /* Invoke the JSON parser backend */ + ret = flb_parser_regex_do(p, buf, len, &out_buf, &out_size, &out_time); + TEST_CHECK(ret != -1); + TEST_CHECK(out_buf != NULL); + + /* Adjust time without year */ + epoch = t->epoch + year_diff; + + /* Check time */ + TEST_CHECK(out_time.tm.tv_sec == epoch); + nsec = t->frac_seconds * 1000000000; + TEST_CHECK(out_time.tm.tv_nsec == nsec); + + if (t->utc_offset != 0) { + p->time_offset = toff; + } + + flb_free(out_buf); + } + + flb_parser_exit(config); + flb_config_exit(config); +} + +static char *get_msgpack_map_key(void *buf, size_t buf_size, char *key) { + int i; + size_t off = 0; + int key_size; + char *ptr = NULL; + msgpack_unpacked result; + msgpack_object map; + msgpack_object k; + msgpack_object v; + + msgpack_unpacked_init(&result); + msgpack_unpack_next(&result, buf, buf_size, &off); + + map = result.data; + + if (map.type != MSGPACK_OBJECT_MAP) { + msgpack_unpacked_destroy(&result); + return NULL; + } + + key_size = strlen(key); + + + /* printf("map_size: %d\n", map.via.map.size); */ + + for (i = 0; i < map.via.map.size; i++) { + k = map.via.map.ptr[i].key; + v = map.via.map.ptr[i].val; + if (k.type != MSGPACK_OBJECT_STR) { + continue; + } + /* printf("key(%.*s)(%d) == (%s)(%d)\n", k.via.str.size, k.via.str.ptr, k.via.str.size, key, key_size); */ + if (k.via.str.size == key_size && strncmp(key, (char *) k.via.str.ptr, k.via.str.size) == 0) { + ptr = flb_strndup(v.via.str.ptr, v.via.str.size); + break; + } + } + + msgpack_unpacked_destroy(&result); + + return ptr; +} + +static int a_mysql_unquote_test(struct flb_parser *p, char *source, char *expected) { + + int ret; + void *out_buf; + size_t out_size; + struct flb_time out_time; + char *val001; + + + ret = flb_parser_regex_do(p, source, strlen(source), &out_buf, &out_size, &out_time); + + TEST_CHECK(ret != -1); + TEST_CHECK(out_buf != NULL); + if(ret < 0 || out_buf == NULL) return -1; + + val001 = get_msgpack_map_key(out_buf, out_size, "key001"); + if(!TEST_CHECK(val001 != NULL)) { + flb_free(out_buf); + return -1; + } + + TEST_CHECK_(strcmp(val001,expected) == 0, "source(%s) expected(%s) got(%s)", source, expected, val001); + flb_free(val001); + flb_free(out_buf); + + return 1; +} + + +void test_mysql_unquoted() +{ + struct flb_parser *p; + struct flb_config *config; + + config = flb_config_init(); + + /* Load parsers */ + load_regex_parsers(config); + + p = flb_parser_get("mysql_quoted_stuff", config); + TEST_CHECK(p != NULL); + + a_mysql_unquote_test(p,"2010-01-01 02:10:22,plain", "plain"); + a_mysql_unquote_test(p,"2010-01-01 02:10:22,''", ""); + a_mysql_unquote_test(p,"2010-01-01 02:10:22,'333'", "333"); + a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\n'", "\n"); + a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\r'", "\r"); + a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\''", "'"); + a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\\"'", "\""); + a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\\\'", "\\"); + a_mysql_unquote_test(p,"2010-01-01 02:10:22,'abc\\nE\\\\'", "abc\nE\\"); + + flb_parser_exit(config); + flb_config_exit(config); + + +} + + +TEST_LIST = { + { "tzone_offset", test_parser_tzone_offset}, + { "time_lookup", test_parser_time_lookup}, + { "json_time_lookup", test_json_parser_time_lookup}, + { "regex_time_lookup", test_regex_parser_time_lookup}, + { "mysql_unquoted" , test_mysql_unquoted }, + { 0 } +}; |