path: root/fluent-bit/tests/internal/parser.c

/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

#include <fluent-bit/flb_info.h>
#include <fluent-bit/flb_mem.h>
#include <fluent-bit/flb_parser.h>
#include <fluent-bit/flb_error.h>
#include <fluent-bit/flb_str.h>
#include <fluent-bit/flb_sds.h>

#include <time.h>
#include <string.h>
#include "flb_tests_internal.h"

/* Parsers configuration */
#define JSON_PARSERS  FLB_TESTS_DATA_PATH "/data/parser/json.conf"
#define REGEX_PARSERS FLB_TESTS_DATA_PATH "/data/parser/regex.conf"

/* Templates */
#define JSON_FMT_01  "{\"key001\": 12345, \"key002\": 0.99, \"time\": \"%s\"}"
#define REGEX_FMT_01 "12345 0.99 %s"

#define isleap(y) ((y) % 4 == 0 && ((y) % 400 == 0 || (y) % 100 != 0))
#define year2sec(y) (isleap(y) ? 31622400 : 31536000)

/* Timezone */
struct tz_check {
    char *val;
    int diff;
};


struct tz_check tz_entries_ok[] = {
    {"+0000",       0},
    {"+00:00",      0},
    {"+00:59",   3540},
    {"-0600",  -21000},
    {"-06:00", -21000},
};

struct tz_check tz_entries_error[] = {
    {"0000",   0},
    {"+00:90", 0},
    {"--600",  0},
    {"-06:00", -21000},
};

/* Time Lookup */
struct time_check {
    char *parser_name;
    char *time_string;
    time_t epoch;
    double frac_seconds;

    /*
     * Some tests requires to set the UTC offset to the given time,
     * when this flag is enabled the parser is adjusted for each test.
     */
    int utc_offset;
};

struct time_check time_entries[] = {
    /*
     * Samples generated by scripts/dates.sh
     * =====================================
     * UTC => 07/17/2017 20:17:03 +0000, 1500322623
     * IST => 07/18/2017 01:47:03 +0530, 1500322623
     * JP  => 07/18/2017 05:17:03 +0900, 1500322623
     * ZW  => 07/17/2017 22:17:03 +0200, 1500322623
     */

    /*
     * No year tests (old Syslog)
     * ==========================
     */

    /* Fixed UTC Offset = -0600 (-21600) */
    {"no_year"     , "Feb 16 04:06:58"           , 1487239618, 0     , -21600},
    {"no_year_N"   , "Feb 16 04:06:58.1234"      , 1487239618, 0.1234, -21600},
    {"no_year_NC"  , "Feb 16 04:06:58,1234"      , 1487239618, 0.1234, -21600},

    /* No year with imezone specified */
    {"no_year_TZ"  , "Feb 16 04:06:58 -0600"     , 1487239618, 0     ,      0},
    {"no_year_N_TZ", "Feb 16 04:06:58.1234 -0600", 1487239618, 0.1234,      0},
    {"no_year_NC_TZ","Feb 16 04:06:58,1234 -0600", 1487239618, 0.1234,      0},

    /* Same date for different timezones, same timestamp */
    {"generic_TZ"   , "07/17/2017 20:17:03 +0000"  , 1500322623, 0,   0},
    {"generic_TZ"   , "07/18/2017 01:47:03 +0530"  , 1500322623, 0,   0},
    {"generic_TZ"   , "07/18/2017 05:17:03 +0900"  , 1500322623, 0,   0},
    {"generic_TZ"   , "07/17/2017 22:17:03 +0200"  , 1500322623, 0,   0},
    {"generic_N_TZ" , "07/17/2017 22:17:03.1 +0200", 1500322623, 0.1, 0},
    {"generic_NC_TZ", "07/17/2017 22:17:03,1 +0200",  1500322623, 0.1, 0},
#ifndef __APPLE__
    {"generic_TZ"   , "07/18/2017 01:47:03 +05:30"  , 1500322623, 0,   0},
    {"generic_N_TZ" , "07/17/2017 22:17:03.1 +02:00", 1500322623, 0.1, 0},
    {"generic_NC_TZ", "07/17/2017 22:17:03,1 +02:00", 1500322623, 0.1, 0},
    {"generic_NL_TZ", "07/17/2017 22:17:03:1 +02:00", 1500322623, 0.1, 0},
#endif
    /* Same date for different timezones, same timestamp w/ fixed UTC offset */
    {"generic"   , "07/18/2017 01:47:03"   , 1500322623, 0,   19800},
    {"generic"   , "07/18/2017 05:17:03"   , 1500322623, 0,   32400},
    {"generic"   , "07/17/2017 22:17:03"   , 1500322623, 0,    7200},
    {"generic_N" , "07/17/2017 22:17:03.1" , 1500322623, 0.1,  7200},
    {"generic_NC", "07/17/2017 22:17:03,1" , 1500322623, 0.1,  7200},

    /* default UTC: the following timings 'are' in UTC already */
    {"default_UTC"    , "07/17/2017 20:17:03"      , 1500322623, 0     , 0},
    {"default_UTC_Z"  , "07/17/2017 20:17:03Z"     , 1500322623, 0     , 0},
    {"default_UTC_N_Z", "07/17/2017 20:17:03.1234Z", 1500322623, 0.1234, 0},
    {"default_UTC_NC_Z","07/17/2017 20:17:03,1234Z", 1500322623, 0.1234, 0},

    {"apache_error", "Fri Jul 17 20:17:03.1234 2017", 1500322623, 0.1234, 0}
};


int flb_parser_json_do(struct flb_parser *parser,
                       char *buf, size_t length,
                       void **out_buf, size_t *out_size,
                       struct flb_time *out_time);

int flb_parser_regex_do(struct flb_parser *parser,
                        char *buf, size_t length,
                        void **out_buf, size_t *out_size,
                        struct flb_time *out_time);

/* Parse timezone string and get the offset */
void test_parser_tzone_offset()
{
    int i;
    int len;
    int ret;
    int diff;
    struct tz_check *t;

    /* Valid offsets */
    for (i = 0; i < sizeof(tz_entries_ok) / sizeof(struct tz_check); i++) {
        t = &tz_entries_ok[0];
        len = strlen(t->val);

        ret = flb_parser_tzone_offset(t->val, len, &diff);
        TEST_CHECK(ret == 0 && diff == t->diff);
    }

    /* Invalid offsets */
    for (i = 0; i < sizeof(tz_entries_error) / sizeof(struct tz_check); i++) {
        t = &tz_entries_error[0];
        len = strlen(t->val);

        ret = flb_parser_tzone_offset(t->val, len, &diff);
        TEST_CHECK(ret != 0);
    }
}

static void load_json_parsers(struct flb_config *config)
{
    int ret;

    ret = flb_parser_conf_file(JSON_PARSERS, config);
    TEST_CHECK(ret == 0);
}

static void load_regex_parsers(struct flb_config *config)
{
    int ret;

    ret = flb_parser_conf_file(REGEX_PARSERS, config);
    TEST_CHECK(ret == 0);
}

void test_parser_time_lookup()
{
    int i;
    int j;
    int len;
    int ret;
    int toff;
    int year_diff = 0;
    double ns;
    time_t now;
    time_t epoch;
    struct flb_parser *p;
    struct flb_config *config;
    struct time_check *t;
    struct flb_tm tm;

    config = flb_config_init();

    load_json_parsers(config);

    /* Iterate tests */
    now = time(NULL);
    for (i = 0; i < sizeof(time_entries) / sizeof(struct time_check); i++) {
        t = &time_entries[i];
        p = flb_parser_get(t->parser_name, config);
        TEST_CHECK(p != NULL);

        if (p == NULL) {
            continue;
        }

        /* Alter time offset if set */
        toff = 0;
        if (t->utc_offset != 0) {
            toff = p->time_offset;
            p->time_offset = t->utc_offset;
        }

        /* Adjust timestamp for parsers using no-year */
        if (p->time_with_year == FLB_FALSE) {
            time_t time_test = t->epoch;
            struct tm tm_now;
            struct tm tm_test;

            gmtime_r(&now, &tm_now);
            gmtime_r(&time_test, &tm_test);

            year_diff = 0;
            for (j = tm_test.tm_year; j < tm_now.tm_year; j++) {
                year_diff += year2sec(tm_test.tm_mon < 2 ? j : j + 1);
            }
        }
        else {
            year_diff = 0;
        }

        /* Lookup time */
        len = strlen(t->time_string);
        ret = flb_parser_time_lookup(t->time_string, len, now, p, &tm, &ns);
        if(!(TEST_CHECK(ret == 0))) {
            TEST_MSG("time lookup error: parser:'%s'  timestr:'%s'", t->parser_name, t->time_string);
            continue;
        }

        epoch = flb_parser_tm2time(&tm);
        epoch -= year_diff;
        TEST_CHECK(t->epoch == epoch);
        TEST_CHECK(t->frac_seconds == ns);

        if (t->utc_offset != 0) {
            p->time_offset = toff;
        }
    }

    flb_parser_exit(config);
    flb_config_exit(config);
}

/* Do time lookup using the JSON parser backend*/
void test_json_parser_time_lookup()
{
    int i;
    int j;
    int ret;
    int len;
    int toff;
    int year_diff = 0;
    time_t epoch;
    long nsec;
    char buf[512];
    void *out_buf;
    size_t out_size;
    struct flb_time out_time;
    struct flb_parser *p;
    struct flb_config *config;
    struct time_check *t;

    config = flb_config_init();

    /* Load parsers */
    load_json_parsers(config);

    for (i = 0; i < sizeof(time_entries) / sizeof(struct time_check); i++) {
        t = &time_entries[i];
        p = flb_parser_get(t->parser_name, config);
        TEST_CHECK(p != NULL);

        if (p == NULL) {
            continue;
        }

        /* Alter time offset if set */
        toff = 0;
        if (t->utc_offset != 0) {
            toff = p->time_offset;
            p->time_offset = t->utc_offset;
        }

        /* Adjust timestamp for parsers using no-year */
        if (p->time_with_year == FLB_FALSE) {
            time_t time_now = time(NULL);
            time_t time_test = t->epoch;
            struct tm tm_now;
            struct tm tm_test;

            gmtime_r(&time_now, &tm_now);
            gmtime_r(&time_test, &tm_test);

            year_diff = 0;
            for (j = tm_test.tm_year; j < tm_now.tm_year; j++) {
                year_diff += year2sec(tm_test.tm_mon < 2 ? j : j + 1);
            }
        }
        else {
            year_diff = 0;
        }

        /* Compose the string */
        len = snprintf(buf, sizeof(buf) - 1, JSON_FMT_01, t->time_string);

        /* Invoke the JSON parser backend */
        ret = flb_parser_json_do(p, buf, len, &out_buf, &out_size, &out_time);
        TEST_CHECK(ret != -1);
        TEST_CHECK(out_buf != NULL);

        /* Check time */
        epoch = t->epoch + year_diff;

        TEST_CHECK(out_time.tm.tv_sec == epoch);
        nsec = t->frac_seconds * 1000000000;
        TEST_CHECK(out_time.tm.tv_nsec == nsec);

        if (t->utc_offset != 0) {
            p->time_offset = toff;
        }

        flb_free(out_buf);
    }

    flb_parser_exit(config);
    flb_config_exit(config);
}

/* Do time lookup using the Regex parser backend*/
void test_regex_parser_time_lookup()
{
    int i;
    int j;
    int ret;
    int len;
    int toff;
    int year_diff = 0;
    time_t epoch;
    long nsec;
    char buf[512];
    void *out_buf;
    size_t out_size;
    struct flb_time out_time;
    struct flb_parser *p;
    struct flb_config *config;
    struct time_check *t;

    config = flb_config_init();

    /* Load parsers */
    load_regex_parsers(config);

    for (i = 0; i < sizeof(time_entries) / sizeof(struct time_check); i++) {
        t = &time_entries[i];
        p = flb_parser_get(t->parser_name, config);
        TEST_CHECK(p != NULL);

        if (p == NULL) {
            continue;
        }

        /* Alter time offset if set */
        toff = 0;
        if (t->utc_offset != 0) {
            toff = p->time_offset;
            p->time_offset = t->utc_offset;
        }

        /* Adjust timestamp for parsers using no-year */
        if (p->time_with_year == FLB_FALSE) {
            time_t time_now = time(NULL);
            time_t time_test = t->epoch;
            struct tm tm_now;
            struct tm tm_test;

            gmtime_r(&time_now, &tm_now);
            gmtime_r(&time_test, &tm_test);

            year_diff = 0;
            for (j = tm_test.tm_year; j < tm_now.tm_year; j++) {
                year_diff += year2sec(tm_test.tm_mon < 2 ? j : j + 1);
            }
        }
        else {
            year_diff = 0;
        }

        /* Compose the string */
        len = snprintf(buf, sizeof(buf) - 1, REGEX_FMT_01, t->time_string);

        /* Invoke the JSON parser backend */
        ret = flb_parser_regex_do(p, buf, len, &out_buf, &out_size, &out_time);
        TEST_CHECK(ret != -1);
        TEST_CHECK(out_buf != NULL);

        /* Adjust time without year */
        epoch = t->epoch + year_diff;

        /* Check time */
        TEST_CHECK(out_time.tm.tv_sec == epoch);
        nsec = t->frac_seconds * 1000000000;
        TEST_CHECK(out_time.tm.tv_nsec == nsec);

        if (t->utc_offset != 0) {
            p->time_offset = toff;
        }

        flb_free(out_buf);
    }

    flb_parser_exit(config);
    flb_config_exit(config);
}

static char *get_msgpack_map_key(void *buf, size_t buf_size, char *key) {
    int i;
    size_t off = 0;
    int key_size;
    char *ptr = NULL;
    msgpack_unpacked result;
    msgpack_object map;
    msgpack_object k;
    msgpack_object v;

    msgpack_unpacked_init(&result);
    msgpack_unpack_next(&result, buf, buf_size, &off);

    map = result.data;

    if (map.type != MSGPACK_OBJECT_MAP) {
        msgpack_unpacked_destroy(&result);
        return NULL;
    }

    key_size = strlen(key);


    /* printf("map_size: %d\n", map.via.map.size); */

    for (i = 0; i < map.via.map.size; i++) {
        k = map.via.map.ptr[i].key;
        v = map.via.map.ptr[i].val;
        if (k.type != MSGPACK_OBJECT_STR) {
            continue;
        }
        /* printf("key(%.*s)(%d) == (%s)(%d)\n",  k.via.str.size, k.via.str.ptr, k.via.str.size, key, key_size); */
        if (k.via.str.size == key_size && strncmp(key, (char *) k.via.str.ptr,  k.via.str.size) == 0) {
            ptr =  flb_strndup(v.via.str.ptr, v.via.str.size);
            break;
        }
    }

    msgpack_unpacked_destroy(&result);

    return ptr;
}

static int a_mysql_unquote_test(struct flb_parser *p, char *source, char *expected) {

    int ret;
    void *out_buf;
    size_t out_size;
    struct flb_time out_time;
    char *val001;


    ret = flb_parser_regex_do(p, source, strlen(source), &out_buf, &out_size, &out_time);

    TEST_CHECK(ret != -1);
    TEST_CHECK(out_buf != NULL);
    if(ret < 0 || out_buf == NULL) return -1;

    val001 = get_msgpack_map_key(out_buf, out_size, "key001");
    if(!TEST_CHECK(val001 != NULL)) {
        flb_free(out_buf);
        return -1;
    }

    TEST_CHECK_(strcmp(val001,expected) == 0,  "source(%s) expected(%s) got(%s)", source, expected, val001);
    flb_free(val001);
    flb_free(out_buf);

    return 1;
}


void test_mysql_unquoted()
{
    struct flb_parser *p;
    struct flb_config *config;

    config = flb_config_init();

    /* Load parsers */
    load_regex_parsers(config);

    p = flb_parser_get("mysql_quoted_stuff", config);
    TEST_CHECK(p != NULL);

    a_mysql_unquote_test(p,"2010-01-01 02:10:22,plain",    "plain");
    a_mysql_unquote_test(p,"2010-01-01 02:10:22,''",       "");
    a_mysql_unquote_test(p,"2010-01-01 02:10:22,'333'",    "333");
    a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\n'",    "\n");
    a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\r'",    "\r");
    a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\''",    "'");
    a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\\"'",   "\"");
    a_mysql_unquote_test(p,"2010-01-01 02:10:22,'\\\\'",   "\\");
    a_mysql_unquote_test(p,"2010-01-01 02:10:22,'abc\\nE\\\\'",   "abc\nE\\");

    flb_parser_exit(config);
    flb_config_exit(config);


}


TEST_LIST = {
    { "tzone_offset", test_parser_tzone_offset},
    { "time_lookup", test_parser_time_lookup},
    { "json_time_lookup", test_json_parser_time_lookup},
    { "regex_time_lookup", test_regex_parser_time_lookup},
    { "mysql_unquoted" , test_mysql_unquoted },
    { 0 }
};