diff options
Diffstat (limited to '')
| -rw-r--r-- | fluent-bit/tests/internal/signv4.c | 666 |
1 files changed, 666 insertions, 0 deletions
diff --git a/fluent-bit/tests/internal/signv4.c b/fluent-bit/tests/internal/signv4.c new file mode 100644 index 00000000..7c691bc3 --- /dev/null +++ b/fluent-bit/tests/internal/signv4.c @@ -0,0 +1,666 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2020 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * AWS Signv4 documentation + * ======================== + * + * https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + * + * AWS Signv4 Test Suite + * ===================== + * + * AWS provides a test suite that can be used to validate certain requests type and + * expected signatures. The following unit test file, uses the suite provided and + * provides certain wrappers to validate expected results. + */ + +#include <fluent-bit/flb_info.h> +#include <fluent-bit/flb_mem.h> +#include <fluent-bit/flb_kv.h> +#include <fluent-bit/flb_http_client.h> +#include <fluent-bit/flb_upstream.h> +#include <fluent-bit/flb_signv4.h> +#include <fluent-bit/flb_aws_credentials.h> +#include <monkey/mk_core.h> + +#include "flb_tests_internal.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> +#include <stdlib.h> + +/* Test suite entry point */ +#define AWS_SUITE FLB_TESTS_DATA_PATH "data/signv4/aws-sig-v4-test-suite/" + +/* Credentials Environment Variables */ +#define AWS_ACCESS_KEY_ID "AWS_ACCESS_KEY_ID" +#define AWS_SECRET_ACCESS_KEY "AWS_SECRET_ACCESS_KEY" +#define AWS_SESSION_TOKEN "AWS_SESSION_TOKEN" + +struct request { + int method_i; + flb_sds_t method; + flb_sds_t uri; + flb_sds_t uri_full; + flb_sds_t query_string; + flb_sds_t payload; + struct mk_list headers; +}; + +struct aws_test { + flb_sds_t name; /* test name */ + flb_sds_t authz; + flb_sds_t creq; + flb_sds_t req; + flb_sds_t sreq; + flb_sds_t sts; + struct request *r; + struct flb_http_client *c; + struct mk_list _head; +}; + +static struct request *http_request_create(char *request) +{ + int len; + char *sep; + char *start; + char *end; + char *p; + char *br = NULL; + flb_sds_t tmp; + flb_sds_t key; + flb_sds_t val; + flb_sds_t payload = NULL; + struct flb_kv *kv = NULL; + struct request *req; + + req = flb_calloc(1, sizeof(struct request)); + if (!req) { + flb_errno(); + return NULL; + } + mk_list_init(&req->headers); + + /* method */ + p = strchr(request, ' '); + req->method = flb_sds_create_size(10); + if (!req->method) { + flb_free(req); + return NULL; + } + + tmp = flb_sds_copy(req->method, request, p - request); + if (!tmp) { + flb_sds_destroy(req->method); + flb_free(req); + return NULL; + } + req->method = tmp; + + if (strcmp(req->method, "GET") == 0) { + req->method_i = FLB_HTTP_GET; + } + else if (strcmp(req->method, "POST") == 0) { + req->method_i = FLB_HTTP_POST; + } + + /* URI */ + start = p + 1; + p = strchr(start, '\n'); + if (!p) { + flb_sds_destroy(req->method); + flb_free(req); + return NULL; + } + p--; + if ((p - 8) <= start) { + return NULL; + } + end = (p - 8); + + len = end - start; + req->uri = flb_sds_create_size(len); + tmp = flb_sds_copy(req->uri, start, len); + if (!tmp) { + flb_sds_destroy(req->method); + flb_sds_destroy(req->uri); + flb_free(req); + return NULL; + } + + req->uri = tmp; + req->uri_full = flb_sds_create(req->uri); + + /* Query string: it might be inside the URI */ + start = req->uri; + p = strchr(start, '?'); + if (p) { + flb_sds_len_set(req->uri, (p - req->uri)); + len = flb_sds_len(req->uri) - (p - req->uri); + *p = '\0'; /* terminate the string */ + start = p + 1; + req->query_string = flb_sds_create_len(start, len); + } + + /* Headers, everything after the first LF (\n) */ + p = strchr(request, '\n'); + p++; + + len = strlen(request) - (p - request); + start = p; + do { + /* HTTP line folding (multi line header) */ + if ((*start == ' ' || *start == '\t') && kv) { + key = flb_sds_create(kv->key); + sep = start + 1; + } + else { + /* key */ + sep = strchr(start, ':'); + if (!sep) { + break; + } + key = flb_sds_create_len(start, sep - start); + } + + /* value */ + start = sep + 1; + br = strchr(start, '\n'); + if (!br) { + break; + } + + val = flb_sds_create_len(start, br - start); + kv = flb_kv_item_create_len(&req->headers, + key, flb_sds_len(key), val, flb_sds_len(val)); + flb_sds_destroy(key); + flb_sds_destroy(val); + + /* next header */ + start = br + 1; + p = strchr(start, '\n'); + } while (p && *p == '\n'); + + /* Is this a POST request with a payload ? */ + if (p && *p == '\n') { + p++; + if (p) { + len = strlen(request) - (p - request); + payload = flb_sds_create_len(p, len); + } + } + else { + /* Append any remaining headers, aws tests do not end files with a \n */ + br++; + if ((br - request) - len) { + start = br; + sep = strchr(start, ':'); + + key = flb_sds_create_len(start, sep - start); + val = flb_sds_create(sep + 1); + flb_kv_item_create_len(&req->headers, + key, flb_sds_len(key), val, flb_sds_len(val)); + flb_sds_destroy(key); + flb_sds_destroy(val); + } + } + + if (payload) { + req->payload = payload; + } + + return req; +} + +static void http_request_destroy(struct request *req) +{ + if (!req) { + return; + } + + if (req->method) { + flb_sds_destroy(req->method); + } + if (req->uri) { + flb_sds_destroy(req->uri); + } + if (req->uri_full) { + flb_sds_destroy(req->uri_full); + } + if (req->query_string) { + flb_sds_destroy(req->query_string); + } + if (req->payload) { + flb_sds_destroy(req->payload); + } + + flb_kv_release(&req->headers); + flb_free(req); +} + +/* Convert a TXT HTTP request to a Fluent Bit http_client context */ +static struct flb_http_client *convert_request_file(char *request, + struct request **r, + struct flb_config *config) +{ + struct flb_upstream *u; + struct flb_connection *u_conn; + struct flb_http_client *c; + struct mk_list *head; + struct flb_kv *kv; + struct request *req; + + /* Fake Upstream context, required by http client */ + u = flb_upstream_create(config, "127.0.0.1", 80, 0, NULL); + if (!u) { + fprintf(stderr, "error creating upstream context"); + flb_free(config); + return NULL; + } + + /* Fake upstream connection */ + u_conn = flb_calloc(1, sizeof(struct flb_connection)); + if (!u_conn) { + flb_errno(); + flb_upstream_destroy(u); + flb_free(config); + } + u_conn->upstream = u; + + /* Convert TXT HTTP request to our local 'request' structure */ + req = http_request_create(request); + if (!req) { + fprintf(stderr, "error parsing txt http request"); + exit(1); + } + + /* HTTP Client context */ + c = flb_http_client(u_conn, req->method_i, req->uri_full, + req->payload, req->payload ? flb_sds_len(req->payload): -1, + NULL, -1, NULL, 0); + + /* + * flb_http_client automatically adds host and content-length + * for the tests we remove these since all headers come from + * the the test file + */ + mk_list_foreach(head, &c->headers) { + kv = mk_list_entry(head, struct flb_kv, _head); + if (strncasecmp(kv->key, "Host", 4) == 0) { + flb_kv_item_destroy(kv); + break; + } + } + mk_list_foreach(head, &c->headers) { + kv = mk_list_entry(head, struct flb_kv, _head); + if (strncasecmp(kv->key, "Content-Length", 14) == 0) { + flb_kv_item_destroy(kv); + break; + } + } + + /* Append registered headers */ + mk_list_foreach(head, &req->headers) { + kv = mk_list_entry(head, struct flb_kv, _head); + flb_http_add_header(c, + kv->key, flb_sds_len(kv->key), + kv->val, flb_sds_len(kv->val)); + } + + *r = req; + return c; +} + +static flb_sds_t file_to_buffer(char *path, char *context, char *ext) +{ + char abs_path[2048]; + char *buf; + flb_sds_t data; + + snprintf(abs_path, sizeof(abs_path) - 1, "%s/%s.%s", path, context, ext); + buf = mk_file_to_buffer(abs_path); + if (!buf) { + return NULL; + } + + data = flb_sds_create(buf); + if (!data) { + fprintf(stderr, "error allocating sds buffer to test %s file\n", abs_path); + flb_free(buf); + return NULL; + } + flb_free(buf); + + return data; +} + +static void aws_test_destroy(struct aws_test *awt) +{ + if (awt->name) { + flb_sds_destroy(awt->name); + } + if (awt->authz) { + flb_sds_destroy(awt->authz); + } + if (awt->creq) { + flb_sds_destroy(awt->creq); + } + if (awt->req) { + flb_sds_destroy(awt->req); + } + if (awt->sreq) { + flb_sds_destroy(awt->sreq); + } + if (awt->sts) { + flb_sds_destroy(awt->sts); + } + + if (awt->c) { + flb_upstream_destroy(awt->c->u_conn->upstream); + flb_free(awt->c->u_conn); + flb_http_client_destroy(awt->c); + } + + http_request_destroy(awt->r); + flb_free(awt); +} + +static struct aws_test *aws_test_create(char *path, char *context, + struct flb_config *config) +{ + struct aws_test *awt; + + awt = flb_calloc(1, sizeof(struct aws_test)); + if (!awt) { + flb_errno(); + return NULL; + } + + awt->name = flb_sds_create(context); + if (!awt->name) { + fprintf(stderr, "cannot allocate awt name\n"); + goto error; + } + + /* If no 'authz' file is found, return right away */ + awt->authz = file_to_buffer(path, context, "authz"); + if (!awt->authz) { + aws_test_destroy(awt); + return NULL; + } + + awt->creq = file_to_buffer(path, context, "creq"); + if (!awt->creq) { + fprintf(stderr, "error reading creq file"); + goto error; + } + + awt->req = file_to_buffer(path, context, "req"); + if (!awt->req) { + fprintf(stderr, "error reading req file"); + goto error; + } + + awt->sreq = file_to_buffer(path, context, "sreq"); + if (!awt->sreq) { + fprintf(stderr, "error reading req file"); + goto error; + } + + awt->sts = file_to_buffer(path, context, "sts"); + if (!awt->sts) { + fprintf(stderr, "error reading req file"); + goto error; + } + + /* Convert TXT HTTP request to http_client context */ + awt->c = convert_request_file(awt->req, &awt->r, config); + if (!awt->c) { + fprintf(stderr, "error converting TXT request to a context: %s", awt->name); + goto error; + } + + return awt; + + error: + //aws_test_destroy(awt); + return NULL; +} + +static int load_aws_test_directory(struct mk_list *list, char *ut_path, + struct flb_config *config) +{ + int ret; + struct dirent *e; + DIR *dir; + char path[2048]; + char ut[4096]; + struct stat st; + struct aws_test *awt; + + dir = opendir(ut_path); + if (!dir) { + flb_errno(); + flb_error("signv4: cannot open test suite located at '%s'", AWS_SUITE); + return -1; + } + + /* Read directory entries */ + while ((e = readdir(dir)) != NULL) { + if (*e->d_name == '.') { + continue; + } + + snprintf(path, sizeof(path) - 1, "%s%s", ut_path, e->d_name); + ret = stat(path, &st); + if (ret == -1) { + continue; + } + + /* only process directories */ + if (!S_ISDIR(st.st_mode)) { + continue; + } + + /* check for unit test file */ + snprintf(ut, sizeof(ut) - 1, "%s%s/%s.req", path, e->d_name, e->d_name); + ret = stat(path, &st); + if (ret == -1) { + continue; + } + + awt = aws_test_create(path, e->d_name, config); + if (!awt) { + continue; + } + mk_list_add(&awt->_head, list); + } + + closedir(dir); + return 0; +} + +static void aws_tests_destroy(struct mk_list *list) +{ + struct mk_list *tmp; + struct mk_list *head; + struct aws_test *awt; + + mk_list_foreach_safe(head, tmp, list) { + awt = mk_list_entry(head, struct aws_test, _head); + mk_list_del(&awt->_head); + aws_test_destroy(awt); + } + + flb_free(list); +} + +static struct mk_list *aws_tests_create(struct flb_config *config) +{ + struct mk_list *list; + char path[2048]; + + printf("\n"); + + list = flb_malloc(sizeof(struct mk_list)); + if (!list) { + flb_errno(); + return NULL; + } + mk_list_init(list); + + /* Load base path for AWS test suite, some sub-directories will be skipped */ + load_aws_test_directory(list, AWS_SUITE, config); + + /* Load pending sub-directories */ + snprintf(path, sizeof(path) - 1, "%s/normalize-path/", AWS_SUITE); + load_aws_test_directory(list, path, config); + + snprintf(path, sizeof(path) - 1, "%s/post-sts-token/", AWS_SUITE); + load_aws_test_directory(list, path, config); + + return list; +} + +static void aws_test_suite() +{ + int ret; + time_t t; + char *region = NULL; + char *access_key = NULL; + char *service = NULL; + char *secret_key = NULL; + flb_sds_t signature; + struct mk_list *head; + struct mk_list *tests; + struct flb_config *config; + struct aws_test *awt; + struct flb_aws_provider *provider; + + config = flb_calloc(1, sizeof(struct flb_config)); + if (!config) { + flb_errno(); + return; + } + mk_list_init(&config->upstreams); + + /* Get a list of tests */ + tests = aws_tests_create(config); + TEST_CHECK(tests != NULL); + if (!tests) { + flb_free(config); + return; + } + + /* Convert static '20150830T123600Z' to unix timestamp */ + t = 1440938160; + region = "us-east-1"; + access_key = "AKIDEXAMPLE"; + service = "service"; + secret_key = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; + + /* credentials */ + ret = setenv(AWS_ACCESS_KEY_ID, access_key, 1); + if (ret < 0) { + flb_errno(); + return; + } + ret = setenv(AWS_SECRET_ACCESS_KEY, secret_key, 1); + if (ret < 0) { + flb_errno(); + return; + } + provider = flb_aws_env_provider_create(); + if (!provider) { + flb_errno(); + return; + } + + /* Iterate tests and sign the requests */ + mk_list_foreach(head, tests) { + awt = mk_list_entry(head, struct aws_test, _head); + fprintf(stderr, "[AWS Signv4 Unit Test] %-50s", awt->name); + signature = flb_signv4_do(awt->c, + FLB_TRUE, /* normalize URI ? */ + FLB_FALSE, /* add x-amz-date header ? */ + t, region, service, + 0, NULL, + provider); + TEST_CHECK(signature != NULL); + if (signature) { + ret = strncmp(awt->authz, signature, flb_sds_len(awt->authz)); + TEST_CHECK(ret == 0); + if (ret != 0) { + fprintf(stderr, "\t\tFAIL"); + fprintf(stderr, + ">\n> signature check failed...\n received: %s\n expected: %s\n", + signature, awt->authz); + } + else { + fprintf(stderr, "PASS"); + } + flb_sds_destroy(signature); + } + fprintf(stderr, "\n"); + } + + aws_tests_destroy(tests); + flb_aws_provider_destroy(provider); + flb_free(config); +} + +static void check_normalize(char *s, size_t len, char *out) +{ + flb_sds_t o; + + o = flb_signv4_uri_normalize_path(s, len); + TEST_CHECK(strcmp(o, out) == 0); + flb_sds_destroy(o); +} + +void normalize() +{ + /* get-relative */ + check_normalize("/example/..", 11, "/"); + + /* get-relative-relative */ + check_normalize("/example1/example2/../..", 24, "/"); + + /* get-slash */ + check_normalize("//", 2, "/"); + + /* get-slash-dot-slash */ + check_normalize("/./", 3, "/"); + + /* get-slashes */ + check_normalize("//example//", 11, "/example/"); + + /* get-slash-pointless-dot */ + check_normalize("/./example", 10, "/example"); +} + +TEST_LIST = { + { "aws_test_suite", aws_test_suite}, + { "normalize", normalize}, + { 0 } +}; |