From 06eaf7232e9a920468c0f8d74dcf2fe8b555501c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 13 Apr 2024 14:24:36 +0200 Subject: Adding upstream version 1:10.11.6. Signed-off-by: Daniel Baumann --- storage/maria/libmarias3/src/assume_role.c | 703 ++++++++++++++ storage/maria/libmarias3/src/assume_role.h | 22 + storage/maria/libmarias3/src/common.h | 43 + storage/maria/libmarias3/src/debug.c | 34 + storage/maria/libmarias3/src/debug.h | 56 ++ storage/maria/libmarias3/src/error.c | 39 + storage/maria/libmarias3/src/error.h | 27 + storage/maria/libmarias3/src/include.am | 37 + storage/maria/libmarias3/src/marias3.c | 668 ++++++++++++++ storage/maria/libmarias3/src/memory.h | 26 + storage/maria/libmarias3/src/request.c | 967 ++++++++++++++++++++ storage/maria/libmarias3/src/request.h | 63 ++ storage/maria/libmarias3/src/response.c | 512 +++++++++++ storage/maria/libmarias3/src/response.h | 32 + storage/maria/libmarias3/src/sha256-internal.c | 251 +++++ storage/maria/libmarias3/src/sha256.c | 131 +++ storage/maria/libmarias3/src/sha256.h | 25 + storage/maria/libmarias3/src/sha256_i.h | 66 ++ storage/maria/libmarias3/src/structs.h | 83 ++ storage/maria/libmarias3/src/xml.c | 1157 ++++++++++++++++++++++++ storage/maria/libmarias3/src/xml.h | 197 ++++ 21 files changed, 5139 insertions(+) create mode 100644 storage/maria/libmarias3/src/assume_role.c create mode 100644 storage/maria/libmarias3/src/assume_role.h create mode 100644 storage/maria/libmarias3/src/common.h create mode 100644 storage/maria/libmarias3/src/debug.c create mode 100644 storage/maria/libmarias3/src/debug.h create mode 100644 storage/maria/libmarias3/src/error.c create mode 100644 storage/maria/libmarias3/src/error.h create mode 100644 storage/maria/libmarias3/src/include.am create mode 100644 storage/maria/libmarias3/src/marias3.c create mode 100644 storage/maria/libmarias3/src/memory.h create mode 100644 storage/maria/libmarias3/src/request.c create mode 100644 storage/maria/libmarias3/src/request.h create mode 100644 storage/maria/libmarias3/src/response.c create mode 100644 storage/maria/libmarias3/src/response.h create mode 100644 storage/maria/libmarias3/src/sha256-internal.c create mode 100644 storage/maria/libmarias3/src/sha256.c create mode 100644 storage/maria/libmarias3/src/sha256.h create mode 100644 storage/maria/libmarias3/src/sha256_i.h create mode 100644 storage/maria/libmarias3/src/structs.h create mode 100644 storage/maria/libmarias3/src/xml.c create mode 100644 storage/maria/libmarias3/src/xml.h (limited to 'storage/maria/libmarias3/src') diff --git a/storage/maria/libmarias3/src/assume_role.c b/storage/maria/libmarias3/src/assume_role.c new file mode 100644 index 00000000..255b1eca --- /dev/null +++ b/storage/maria/libmarias3/src/assume_role.c @@ -0,0 +1,703 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2020 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "config.h" +#include "common.h" +#include "sha256.h" + +#include + +const char *default_iam_domain = "iam.amazonaws.com"; +const char *default_sts_domain = "sts.amazonaws.com"; +const char *iam_request_region = "us-east-1"; + +static void set_error(ms3_st *ms3, const char *error) +{ + ms3_cfree(ms3->last_error); + + if (!error) + { + ms3->last_error = NULL; + return; + } + + ms3->last_error = ms3_cstrdup(error); +} + +static void set_error_nocopy(ms3_st *ms3, char *error) +{ + ms3_cfree(ms3->last_error); + + if (!error) + { + ms3->last_error = NULL; + return; + } + + ms3->last_error = error; +} + +static size_t header_callback(char *buffer, size_t size, + size_t nitems, void *userdata) +{ + ms3debug("%.*s\n", (int)(nitems * size), buffer); + + if (userdata) + { + // HEAD request + if (!strncasecmp(buffer, "Last-Modified", 13)) + { + ms3_status_st *status = (ms3_status_st *) userdata; + // Date/time, format: Fri, 15 Mar 2019 16:58:54 GMT + struct tm ttmp = {0}; + strptime(buffer + 15, "%a, %d %b %Y %H:%M:%S %Z", &ttmp); + status->created = mktime(&ttmp); + } + else if (!strncasecmp(buffer, "Content-Length", 14)) + { + ms3_status_st *status = (ms3_status_st *) userdata; + // Length + status->length = strtoull(buffer + 16, NULL, 10); + } + } + + return nitems * size; +} + +static size_t body_callback(void *buffer, size_t size, + size_t nitems, void *userdata) +{ + uint8_t *ptr; + size_t realsize = nitems * size; + + struct memory_buffer_st *mem = (struct memory_buffer_st *)userdata; + + if (realsize + mem->length >= mem->alloced) + { + size_t additional_size = mem->buffer_chunk_size; + + if (realsize >= mem->buffer_chunk_size) + { + additional_size = (ceil((double)realsize / (double)mem->buffer_chunk_size) + 1) + * mem->buffer_chunk_size; + } + + ptr = (uint8_t *)ms3_crealloc(mem->data, mem->alloced + additional_size); + + if (!ptr) + { + ms3debug("Curl response OOM"); + return 0; + } + + mem->alloced += additional_size; + mem->data = ptr; + } + + memcpy(&(mem->data[mem->length]), buffer, realsize); + mem->length += realsize; + mem->data[mem->length] = '\0'; + + ms3debug("Read %zu bytes, buffer %zu bytes", realsize, mem->length); +// ms3debug("Data: %s", (char*)buffer); + return nitems * size; +} + +static uint8_t build_assume_role_request_uri(CURL *curl, const char *base_domain, const char *query, bool use_http) +{ + char uri_buffer[MAX_URI_LENGTH]; + const char *domain; + const uint8_t path_parts = 10; // "https://" + "." + "/" + const char *http_protocol = "http"; + const char *https_protocol = "https"; + const char *protocol; + + if (base_domain) + { + domain = base_domain; + } + else + { + domain = default_sts_domain; + } + + if (use_http) + { + protocol = http_protocol; + } + else + { + protocol = https_protocol; + } + + if (query) + { + if (path_parts + strlen(domain) + strlen(query) >= MAX_URI_LENGTH - 1) + { + return MS3_ERR_URI_TOO_LONG; + } + + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/?%s", protocol, + domain, query); + } + else + { + return MS3_ERR_PARAMETER; + } + + ms3debug("URI: %s", uri_buffer); + curl_easy_setopt(curl, CURLOPT_URL, uri_buffer); + return 0; +} + +static char *generate_assume_role_query(CURL *curl, const char *action, size_t role_duration, + const char *version, const char *role_session_name, const char *role_arn, + const char *continuation, char *query_buffer) +{ + size_t query_buffer_length = 0; + char *encoded; + query_buffer[0] = '\0'; + + if (action) + { + encoded = curl_easy_escape(curl, action, (int)strlen(action)); + query_buffer_length = strlen(query_buffer); + if (query_buffer_length) + { + snprintf(query_buffer + query_buffer_length, 3072 - query_buffer_length, + "&Action=%s", encoded); + } + else + { + snprintf(query_buffer, 3072, "Action=%s", encoded); + } + curl_free(encoded); + } + if (role_duration >= 900 && role_duration <= 43200) + { + query_buffer_length = strlen(query_buffer); + if (query_buffer_length) + { + snprintf(query_buffer + query_buffer_length, 3072 - query_buffer_length, + "&DurationSeconds=%zu", role_duration); + } + else + { + snprintf(query_buffer, 3072, "DurationSeconds=%zu", role_duration); + } + } + if (continuation) + { + encoded = curl_easy_escape(curl, continuation, (int)strlen(continuation)); + query_buffer_length = strlen(query_buffer); + if (query_buffer_length) + { + snprintf(query_buffer + query_buffer_length, 3072 - query_buffer_length, + "&Marker=%s", encoded); + } + else + { + snprintf(query_buffer, 3072, "Marker=%s", encoded); + } + curl_free(encoded); + } + if (role_arn) + { + encoded = curl_easy_escape(curl, role_arn, (int)strlen(role_arn)); + query_buffer_length = strlen(query_buffer); + if (query_buffer_length) + { + snprintf(query_buffer + query_buffer_length, 3072 - query_buffer_length, + "&RoleArn=%s", encoded); + } + else + { + snprintf(query_buffer, 3072, "RoleArn=%s", encoded); + } + curl_free(encoded); + } + if (role_session_name) + { + encoded = curl_easy_escape(curl, role_session_name, (int)strlen(role_session_name)); + query_buffer_length = strlen(query_buffer); + if (query_buffer_length) + { + snprintf(query_buffer + query_buffer_length, 3072 - query_buffer_length, + "&RoleSessionName=%s", encoded); + } + else + { + snprintf(query_buffer, 3072, "RoleSessionName=%s", encoded); + } + curl_free(encoded); + } + if (version) + { + encoded = curl_easy_escape(curl, version, (int)strlen(version)); + query_buffer_length = strlen(query_buffer); + if (query_buffer_length) + { + snprintf(query_buffer + query_buffer_length, 3072 - query_buffer_length, + "&Version=%s", encoded); + } + else + { + snprintf(query_buffer, 3072, "Version=%s", encoded); + } + curl_free(encoded); + } + + return query_buffer; +} + + +static uint8_t generate_assume_role_request_hash(uri_method_t method, const char *query, char *post_hash, + struct curl_slist *headers, char *return_hash) +{ + char signing_data[3072]; + size_t pos = 0; + uint8_t sha256hash[32]; // SHA_256 binary length + uint8_t hash_pos = 0; + uint8_t i; + struct curl_slist *current_header = headers; + + // Method first + switch (method) + { + case MS3_GET: + { + sprintf(signing_data, "GET\n"); + pos += 4; + break; + } + + case MS3_HEAD: + { + sprintf(signing_data, "HEAD\n"); + pos += 5; + break; + } + + case MS3_PUT: + { + sprintf(signing_data, "PUT\n"); + pos += 4; + break; + } + + case MS3_DELETE: + { + sprintf(signing_data, "DELETE\n"); + pos += 7; + break; + } + + default: + { + ms3debug("Bad method detected"); + return MS3_ERR_IMPOSSIBLE; + } + } + + // URL query (if exists) + if (query) + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, "/\n%s\n", query); + pos += strlen(query) + 3; + } + else + { + sprintf(signing_data + pos, "\n"); + pos++; + } + + do + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, "%s\n", + current_header->data); + pos += strlen(current_header->data) + 1; + } + while ((current_header = current_header->next)); + + // List if header names + // The newline between headers and this is important + snprintf(signing_data + pos, sizeof(signing_data) - pos, + "\nhost;x-amz-content-sha256;x-amz-date\n"); + pos += 38; + + // Hash of post data (can be hash of empty) + snprintf(signing_data + pos, sizeof(signing_data) - pos, "%.*s", 64, post_hash); + //pos+= 64; + + // Hash all of the above + sha256((uint8_t *)signing_data, strlen(signing_data), (uint8_t *)sha256hash); + + for (i = 0; i < 32; i++) + { + sprintf(return_hash + hash_pos, "%.2x", sha256hash[i]); + hash_pos += 2; + } + + ms3debug("Signature data: %s", signing_data); + ms3debug("Signature: %.*s", 64, return_hash); + + return 0; +} + +static uint8_t +build_assume_role_request_headers(CURL *curl, struct curl_slist **head, + const char *base_domain, + const char* endpoint_type, + const char *region, const char *key, + const char *secret, const char *query, + uri_method_t method, + struct put_buffer_st *post_data) +{ + uint8_t ret = 0; + time_t now; + struct tm tmp_tm; + char headerbuf[3072]; + char secrethead[45]; + char date[9]; + char sha256hash[65]; + char post_hash[65]; + uint8_t tmp_hash[32]; + // Alternate between these two so hmac doesn't overwrite itself + uint8_t hmac_hash[32]; + uint8_t hmac_hash2[32]; + uint8_t hash_pos = 0; + const char *domain; + const char *type; + struct curl_slist *headers = NULL; + uint8_t offset; + uint8_t i; + struct curl_slist *current_header; + + // Host header + if (base_domain) + { + domain = base_domain; + } + else + { + domain = default_sts_domain; + } + + if (endpoint_type) + { + type = endpoint_type; + } + else + { + type = "sts"; + } + + snprintf(headerbuf, sizeof(headerbuf), "host:%s", domain); + + headers = curl_slist_append(headers, headerbuf); + *head = headers; + + // Hash post data + sha256(post_data->data, post_data->length, tmp_hash); + + for (i = 0; i < 32; i++) + { + sprintf(post_hash + hash_pos, "%.2x", tmp_hash[i]); + hash_pos += 2; + } + + snprintf(headerbuf, sizeof(headerbuf), "x-amz-content-sha256:%.*s", 64, + post_hash); + headers = curl_slist_append(headers, headerbuf); + + // Date/time header + time(&now); + snprintf(headerbuf, sizeof(headerbuf), "x-amz-date:"); + offset = strlen(headerbuf); + gmtime_r(&now, &tmp_tm); + strftime(headerbuf + offset, sizeof(headerbuf) - offset, "%Y%m%dT%H%M%SZ", + &tmp_tm); + headers = curl_slist_append(headers, headerbuf); + + // Builds the request hash + ret = generate_assume_role_request_hash(method, query, post_hash, headers, sha256hash); + + if (ret) + { + return ret; + } + + // User signing key hash + // Date hashed using AWS4:secret_key + snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", 40, secret); + strftime(headerbuf, sizeof(headerbuf), "%Y%m%d", &tmp_tm); + hmac_sha256((uint8_t *)secrethead, strlen(secrethead), (uint8_t *)headerbuf, + strlen(headerbuf), hmac_hash); + + // Region signed by above key + hmac_sha256(hmac_hash, 32, (uint8_t *)region, strlen(region), + hmac_hash2); + + // Service signed by above key + hmac_sha256(hmac_hash2, 32, (uint8_t *)type, strlen(type), + hmac_hash); + + // Request version signed by above key (always "aws4_request") + sprintf(headerbuf, "aws4_request"); + hmac_sha256(hmac_hash, 32, (uint8_t *)headerbuf, strlen(headerbuf), + hmac_hash2); + + // Sign everything with the key + snprintf(headerbuf, sizeof(headerbuf), "AWS4-HMAC-SHA256\n"); + offset = strlen(headerbuf); + strftime(headerbuf + offset, sizeof(headerbuf) - offset, "%Y%m%dT%H%M%SZ\n", + &tmp_tm); + offset = strlen(headerbuf); + strftime(date, 9, "%Y%m%d", &tmp_tm); + snprintf(headerbuf + offset, sizeof(headerbuf) - offset, + "%.*s/%s/%s/aws4_request\n%.*s", 8, date, region, type, 64, sha256hash); + ms3debug("Data to sign: %s", headerbuf); + hmac_sha256(hmac_hash2, 32, (uint8_t *)headerbuf, strlen(headerbuf), + hmac_hash); + + hash_pos = 0; + + for (i = 0; i < 32; i++) + { + sprintf(sha256hash + hash_pos, "%.2x", hmac_hash[i]); + hash_pos += 2; + } + + // Make auth header + snprintf(headerbuf, sizeof(headerbuf), + "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=%s", + key, date, region, type, sha256hash); + + headers = curl_slist_append(headers, headerbuf); + + // Disable this header or PUT will barf with a 501 + sprintf(headerbuf, "Transfer-Encoding:"); + headers = curl_slist_append(headers, headerbuf); + + current_header = headers; + + do + { + ms3debug("Header: %s", current_header->data); + } + while ((current_header = current_header->next)); + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + return 0; +} + +uint8_t execute_assume_role_request(ms3_st *ms3, command_t cmd, + const uint8_t *data, size_t data_size, + char *continuation) +{ + CURL *curl = NULL; + struct curl_slist *headers = NULL; + uint8_t res = 0; + struct memory_buffer_st mem; + uri_method_t method; + char *query = NULL; + struct put_buffer_st post_data; + CURLcode curl_res; + long response_code = 0; + char* endpoint = NULL; + const char* region = iam_request_region; + char endpoint_type[8]; + + mem.data = NULL; + mem.length = 0; + mem.alloced = 1; + mem.buffer_chunk_size = ms3->buffer_chunk_size; + + post_data.data = (uint8_t *) data; + post_data.length = data_size; + post_data.offset = 0; + + curl = ms3->curl; + + if (!ms3->first_run) + { + curl_easy_reset(curl); + } + else + { + ms3->first_run = false; + } + + if (cmd == MS3_CMD_ASSUME_ROLE) + { + query = generate_assume_role_query(curl, "AssumeRole", ms3->role_session_duration, "2011-06-15", "libmariaS3", + ms3->iam_role_arn, continuation, ms3->query_buffer); + endpoint = ms3->sts_endpoint; + region = ms3->sts_region; + sprintf(endpoint_type, "sts"); + method = MS3_GET; + } + else if (cmd == MS3_CMD_LIST_ROLE) + { + query = generate_assume_role_query(curl, "ListRoles", 0, "2010-05-08", NULL, NULL, continuation, ms3->query_buffer); + endpoint = ms3->iam_endpoint; + sprintf(endpoint_type, "iam"); + method = MS3_GET; + } + + res = build_assume_role_request_uri(curl, endpoint, query, ms3->use_http); + + if (res) + { + return res; + } + + res = build_assume_role_request_headers(curl, &headers, endpoint, + endpoint_type, region, + ms3->s3key, ms3->s3secret, query, + method, &post_data); + + if (res) + { + ms3_cfree(mem.data); + curl_slist_free_all(headers); + + return res; + } + + if (ms3->disable_verification) + { + ms3debug("Disabling SSL verification"); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); + } + + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); + curl_res = curl_easy_perform(curl); + + if (curl_res != CURLE_OK) + { + ms3debug("Curl error: %s", curl_easy_strerror(curl_res)); + set_error(ms3, curl_easy_strerror(curl_res)); + ms3_cfree(mem.data); + curl_slist_free_all(headers); + + return MS3_ERR_REQUEST_ERROR; + } + + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + ms3debug("Response code: %ld", response_code); + + if (response_code == 404) + { + char *message = parse_error_message((char *)mem.data, mem.length); + + if (message) + { + ms3debug("Response message: %s", message); + } + + set_error_nocopy(ms3, message); + res = MS3_ERR_NOT_FOUND; + } + else if (response_code == 403) + { + char *message = parse_error_message((char *)mem.data, mem.length); + + if (message) + { + ms3debug("Response message: %s", message); + } + + set_error_nocopy(ms3, message); + res = MS3_ERR_AUTH; + } + else if (response_code >= 400) + { + char *message = parse_error_message((char *)mem.data, mem.length); + + if (message) + { + ms3debug("Response message: %s", message); + } + + set_error_nocopy(ms3, message); + res = MS3_ERR_SERVER; + } + + switch (cmd) + { + case MS3_CMD_LIST_ROLE: + { + char *cont = NULL; + res = parse_role_list_response((const char *)mem.data, mem.length, ms3->iam_role ,ms3->iam_role_arn, &cont); + + if (cont && res) + { + res = execute_assume_role_request(ms3, cmd, data, data_size, cont); + if (res) + { + ms3_cfree(cont); + ms3_cfree(mem.data); + curl_slist_free_all(headers); + return res; + } + ms3_cfree(cont); + } + + ms3_cfree(mem.data); + break; + } + + case MS3_CMD_ASSUME_ROLE: + { + if (res) + { + ms3_cfree(mem.data); + curl_slist_free_all(headers); + return res; + } + res = parse_assume_role_response((const char *)mem.data, mem.length, ms3->role_key, ms3->role_secret, ms3->role_session_token); + ms3_cfree(mem.data); + break; + } + + case MS3_CMD_LIST: + case MS3_CMD_LIST_RECURSIVE: + case MS3_CMD_PUT: + case MS3_CMD_GET: + case MS3_CMD_DELETE: + case MS3_CMD_HEAD: + case MS3_CMD_COPY: + default: + { + ms3_cfree(mem.data); + ms3debug("Bad cmd detected"); + res = MS3_ERR_IMPOSSIBLE; + } + } + + curl_slist_free_all(headers); + + return res; +} diff --git a/storage/maria/libmarias3/src/assume_role.h b/storage/maria/libmarias3/src/assume_role.h new file mode 100644 index 00000000..1a2c861a --- /dev/null +++ b/storage/maria/libmarias3/src/assume_role.h @@ -0,0 +1,22 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2020 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#pragma once + +uint8_t execute_assume_role_request(ms3_st *ms3, command_t cmd, const uint8_t *data, size_t data_size, char *continuation); diff --git a/storage/maria/libmarias3/src/common.h b/storage/maria/libmarias3/src/common.h new file mode 100644 index 00000000..9d4c2b08 --- /dev/null +++ b/storage/maria/libmarias3/src/common.h @@ -0,0 +1,43 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#pragma once + +#ifdef __cplusplus +#include +#endif + +#include +#include +#include +#include +#include + +#include + +#include + +#include "memory.h" +#include "debug.h" +#include "error.h" +#include "structs.h" +#include "response.h" +#include "request.h" +#include "assume_role.h" + diff --git a/storage/maria/libmarias3/src/debug.c b/storage/maria/libmarias3/src/debug.c new file mode 100644 index 00000000..33fe1764 --- /dev/null +++ b/storage/maria/libmarias3/src/debug.c @@ -0,0 +1,34 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "config.h" +#include "common.h" +#include + +static bool debugging_enabled = false; + +void ms3debug_set(bool enabled) +{ + debugging_enabled = enabled; +} + +bool ms3debug_get(void) +{ + return debugging_enabled; +} diff --git a/storage/maria/libmarias3/src/debug.h b/storage/maria/libmarias3/src/debug.h new file mode 100644 index 00000000..7e4f1d2c --- /dev/null +++ b/storage/maria/libmarias3/src/debug.h @@ -0,0 +1,56 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#pragma once + +#include "config.h" +#include + +void ms3debug_set(bool enabled); +bool ms3debug_get(void); + +#define ms3debug(MSG, ...) do { \ + if (ms3debug_get()) \ + { \ + fprintf(stderr, "[libmarias3] %s:%d " MSG "\n", __FILE__, __LINE__, ##__VA_ARGS__); \ + } \ +} while(0) +#define ms3debug_hex(DATA, LEN) do { \ + size_t hex_it; \ + fprintf(stderr, "[libmarias3] %s:%d packet hex: ", __FILE__, __LINE__); \ + for (hex_it = 0; hex_it < LEN ; hex_it++) \ + { \ + fprintf(stderr, "%02X ", (unsigned char)DATA[hex_it]); \ + } \ + fprintf(stderr, "\n"); \ + fprintf(stderr, "[libmarias3] %s:%d printable packet data: ", __FILE__, __LINE__); \ + for (hex_it = 0; hex_it < LEN ; hex_it++) \ + { \ + if (((unsigned char)DATA[hex_it] < 0x32) or (((unsigned char)DATA[hex_it] > 0x7e))) \ + { \ + fprintf(stderr, "."); \ + } \ + else \ + { \ + fprintf(stderr, "%c", (unsigned char)DATA[hex_it]); \ + } \ + } \ + fprintf(stderr, "\n"); \ +} while(0) + diff --git a/storage/maria/libmarias3/src/error.c b/storage/maria/libmarias3/src/error.c new file mode 100644 index 00000000..edf95d05 --- /dev/null +++ b/storage/maria/libmarias3/src/error.c @@ -0,0 +1,39 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "config.h" +#include "common.h" + +// NOTE: for every new error, add an entry to errmsgs here + +const char *errmsgs[] = +{ + "No error", + "Parameter error", + "No data", + "Could not parse response XML", + "Generated URI too long", + "Error making REST request", + "Out of memory", + "Impossible condition detected", + "Authentication error", + "File not found", + "S3 server error", + "Data too big. Maximum data size is 4GB" +}; diff --git a/storage/maria/libmarias3/src/error.h b/storage/maria/libmarias3/src/error.h new file mode 100644 index 00000000..e106b285 --- /dev/null +++ b/storage/maria/libmarias3/src/error.h @@ -0,0 +1,27 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#pragma once + +#include "config.h" + +#define baderror "No such error code" + +// extern and define in C file so we don't get redefinition at link time +extern const char *errmsgs[]; diff --git a/storage/maria/libmarias3/src/include.am b/storage/maria/libmarias3/src/include.am new file mode 100644 index 00000000..b8c07a6f --- /dev/null +++ b/storage/maria/libmarias3/src/include.am @@ -0,0 +1,37 @@ +# vim:ft=automake +# included from Top Level Makefile.am +# All paths should be given relative to the root + +noinst_HEADERS+= src/common.h +noinst_HEADERS+= src/debug.h +noinst_HEADERS+= src/error.h +noinst_HEADERS+= src/structs.h +noinst_HEADERS+= src/request.h +noinst_HEADERS+= src/response.h +noinst_HEADERS+= src/xml.h +noinst_HEADERS+= src/memory.h +noinst_HEADERS+= src/sha256.h +noinst_HEADERS+= src/sha256_i.h +noinst_HEADERS+= src/assume_role.h + +lib_LTLIBRARIES+= src/libmarias3.la +src_libmarias3_la_SOURCES= +src_libmarias3_la_LIBADD= +src_libmarias3_la_LDFLAGS= +src_libmarias3_la_CFLAGS= -DBUILDING_MS3 + +src_libmarias3_la_SOURCES+= src/marias3.c +src_libmarias3_la_SOURCES+= src/request.c +src_libmarias3_la_SOURCES+= src/response.c +src_libmarias3_la_SOURCES+= src/assume_role.c +src_libmarias3_la_SOURCES+= src/error.c +src_libmarias3_la_SOURCES+= src/debug.c + +src_libmarias3_la_SOURCES+= src/sha256.c +src_libmarias3_la_SOURCES+= src/sha256-internal.c + +src_libmarias3_la_SOURCES+= src/xml.c + +src_libmarias3_la_LDFLAGS+= -version-info ${LIBMARIAS3_LIBRARY_VERSION} + +src_libmarias3_la_LIBADD+= @LIBCURL_LIBS@ @LIBM@ diff --git a/storage/maria/libmarias3/src/marias3.c b/storage/maria/libmarias3/src/marias3.c new file mode 100644 index 00000000..74d7233a --- /dev/null +++ b/storage/maria/libmarias3/src/marias3.c @@ -0,0 +1,668 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "config.h" +#include "common.h" + +#include +#include +#include + +ms3_malloc_callback ms3_cmalloc = (ms3_malloc_callback)malloc; +ms3_free_callback ms3_cfree = (ms3_free_callback)free; +ms3_realloc_callback ms3_crealloc = (ms3_realloc_callback)realloc; +ms3_strdup_callback ms3_cstrdup = (ms3_strdup_callback)strdup; +ms3_calloc_callback ms3_ccalloc = (ms3_calloc_callback)calloc; + + +/* Thread locking code for OpenSSL < 1.1.0 */ +#include +#ifndef RTLD_DEFAULT +#define RTLD_DEFAULT ((void *)0) +#endif +static pthread_mutex_t *mutex_buf = NULL; +#define CRYPTO_LOCK 1 +static void (*openssl_set_id_callback)(unsigned long (*func)(void)); +static void (*openssl_set_locking_callback)(void (*func)(int mode,int type, const char *file,int line)); +static int (*openssl_num_locks)(void); + +static void locking_function(int mode, int n, const char *file, int line) +{ + (void) file; + (void) line; + if(mode & CRYPTO_LOCK) + pthread_mutex_lock(&(mutex_buf[n])); + else + pthread_mutex_unlock(&(mutex_buf[n])); +} + +static int curl_needs_openssl_locking() +{ + curl_version_info_data *data = curl_version_info(CURLVERSION_NOW); + + if (data->ssl_version == NULL) + { + return 0; + } + + if (strncmp(data->ssl_version, "OpenSSL", 7) != 0) + { + return 0; + } + if (data->ssl_version[8] == '0') + { + return 1; + } + if ((data->ssl_version[8] == '1') && (data->ssl_version[10] == '0')) + { + openssl_set_id_callback = dlsym(RTLD_DEFAULT, "CRYPTO_set_id_callback"); + openssl_set_locking_callback = dlsym(RTLD_DEFAULT, "CRYPTO_set_locking_callback"); + openssl_num_locks = dlsym(RTLD_DEFAULT, "CRYPTO_num_locks"); + return openssl_set_id_callback != NULL && + openssl_set_locking_callback != NULL && + openssl_num_locks != NULL; + } + return 0; +} + +static unsigned long __attribute__((unused)) id_function(void) +{ + return ((unsigned long)pthread_self()); +} + +uint8_t ms3_library_init_malloc(ms3_malloc_callback m, + ms3_free_callback f, ms3_realloc_callback r, + ms3_strdup_callback s, ms3_calloc_callback c) +{ + if (!m || !f || !r || !s || !c) + { + return MS3_ERR_PARAMETER; + } + + ms3_cmalloc = m; + ms3_cfree = f; + ms3_crealloc = r; + ms3_cstrdup = s; + ms3_ccalloc = c; + + if (curl_needs_openssl_locking()) + { + int i; + mutex_buf = ms3_cmalloc(openssl_num_locks() * sizeof(pthread_mutex_t)); + if(mutex_buf) + { + for(i = 0; i < openssl_num_locks(); i++) + pthread_mutex_init(&(mutex_buf[i]), NULL); + openssl_set_id_callback(id_function); + openssl_set_locking_callback(locking_function); + } + } + + if (curl_global_init_mem(CURL_GLOBAL_DEFAULT, m, f, r, s, c)) + { + return MS3_ERR_PARAMETER; + } + + return 0; +} + +void ms3_library_init(void) +{ + if (curl_needs_openssl_locking()) + { + int i; + mutex_buf = malloc(openssl_num_locks() * sizeof(pthread_mutex_t)); + if(mutex_buf) + { + for(i = 0; i < openssl_num_locks(); i++) + pthread_mutex_init(&(mutex_buf[i]), NULL); + openssl_set_id_callback(id_function); + openssl_set_locking_callback(locking_function); + } + } + curl_global_init(CURL_GLOBAL_DEFAULT); +} + +void ms3_library_deinit(void) +{ + int i; + if (mutex_buf) + { + openssl_set_id_callback(NULL); + openssl_set_locking_callback(NULL); + for(i = 0; i < openssl_num_locks(); i++) + pthread_mutex_destroy(&(mutex_buf[i])); + ms3_cfree(mutex_buf); + mutex_buf = NULL; + } + curl_global_cleanup(); +} + +ms3_st *ms3_init(const char *s3key, const char *s3secret, + const char *region, + const char *base_domain) +{ + ms3_st *ms3; + + if ((s3key == NULL) || (s3secret == NULL)) + { + return NULL; + } + + ms3 = ms3_cmalloc(sizeof(ms3_st)); + + ms3->s3key = ms3_cstrdup(s3key); + ms3->s3secret = ms3_cstrdup(s3secret); + ms3->region = ms3_cstrdup(region); + ms3->port = 0; /* The default value */ + + if (base_domain && strlen(base_domain)) + { + struct sockaddr_in sa; + ms3->base_domain = ms3_cstrdup(base_domain); + if (inet_pton(AF_INET, base_domain, &(sa.sin_addr))) + { + ms3->list_version = 1; + ms3->protocol_version = 1; + } + else if (strcmp(base_domain, "s3.amazonaws.com") == 0) + { + ms3->list_version = 2; + ms3->protocol_version = 2; + } + else + { + // Assume that S3-compatible APIs can't support v2 list + ms3->list_version = 1; + ms3->protocol_version = 2; + } + } + else + { + ms3->base_domain = NULL; + ms3->list_version = 2; + ms3->protocol_version = 2; + } + + ms3->buffer_chunk_size = READ_BUFFER_DEFAULT_SIZE; + + ms3->curl = curl_easy_init(); + ms3->last_error = NULL; + ms3->use_http = false; + ms3->disable_verification = false; + ms3->first_run = true; + ms3->path_buffer = ms3_cmalloc(sizeof(char) * 1024); + ms3->query_buffer = ms3_cmalloc(sizeof(char) * 3072); + ms3->list_container.pool = NULL; + ms3->list_container.next = NULL; + ms3->list_container.start = NULL; + ms3->list_container.pool_list = NULL; + ms3->list_container.pool_free = 0; + + ms3->iam_role = NULL; + ms3->role_key = NULL; + ms3->role_secret = NULL; + ms3->role_session_token = NULL; + ms3->iam_endpoint = NULL; + ms3->sts_endpoint = NULL; + ms3->sts_region = NULL; + ms3->iam_role_arn = NULL; + + return ms3; +} + +uint8_t ms3_init_assume_role(ms3_st *ms3, const char *iam_role, const char *sts_endpoint, const char *sts_region) +{ + uint8_t ret=0; + + if (iam_role == NULL) + { + return MS3_ERR_PARAMETER; + } + ms3->iam_role = ms3_cstrdup(iam_role); + + if (sts_endpoint && strlen(sts_endpoint)) + { + ms3->sts_endpoint = ms3_cstrdup(sts_endpoint); + } + else + { + ms3->sts_endpoint = ms3_cstrdup("sts.amazonaws.com"); + } + + if (sts_region && strlen(sts_region)) + { + ms3->sts_region = ms3_cstrdup(sts_region); + } + else + { + ms3->sts_region = ms3_cstrdup("us-east-1"); + } + + ms3->iam_endpoint = ms3_cstrdup("iam.amazonaws.com"); + + ms3->iam_role_arn = ms3_cmalloc(sizeof(char) * 2048); + ms3->iam_role_arn[0] = '\0'; + ms3->role_key = ms3_cmalloc(sizeof(char) * 128); + ms3->role_key[0] = '\0'; + ms3->role_secret = ms3_cmalloc(sizeof(char) * 1024); + ms3->role_secret[0] = '\0'; + // aws says theres no maximum length here.. 2048 might be overkill + ms3->role_session_token = ms3_cmalloc(sizeof(char) * 2048); + ms3->role_session_token[0] = '\0'; + // 0 will uses the default and not set a value in the request + ms3->role_session_duration = 0; + + ret = ms3_assume_role(ms3); + + return ret; +} + +uint8_t ms3_ec2_set_cred(ms3_st *ms3, const char *iam_role, + const char *s3key, const char *s3secret, + const char *token) +{ + uint8_t ret=0; + + if (iam_role == NULL || token == NULL || s3key == NULL || s3secret == NULL) + { + return MS3_ERR_PARAMETER; + } + ms3->iam_role = ms3_cstrdup(iam_role); + ms3->role_key = ms3_cstrdup(s3key); + ms3->role_secret = ms3_cstrdup(s3secret); + ms3->role_session_token = ms3_cstrdup(token); + + return ret; +} + +static void list_free(ms3_st *ms3) +{ + ms3_list_st *list = ms3->list_container.start; + struct ms3_pool_alloc_list_st *plist = NULL, *next = NULL; + while (list) + { + ms3_cfree(list->key); + list = list->next; + } + plist = ms3->list_container.pool_list; + while (plist) + { + next = plist->prev; + ms3_cfree(plist->pool); + ms3_cfree(plist); + plist = next; + } + ms3->list_container.pool = NULL; + ms3->list_container.next = NULL; + ms3->list_container.start = NULL; + ms3->list_container.pool_list = NULL; + ms3->list_container.pool_free = 0; +} + +void ms3_deinit(ms3_st *ms3) +{ + if (!ms3) + { + return; + } + + ms3debug("deinit: 0x%" PRIXPTR, (uintptr_t)ms3); + ms3_cfree(ms3->s3secret); + ms3_cfree(ms3->s3key); + ms3_cfree(ms3->region); + ms3_cfree(ms3->base_domain); + ms3_cfree(ms3->iam_role); + ms3_cfree(ms3->role_key); + ms3_cfree(ms3->role_secret); + ms3_cfree(ms3->role_session_token); + ms3_cfree(ms3->iam_endpoint); + ms3_cfree(ms3->sts_endpoint); + ms3_cfree(ms3->sts_region); + ms3_cfree(ms3->iam_role_arn); + curl_easy_cleanup(ms3->curl); + ms3_cfree(ms3->last_error); + ms3_cfree(ms3->path_buffer); + ms3_cfree(ms3->query_buffer); + list_free(ms3); + ms3_cfree(ms3); +} + +const char *ms3_server_error(ms3_st *ms3) +{ + if (!ms3) + { + return NULL; + } + + return ms3->last_error; +} + +void ms3_debug(void) +{ + bool state = ms3debug_get(); + ms3debug_set(!state); + + if (state) + { + ms3debug("enabling debug"); + } +} + +const char *ms3_error(uint8_t errcode) +{ + if (errcode >= MS3_ERR_MAX) + { + return baderror; + } + + return errmsgs[errcode]; +} + +uint8_t ms3_list_dir(ms3_st *ms3, const char *bucket, const char *prefix, + ms3_list_st **list) +{ + uint8_t res = 0; + + if (!ms3 || !bucket || !list) + { + return MS3_ERR_PARAMETER; + } + + list_free(ms3); + res = execute_request(ms3, MS3_CMD_LIST, bucket, NULL, NULL, NULL, prefix, NULL, + 0, NULL, + NULL); + *list = ms3->list_container.start; + return res; +} + +uint8_t ms3_list(ms3_st *ms3, const char *bucket, const char *prefix, + ms3_list_st **list) +{ + uint8_t res = 0; + + if (!ms3 || !bucket || !list) + { + return MS3_ERR_PARAMETER; + } + + list_free(ms3); + res = execute_request(ms3, MS3_CMD_LIST_RECURSIVE, bucket, NULL, NULL, NULL, + prefix, NULL, + 0, NULL, + NULL); + *list = ms3->list_container.start; + return res; +} + +uint8_t ms3_put(ms3_st *ms3, const char *bucket, const char *key, + const uint8_t *data, size_t length) +{ + uint8_t res; + + if (!ms3 || !bucket || !key || !data) + { + return MS3_ERR_PARAMETER; + } + + if (length == 0) + { + return MS3_ERR_NO_DATA; + } + + // mhash can't hash more than 4GB it seems + if (length > UINT32_MAX) + { + return MS3_ERR_TOO_BIG; + } + + res = execute_request(ms3, MS3_CMD_PUT, bucket, key, NULL, NULL, NULL, data, + length, NULL, + NULL); + + return res; +} + +uint8_t ms3_get(ms3_st *ms3, const char *bucket, const char *key, + uint8_t **data, size_t *length) +{ + uint8_t res = 0; + struct memory_buffer_st buf; + + buf.data = NULL; + buf.length = 0; + + if (!ms3 || !bucket || !key || key[0] == '\0' || !data || !length) + { + return MS3_ERR_PARAMETER; + } + + res = execute_request(ms3, MS3_CMD_GET, bucket, key, NULL, NULL, NULL, NULL, 0, + NULL, &buf); + *data = buf.data; + *length = buf.length; + return res; +} + +uint8_t ms3_copy(ms3_st *ms3, const char *source_bucket, const char *source_key, + const char *dest_bucket, const char *dest_key) +{ + uint8_t res = 0; + + if (!ms3 || !source_bucket || !source_key || !dest_bucket || !dest_key) + { + return MS3_ERR_PARAMETER; + } + + res = execute_request(ms3, MS3_CMD_COPY, dest_bucket, dest_key, source_bucket, + source_key, NULL, NULL, 0, NULL, NULL); + return res; +} + +uint8_t ms3_move(ms3_st *ms3, const char *source_bucket, const char *source_key, + const char *dest_bucket, const char *dest_key) +{ + uint8_t res = 0; + + if (!ms3 || !source_bucket || !source_key || !dest_bucket || !dest_key) + { + return MS3_ERR_PARAMETER; + } + + res = ms3_copy(ms3, source_bucket, source_key, dest_bucket, dest_key); + + if (res) + { + return res; + } + + res = ms3_delete(ms3, source_bucket, source_key); + + return res; +} + +uint8_t ms3_delete(ms3_st *ms3, const char *bucket, const char *key) +{ + uint8_t res; + + if (!ms3 || !bucket || !key) + { + return MS3_ERR_PARAMETER; + } + + res = execute_request(ms3, MS3_CMD_DELETE, bucket, key, NULL, NULL, NULL, NULL, + 0, NULL, + NULL); + return res; +} + +uint8_t ms3_status(ms3_st *ms3, const char *bucket, const char *key, + ms3_status_st *status) +{ + uint8_t res; + + if (!ms3 || !bucket || !key || !status) + { + return MS3_ERR_PARAMETER; + } + + res = execute_request(ms3, MS3_CMD_HEAD, bucket, key, NULL, NULL, NULL, NULL, 0, + NULL, + status); + return res; +} + +void ms3_list_free(ms3_list_st *list) +{ + // Deprecated + (void) list; +} + +void ms3_free(uint8_t *data) +{ + ms3_cfree(data); +} + +uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value) +{ + if (!ms3) + { + return MS3_ERR_PARAMETER; + } + + switch (option) + { + case MS3_OPT_USE_HTTP: + { + ms3->use_http = ms3->use_http ? 0 : 1; + break; + } + + case MS3_OPT_DISABLE_SSL_VERIFY: + { + ms3->disable_verification = ms3->disable_verification ? 0 : 1; + break; + } + + case MS3_OPT_BUFFER_CHUNK_SIZE: + { + size_t new_size; + + if (!value) + { + return MS3_ERR_PARAMETER; + } + + new_size = *(size_t *)value; + + if (new_size < 1) + { + return MS3_ERR_PARAMETER; + } + + ms3->buffer_chunk_size = new_size; + break; + } + + case MS3_OPT_FORCE_LIST_VERSION: + { + uint8_t list_version; + + if (!value) + { + return MS3_ERR_PARAMETER; + } + + list_version = *(uint8_t *)value; + + if (list_version < 1 || list_version > 2) + { + return MS3_ERR_PARAMETER; + } + + ms3->list_version = list_version; + break; + } + + case MS3_OPT_FORCE_PROTOCOL_VERSION: + { + uint8_t protocol_version; + + if (!value) + { + return MS3_ERR_PARAMETER; + } + + protocol_version = *(uint8_t *)value; + + if (protocol_version < 1 || protocol_version > 2) + { + return MS3_ERR_PARAMETER; + } + + ms3->list_version = protocol_version; + break; + } + + case MS3_OPT_PORT_NUMBER: + { + int port_number; + + if (!value) + { + return MS3_ERR_PARAMETER; + } + memcpy(&port_number, (void*)value, sizeof(int)); + + ms3->port = port_number; + break; + } + default: + return MS3_ERR_PARAMETER; + } + + return 0; +} + +uint8_t ms3_assume_role(ms3_st *ms3) +{ + uint8_t res = 0; + + if (!ms3 || !ms3->iam_role) + { + return MS3_ERR_PARAMETER; + } + + if (!strstr(ms3->iam_role_arn, ms3->iam_role)) + { + ms3debug("Lookup IAM role ARN"); + res = execute_assume_role_request(ms3, MS3_CMD_LIST_ROLE, NULL, 0, NULL); + if(res) + { + return res; + } + + } + ms3debug("Assume IAM role"); + res = execute_assume_role_request(ms3, MS3_CMD_ASSUME_ROLE, NULL, 0, NULL); + + return res; +} + diff --git a/storage/maria/libmarias3/src/memory.h b/storage/maria/libmarias3/src/memory.h new file mode 100644 index 00000000..a676a682 --- /dev/null +++ b/storage/maria/libmarias3/src/memory.h @@ -0,0 +1,26 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#pragma once + +extern ms3_malloc_callback ms3_cmalloc; +extern ms3_free_callback ms3_cfree; +extern ms3_realloc_callback ms3_crealloc; +extern ms3_strdup_callback ms3_cstrdup; +extern ms3_calloc_callback ms3_ccalloc; diff --git a/storage/maria/libmarias3/src/request.c b/storage/maria/libmarias3/src/request.c new file mode 100644 index 00000000..26165474 --- /dev/null +++ b/storage/maria/libmarias3/src/request.c @@ -0,0 +1,967 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "config.h" +#include "common.h" +#include "sha256.h" + +#include + +const char *default_domain = "s3.amazonaws.com"; + +static void set_error(ms3_st *ms3, const char *error) +{ + ms3_cfree(ms3->last_error); + + if (!error) + { + ms3->last_error = NULL; + return; + } + + ms3->last_error = ms3_cstrdup(error); +} + +static void set_error_nocopy(ms3_st *ms3, char *error) +{ + ms3_cfree(ms3->last_error); + + if (!error) + { + ms3->last_error = NULL; + return; + } + + ms3->last_error = error; +} + +static uint8_t build_request_uri(CURL *curl, const char *base_domain, + const char *bucket, const char *object, const char *query, bool use_http, + uint8_t protocol_version) +{ + char uri_buffer[MAX_URI_LENGTH]; + const char *domain; + const uint8_t path_parts = 10; // "https://" + "." + "/" + const char *http_protocol = "http"; + const char *https_protocol = "https"; + const char *protocol; + + if (base_domain) + { + domain = base_domain; + } + else + { + domain = default_domain; + } + + if (use_http) + { + protocol = http_protocol; + } + else + { + protocol = https_protocol; + } + + if (query) + { + if (path_parts + strlen(domain) + strlen(bucket) + strlen(object) + strlen( + query) >= MAX_URI_LENGTH - 1) + { + return MS3_ERR_URI_TOO_LONG; + } + + if (protocol_version == 1) + { + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s?%s", protocol, + domain, bucket, + object, query); + } + else + { + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s?%s", protocol, + bucket, domain, + object, query); + } + } + else + { + if (path_parts + strlen(domain) + strlen(bucket) + strlen( + object) >= MAX_URI_LENGTH - 1) + { + return MS3_ERR_URI_TOO_LONG; + } + + if (protocol_version == 1) + { + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s", protocol, + domain, + bucket, + object); + } + else + { + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s", protocol, + bucket, domain, + object); + } + } + + ms3debug("URI: %s", uri_buffer); + curl_easy_setopt(curl, CURLOPT_URL, uri_buffer); + return 0; +} + +/* Handles object name to path conversion. + * Must always start with a '/' even if object is empty. + * Object should be urlencoded. Unfortunately curl also urlencodes slashes. + * So this breaks up on slashes and reassembles the encoded parts. + * Not very efficient but works until we write a custom encoder. + */ + +static char *generate_path(CURL *curl, const char *object, char *path_buffer) +{ + char *tok_ptr = NULL; + char *save_ptr = NULL; + char *out_ptr = path_buffer; + char *path; + + // Keep scanbuild happy + path_buffer[0] = '\0'; + + if (!object) + { + sprintf(path_buffer, "/"); + return path_buffer; + } + + path = ms3_cstrdup(object); // Because strtok_r is destructive + + tok_ptr = strtok_r((char *)path, "/", &save_ptr); + + while (tok_ptr != NULL) + { + char *encoded = curl_easy_escape(curl, tok_ptr, (int)strlen(tok_ptr)); + snprintf(out_ptr, 1024 - (out_ptr - path_buffer), "/%s", encoded); + out_ptr += strlen(encoded) + 1; + curl_free(encoded); + tok_ptr = strtok_r(NULL, "/", &save_ptr); + } + + if (path_buffer[0] != '/') + { + sprintf(path_buffer, "/"); + } + + ms3_cfree(path); + return path_buffer; +} + + +/* At a later date we need to make this accept multi-param/values to support + * pagination + */ + +static char *generate_query(CURL *curl, const char *value, + const char *continuation, uint8_t list_version, bool use_delimiter, + char *query_buffer) +{ + char *encoded; + query_buffer[0] = '\0'; + + if (use_delimiter) + { + snprintf(query_buffer, 3072, "delimiter=%%2F"); + } + + if (list_version == 2) + { + if (continuation) + { + encoded = curl_easy_escape(curl, continuation, (int)strlen(continuation)); + + if (strlen(query_buffer)) + { + snprintf(query_buffer + strlen(query_buffer), 3072 - strlen(query_buffer), + "&continuation-token=%s&list-type=2", encoded); + } + else + { + snprintf(query_buffer, 3072, "continuation-token=%s&list-type=2", encoded); + } + + curl_free(encoded); + } + else + { + if (strlen(query_buffer)) + { + snprintf(query_buffer + strlen(query_buffer), 3072 - strlen(query_buffer), + "&list-type=2"); + } + else + { + sprintf(query_buffer, "list-type=2"); + } + } + } + else if (continuation) + { + // Continuation is really marker here + encoded = curl_easy_escape(curl, continuation, (int)strlen(continuation)); + + if (strlen(query_buffer)) + { + snprintf(query_buffer + strlen(query_buffer), 3072 - strlen(query_buffer), + "&marker=%s", + encoded); + } + else + { + snprintf(query_buffer, 3072, "marker=%s", encoded); + } + + curl_free(encoded); + } + + if (value) + { + encoded = curl_easy_escape(curl, value, (int)strlen(value)); + + if (strlen(query_buffer)) + { + snprintf(query_buffer + strlen(query_buffer), 3072 - strlen(query_buffer), + "&prefix=%s", + encoded); + } + else + { + snprintf(query_buffer, 3072, "prefix=%s", + encoded); + } + + curl_free(encoded); + } + + return query_buffer; +} + + +/* +\n +\n +\n +\n +\n - host;x-amz-content-sha256;x-amz-date + - empty if no POST data +*/ +static uint8_t generate_request_hash(uri_method_t method, const char *path, + const char *bucket, + const char *query, char *post_hash, struct curl_slist *headers, bool has_source, bool has_token, + char *return_hash) +{ + char signing_data[3072]; + size_t pos = 0; + uint8_t sha256hash[32]; // SHA_256 binary length + uint8_t hash_pos = 0; + uint8_t i; + struct curl_slist *current_header = headers; + + // Method first + switch (method) + { + case MS3_GET: + { + sprintf(signing_data, "GET\n"); + pos += 4; + break; + } + + case MS3_HEAD: + { + sprintf(signing_data, "HEAD\n"); + pos += 5; + break; + } + + case MS3_PUT: + { + sprintf(signing_data, "PUT\n"); + pos += 4; + break; + } + + case MS3_DELETE: + { + sprintf(signing_data, "DELETE\n"); + pos += 7; + break; + } + + default: + { + ms3debug("Bad method detected"); + return MS3_ERR_IMPOSSIBLE; + } + } + + // URL path + if (bucket) + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, "/%s%s\n", bucket, + path); + pos += strlen(path) + strlen(bucket) + 2; + } + else + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, "%s\n", path); + pos += strlen(path) + 1; + } + + // URL query (if exists) + if (query) + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, "%s\n", query); + pos += strlen(query) + 1; + } + else + { + sprintf(signing_data + pos, "\n"); + pos++; + } + + do + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, "%s\n", + current_header->data); + pos += strlen(current_header->data) + 1; + } + while ((current_header = current_header->next)); + + // List if header names + // The newline between headers and this is important + if (has_source && has_token) + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, + "\nhost;x-amz-content-sha256;x-amz-copy-source;x-amz-date;x-amz-security-token\n"); + pos += 77; + } + else if (has_source) + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, + "\nhost;x-amz-content-sha256;x-amz-copy-source;x-amz-date\n"); + pos += 56; + } + else if (has_token) + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, + "\nhost;x-amz-content-sha256;x-amz-date;x-amz-security-token\n"); + pos += 59; + } + else + { + snprintf(signing_data + pos, sizeof(signing_data) - pos, + "\nhost;x-amz-content-sha256;x-amz-date\n"); + pos += 38; + } + + // Hash of post data (can be hash of empty) + snprintf(signing_data + pos, sizeof(signing_data) - pos, "%.*s", 64, post_hash); + //pos+= 64; + ms3debug("Signature data1: %s", signing_data); + + // Hash all of the above + sha256((uint8_t *)signing_data, strlen(signing_data), (uint8_t *)sha256hash); + + for (i = 0; i < 32; i++) + { + sprintf(return_hash + hash_pos, "%.2x", sha256hash[i]); + hash_pos += 2; + } + + ms3debug("Signature data: %s", signing_data); + ms3debug("Signature: %.*s", 64, return_hash); + + return 0; +} + +static uint8_t build_request_headers(CURL *curl, struct curl_slist **head, + const char *base_domain, const char *region, const char *key, + const char *secret, const char *object, const char *query, + uri_method_t method, const char *bucket, const char *source_bucket, + const char *source_key, struct put_buffer_st *post_data, + uint8_t protocol_version, const char *session_token) +{ + uint8_t ret = 0; + time_t now; + struct tm tmp_tm; + char headerbuf[3072]; + char secrethead[45]; + char date[9]; + char sha256hash[65]; + char post_hash[65]; + uint8_t tmp_hash[32]; + // Alternate between these two so hmac doesn't overwrite itself + uint8_t hmac_hash[32]; + uint8_t hmac_hash2[32]; + uint8_t hash_pos = 0; + const char *domain; + struct curl_slist *headers = NULL; + uint8_t offset; + uint8_t i; + bool has_source = false; + bool has_token = false; + struct curl_slist *current_header; + + // Host header + if (base_domain) + { + domain = base_domain; + } + else + { + domain = default_domain; + } + + if (protocol_version == 2) + { + snprintf(headerbuf, sizeof(headerbuf), "host:%s.%s", bucket, domain); + } + else + { + snprintf(headerbuf, sizeof(headerbuf), "host:%s", domain); + } + headers = curl_slist_append(headers, headerbuf); + *head = headers; + + // Hash post data + sha256(post_data->data, post_data->length, tmp_hash); + + for (i = 0; i < 32; i++) + { + sprintf(post_hash + hash_pos, "%.2x", tmp_hash[i]); + hash_pos += 2; + } + + snprintf(headerbuf, sizeof(headerbuf), "x-amz-content-sha256:%.*s", 64, + post_hash); + headers = curl_slist_append(headers, headerbuf); + + if (source_bucket) + { + char *bucket_escape; + char *key_escape; + bucket_escape = curl_easy_escape(curl, source_bucket, (int)strlen(source_bucket)); + key_escape = curl_easy_escape(curl, source_key, (int)strlen(source_key)); + snprintf(headerbuf, sizeof(headerbuf), "x-amz-copy-source:/%s/%s", + bucket_escape, key_escape); + headers = curl_slist_append(headers, headerbuf); + ms3_cfree(bucket_escape); + ms3_cfree(key_escape); + } + + // Date/time header + time(&now); + snprintf(headerbuf, sizeof(headerbuf), "x-amz-date:"); + offset = strlen(headerbuf); + gmtime_r(&now, &tmp_tm); + strftime(headerbuf + offset, sizeof(headerbuf) - offset, "%Y%m%dT%H%M%SZ", + &tmp_tm); + headers = curl_slist_append(headers, headerbuf); + + // Temp Credentials Security Token + if (session_token) + { + snprintf(headerbuf, sizeof(headerbuf), "x-amz-security-token:%s",session_token); + headers = curl_slist_append(headers, headerbuf); + has_token = true; + } + + if (source_bucket) + { + has_source = true; + } + + // Builds the request hash + if (protocol_version == 1) + { + ret = generate_request_hash(method, object, bucket, query, post_hash, headers, + has_source, has_token, + sha256hash); + } + else + { + ret = generate_request_hash(method, object, NULL, query, post_hash, headers, + has_source, has_token, + sha256hash); + } + + if (ret) + { + return ret; + } + + // User signing key hash + // Date hashed using AWS4:secret_key + snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", 40, secret); + strftime(headerbuf, sizeof(headerbuf), "%Y%m%d", &tmp_tm); + hmac_sha256((uint8_t *)secrethead, strlen(secrethead), (uint8_t *)headerbuf, + strlen(headerbuf), hmac_hash); + + // Region signed by above key + hmac_sha256(hmac_hash, 32, (uint8_t *)region, strlen(region), + hmac_hash2); + + // Service signed by above key (s3 always) + sprintf(headerbuf, "s3"); + hmac_sha256(hmac_hash2, 32, (uint8_t *)headerbuf, strlen(headerbuf), + hmac_hash); + + // Request version signed by above key (always "aws4_request") + sprintf(headerbuf, "aws4_request"); + hmac_sha256(hmac_hash, 32, (uint8_t *)headerbuf, strlen(headerbuf), + hmac_hash2); + + // Sign everything with the key + snprintf(headerbuf, sizeof(headerbuf), "AWS4-HMAC-SHA256\n"); + offset = strlen(headerbuf); + strftime(headerbuf + offset, sizeof(headerbuf) - offset, "%Y%m%dT%H%M%SZ\n", + &tmp_tm); + offset = strlen(headerbuf); + strftime(date, 9, "%Y%m%d", &tmp_tm); + snprintf(headerbuf + offset, sizeof(headerbuf) - offset, + "%.*s/%s/s3/aws4_request\n%.*s", 8, date, region, 64, sha256hash); + ms3debug("Data to sign: %s", headerbuf); + hmac_sha256(hmac_hash2, 32, (uint8_t *)headerbuf, strlen(headerbuf), + hmac_hash); + + hash_pos = 0; + + for (i = 0; i < 32; i++) + { + sprintf(sha256hash + hash_pos, "%.2x", hmac_hash[i]); + hash_pos += 2; + } + + // Make auth header + if (source_bucket && session_token) + { + snprintf(headerbuf, sizeof(headerbuf), + "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-date;x-amz-security-token;x-amz-copy-source, Signature=%s", + key, date, region, sha256hash); + } + else if (source_bucket) + { + snprintf(headerbuf, sizeof(headerbuf), + "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-date, Signature=%s", + key, date, region, sha256hash); + } + else if (session_token) + { + snprintf(headerbuf, sizeof(headerbuf), + "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=%s", + key, date, region, sha256hash); + } + else + { + snprintf(headerbuf, sizeof(headerbuf), + "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=%s", + key, date, region, sha256hash); + } + + headers = curl_slist_append(headers, headerbuf); + + // Disable this header or PUT will barf with a 501 + sprintf(headerbuf, "Transfer-Encoding:"); + headers = curl_slist_append(headers, headerbuf); + + if ((method == MS3_PUT) && !source_bucket) + { + snprintf(headerbuf, sizeof(headerbuf), "Content-Length:%zu", post_data->length); + headers = curl_slist_append(headers, headerbuf); + } + + current_header = headers; + + do + { + ms3debug("Header: %s", current_header->data); + } + while ((current_header = current_header->next)); + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + switch (method) + { + case MS3_GET: + { + // Nothing extra to do here + break; + } + + case MS3_HEAD: + { + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + break; + } + + case MS3_PUT: + { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT"); + break; + } + + case MS3_DELETE: + { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + break; + } + + default: + ms3debug("Bad method detected"); + return MS3_ERR_IMPOSSIBLE; + } + + return 0; +} +static size_t header_callback(char *buffer, size_t size, + size_t nitems, void *userdata) +{ + ms3debug("%.*s\n", (int)(nitems * size), buffer); + + if (userdata) + { + // HEAD request + if (!strncasecmp(buffer, "Last-Modified", 13)) + { + ms3_status_st *status = (ms3_status_st *) userdata; + // Date/time, format: Fri, 15 Mar 2019 16:58:54 GMT + struct tm ttmp = {0}; + strptime(buffer + 15, "%a, %d %b %Y %H:%M:%S %Z", &ttmp); + status->created = mktime(&ttmp); + } + else if (!strncasecmp(buffer, "Content-Length", 14)) + { + ms3_status_st *status = (ms3_status_st *) userdata; + // Length + status->length = strtoull(buffer + 16, NULL, 10); + } + } + + return nitems * size; +} + +static size_t body_callback(void *buffer, size_t size, + size_t nitems, void *userdata) +{ + uint8_t *ptr; + size_t realsize = nitems * size; + + struct memory_buffer_st *mem = (struct memory_buffer_st *)userdata; + + if (realsize + mem->length >= mem->alloced) + { + size_t additional_size = mem->buffer_chunk_size; + + if (realsize >= mem->buffer_chunk_size) + { + additional_size = (ceil((double)realsize / (double)mem->buffer_chunk_size) + 1) + * mem->buffer_chunk_size; + } + + ptr = (uint8_t *)ms3_crealloc(mem->data, mem->alloced + additional_size); + + if (!ptr) + { + ms3debug("Curl response OOM"); + return 0; + } + + mem->alloced += additional_size; + mem->data = ptr; + } + + memcpy(&(mem->data[mem->length]), buffer, realsize); + mem->length += realsize; + mem->data[mem->length] = '\0'; + + ms3debug("Read %zu bytes, buffer %zu bytes", realsize, mem->length); +// ms3debug("Data: %s", (char*)buffer); + return nitems * size; +} + +uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, + const char *object, const char *source_bucket, const char *source_object, + const char *filter, const uint8_t *data, size_t data_size, + char *continuation, + void *ret_ptr) +{ + CURL *curl = NULL; + struct curl_slist *headers = NULL; + uint8_t res = 0; + struct memory_buffer_st mem; + uri_method_t method; + char *path = NULL; + char *query = NULL; + struct put_buffer_st post_data; + CURLcode curl_res; + long response_code = 0; + + mem.data = NULL; + mem.length = 0; + mem.alloced = 1; + mem.buffer_chunk_size = ms3->buffer_chunk_size; + + post_data.data = (uint8_t *) data; + post_data.length = data_size; + post_data.offset = 0; + + curl = ms3->curl; + + if (!ms3->first_run) + { + curl_easy_reset(curl); + } + else + { + ms3->first_run = false; + } + + path = generate_path(curl, object, ms3->path_buffer); + + if (cmd == MS3_CMD_LIST_RECURSIVE) + { + query = generate_query(curl, filter, continuation, ms3->list_version, false, + ms3->query_buffer); + } + else if (cmd == MS3_CMD_LIST) + { + query = generate_query(curl, filter, continuation, ms3->list_version, true, + ms3->query_buffer); + } + + res = build_request_uri(curl, ms3->base_domain, bucket, path, query, + ms3->use_http, ms3->protocol_version); + + if (res) + { + return res; + } + + switch (cmd) + { + case MS3_CMD_COPY: + case MS3_CMD_PUT: + method = MS3_PUT; + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (char *)data); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data_size); + break; + + case MS3_CMD_DELETE: + method = MS3_DELETE; + break; + + case MS3_CMD_HEAD: + method = MS3_HEAD; + curl_easy_setopt(curl, CURLOPT_HEADERDATA, ret_ptr); + break; + + case MS3_CMD_LIST: + case MS3_CMD_LIST_RECURSIVE: + case MS3_CMD_GET: + case MS3_CMD_LIST_ROLE: + method = MS3_GET; + break; + + case MS3_CMD_ASSUME_ROLE: + default: + ms3debug("Bad cmd detected"); + ms3_cfree(mem.data); + + return MS3_ERR_IMPOSSIBLE; + } + + if (ms3->iam_role) + { + ms3debug("Using assumed role: %s",ms3->iam_role); + res = build_request_headers(curl, &headers, ms3->base_domain, ms3->region, + ms3->role_key, ms3->role_secret, path, query, method, bucket, source_bucket, + source_object, &post_data, ms3->protocol_version, ms3->role_session_token); + } + else + { + res = build_request_headers(curl, &headers, ms3->base_domain, ms3->region, + ms3->s3key, ms3->s3secret, path, query, method, bucket, source_bucket, + source_object, &post_data, ms3->protocol_version, NULL); + } + if (res) + { + ms3_cfree(mem.data); + curl_slist_free_all(headers); + + return res; + } + + if (ms3->disable_verification) + { + ms3debug("Disabling SSL verification"); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); + } + + if (ms3->port) + curl_easy_setopt(curl, CURLOPT_PORT, (long)ms3->port); + + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); + curl_res = curl_easy_perform(curl); + + if (curl_res != CURLE_OK) + { + ms3debug("Curl error: %s", curl_easy_strerror(curl_res)); + set_error(ms3, curl_easy_strerror(curl_res)); + ms3_cfree(mem.data); + curl_slist_free_all(headers); + + return MS3_ERR_REQUEST_ERROR; + } + + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + ms3debug("Response code: %ld", response_code); + + if (response_code == 404) + { + char *message = parse_error_message((char *)mem.data, mem.length); + + if (message) + { + ms3debug("Response message: %s", message); + } + + set_error_nocopy(ms3, message); + res = MS3_ERR_NOT_FOUND; + } + else if (response_code == 403) + { + char *message = parse_error_message((char *)mem.data, mem.length); + + if (message) + { + ms3debug("Response message: %s", message); + } + + set_error_nocopy(ms3, message); + res = MS3_ERR_AUTH; + } + else if (response_code >= 400) + { + char *message = parse_error_message((char *)mem.data, mem.length); + + if (message) + { + ms3debug("Response message: %s", message); + } + + set_error_nocopy(ms3, message); + res = MS3_ERR_SERVER; + if (ms3->iam_role) + { + res = MS3_ERR_AUTH_ROLE; + } + } + + switch (cmd) + { + case MS3_CMD_LIST_RECURSIVE: + case MS3_CMD_LIST: + { + char *cont = NULL; + parse_list_response((const char *)mem.data, mem.length, &ms3->list_container, ms3->list_version, + &cont); + + if (cont) + { + res = execute_request(ms3, cmd, bucket, object, source_bucket, source_object, + filter, data, data_size, cont, + NULL); + if (res) + { + return res; + } + + ms3_cfree(cont); + } + + ms3_cfree(mem.data); + break; + } + + case MS3_CMD_COPY: + case MS3_CMD_PUT: + { + ms3_cfree(mem.data); + break; + } + + case MS3_CMD_GET: + { + struct memory_buffer_st *buf = (struct memory_buffer_st *) ret_ptr; + + if (res) + { + ms3_cfree(mem.data); + } + else + { + buf->data = mem.data; + buf->length = mem.length; + } + + break; + } + + case MS3_CMD_DELETE: + { + ms3_cfree(mem.data); + break; + } + + case MS3_CMD_HEAD: + { + ms3_cfree(mem.data); + break; + } + + case MS3_CMD_LIST_ROLE: + case MS3_CMD_ASSUME_ROLE: + default: + { + ms3_cfree(mem.data); + ms3debug("Bad cmd detected"); + res = MS3_ERR_IMPOSSIBLE; + } + } + + curl_slist_free_all(headers); + + return res; +} diff --git a/storage/maria/libmarias3/src/request.h b/storage/maria/libmarias3/src/request.h new file mode 100644 index 00000000..9ce8bb5c --- /dev/null +++ b/storage/maria/libmarias3/src/request.h @@ -0,0 +1,63 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#pragma once + +#include "config.h" +#include +#include + +// Maxmum S3 file size is 1024 bytes so for protection we make the maximum +// URI length this +#define MAX_URI_LENGTH 1024 + +#define READ_BUFFER_DEFAULT_SIZE 1024*1024 + +enum uri_method_t +{ + MS3_GET, + MS3_HEAD, + MS3_PUT, + MS3_DELETE +}; + +typedef enum uri_method_t uri_method_t; + +enum command_t +{ + MS3_CMD_LIST, + MS3_CMD_LIST_RECURSIVE, + MS3_CMD_PUT, + MS3_CMD_GET, + MS3_CMD_DELETE, + MS3_CMD_HEAD, + MS3_CMD_COPY, + MS3_CMD_LIST_ROLE, + MS3_CMD_ASSUME_ROLE +}; + +typedef enum command_t command_t; + +struct ms3_st; + +uint8_t execute_request(ms3_st *ms3, command_t command, const char *bucket, + const char *object, const char *source_bucket, const char *source_object, + const char *filter, const uint8_t *data, size_t data_size, + char *continuation, + void *ret_ptr); diff --git a/storage/maria/libmarias3/src/response.c b/storage/maria/libmarias3/src/response.c new file mode 100644 index 00000000..4e976aba --- /dev/null +++ b/storage/maria/libmarias3/src/response.c @@ -0,0 +1,512 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "config.h" +#include "common.h" + +#include "xml.h" + +char *parse_error_message(const char *data, size_t length) +{ + struct xml_document *doc = NULL; + struct xml_node *node = NULL; + struct xml_node *child = NULL; + struct xml_node *root = NULL; + + uint64_t node_it = 0; + + if (!data || !length) + { + return NULL; + } + + doc = xml_parse_document((uint8_t*)data, length); + + if (!doc) + { + return NULL; + } + + root = xml_document_root(doc); + + // First node is Error + child = xml_node_child(root, node_it); + // IAM / STS This will be Error and we need next child + if (!xml_node_name_cmp(child, "Error")) + { + node = xml_node_child(child, node_it); + } + else + { + node = child; + child = root; + } + + if (!node) + { + xml_document_free(doc, false); + return NULL; + } + + while(node) + { + if (!xml_node_name_cmp(node, "Message")) + { + struct xml_string *content = xml_node_content(node); + uint8_t *message = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, message, xml_string_length(content)); + xml_document_free(doc, false); + return (char *)message; + } + + node_it++; + node = xml_node_child(child, node_it); + } + + xml_document_free(doc, false); + return NULL; +} + +static ms3_list_st *get_next_list_ptr(struct ms3_list_container_st *container) +{ + ms3_list_st *new_alloc = NULL; + struct ms3_pool_alloc_list_st *new_pool_next = NULL; + struct ms3_pool_alloc_list_st *new_pool_prev = NULL; + ms3_list_st *ret = NULL; + if (container->pool_free == 0) + { + new_alloc = (ms3_list_st*)ms3_cmalloc(sizeof(ms3_list_st) * 1024); + new_pool_next = (struct ms3_pool_alloc_list_st*)ms3_cmalloc(sizeof(struct ms3_pool_alloc_list_st)); + + if (!new_alloc || !new_pool_next) + { + ms3debug("List realloc OOM"); + return NULL; + } + + new_pool_prev = container->pool_list; + container->pool_list = new_pool_next; + if (new_pool_prev) + { + container->pool_list->prev = new_pool_prev; + } + else + { + container->pool_list->prev = NULL; + } + container->pool_list->pool = new_alloc; + + container->pool_free = 1024; + if (!container->start) + { + container->start = new_alloc; + } + container->pool = container->next = new_alloc; + } + else + { + container->next++; + } + ret = container->next; + container->pool_free--; + return ret; +} + +uint8_t parse_list_response(const char *data, size_t length, struct ms3_list_container_st *list_container, + uint8_t list_version, + char **continuation) +{ + struct xml_document *doc; + struct xml_node *root; + struct xml_node *node; + struct xml_node *child; + char *filename = NULL; + char *filesize = NULL; + char *filedate = NULL; + size_t size = 0; + struct tm ttmp = {0}; + time_t tout = 0; + bool truncated = false; + const char *last_key = NULL; + ms3_list_st *nextptr = NULL, *lastptr = list_container->next; + uint64_t node_it = 0; + + // Empty list + if (!data || !length) + { + return 0; + } + + doc = xml_parse_document((uint8_t*)data, length); + + if (!doc) + { + return MS3_ERR_RESPONSE_PARSE; + } + + /* For version 1: + * If IsTruncated is set, get the last key in the list, this will be used as + * "marker" in the next request. + * For version 2: + * If NextContinuationToken is set, use this for the next request + * + * We use the "continuation" return value for both + */ + + root = xml_document_root(doc); + // First node is ListBucketResponse + node = xml_node_child(root, 0); + + do + { + if (!xml_node_name_cmp(node, "NextContinuationToken")) + { + struct xml_string *content = xml_node_content(node); + *continuation = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, (uint8_t*)*continuation, xml_string_length(content)); + continue; + } + + if (list_version == 1) + { + if (!xml_node_name_cmp(node, "IsTruncated")) + { + struct xml_string *content = xml_node_content(node); + char *trunc_value = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, (uint8_t*)trunc_value, xml_string_length(content)); + + if (!strcmp(trunc_value, "true")) + { + truncated = true; + } + + ms3_cfree(trunc_value); + continue; + } + } + + if (!xml_node_name_cmp(node, "Contents")) + { + bool skip = false; + uint64_t child_it = 0; + // Found contents + child = xml_node_child(node, 0); + + do + { + if (!xml_node_name_cmp(child, "Key")) + { + struct xml_string *content = xml_node_content(child); + filename = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, (uint8_t*)filename, xml_string_length(content)); + + ms3debug("Filename: %s", filename); + + if (filename[strlen((const char *)filename) - 1] == '/') + { + skip = true; + ms3_cfree(filename); + break; + } + + continue; + } + + if (!xml_node_name_cmp(child, "Size")) + { + struct xml_string *content = xml_node_content(child); + filesize = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, (uint8_t*)filesize, xml_string_length(content)); + + ms3debug("Size: %s", filesize); + size = strtoull((const char *)filesize, NULL, 10); + ms3_cfree(filesize); + continue; + } + + if (!xml_node_name_cmp(child, "LastModified")) + { + struct xml_string *content = xml_node_content(child); + filedate = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, (uint8_t*)filedate, xml_string_length(content)); + + ms3debug("Date: %s", filedate); + strptime((const char *)filedate, "%Y-%m-%dT%H:%M:%SZ", &ttmp); + tout = mktime(&ttmp); + ms3_cfree(filedate); + continue; + } + } + while ((child = xml_node_child(node, ++child_it))); + + if (!skip) + { + nextptr = get_next_list_ptr(list_container); + nextptr->next = NULL; + + if (lastptr) + { + lastptr->next = nextptr; + } + lastptr = nextptr; + + if (filename) + { + nextptr->key = (char *)filename; + + if (list_version == 1) + { + last_key = nextptr->key; + } + } + else + { + nextptr->key = NULL; + } + + nextptr->length = size; + nextptr->created = tout; + } + + continue; + } + + if (!xml_node_name_cmp(node, "CommonPrefixes")) + { + child = xml_node_child(node, 0); + + if (!xml_node_name_cmp(child, "Prefix")) + { + struct xml_string *content = xml_node_content(child); + filename = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, (uint8_t*)filename, xml_string_length(content)); + + ms3debug("Filename: %s", filename); + nextptr = get_next_list_ptr(list_container); + nextptr->next = NULL; + + if (lastptr) + { + lastptr->next = nextptr; + } + lastptr = nextptr; + + nextptr->key = (char *)filename; + nextptr->length = 0; + nextptr->created = 0; + } + } + + } + while ((node = xml_node_child(root, ++node_it))); + + if (list_version == 1 && truncated && last_key) + { + *continuation = ms3_cstrdup(last_key); + } + + xml_document_free(doc, false); + return 0; +} + +uint8_t parse_role_list_response(const char *data, size_t length, char *role_name, char *arn, char **continuation) +{ + struct xml_document *doc; + struct xml_node *root; + struct xml_node *list_role_result; + struct xml_node *child; + struct xml_node *roles; + struct xml_node *member; + + char *response_role_name = NULL; + char *response_role_arn = NULL; + uint64_t node_it = 0; + + // Empty list + if (!data || !length) + { + return 0; + } + + doc = xml_parse_document((uint8_t*)data, length); + + if (!doc) + { + return MS3_ERR_RESPONSE_PARSE; + } + + root = xml_document_root(doc); + // First node is listRoleResponse + list_role_result = xml_node_child(root, 0); + child = xml_node_child(list_role_result, 0); + + do + { + + if (!xml_node_name_cmp(child, "Marker")) + { + struct xml_string *content = xml_node_content(child); + *continuation = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, (uint8_t*)*continuation, xml_string_length(content)); + continue; + } + + if (!xml_node_name_cmp(child, "Roles")) + { + uint64_t child_it = 0; + // Found contents + roles = xml_node_child(child, 0); + do + { + // go down one more child to get members + uint64_t roles_it = 0; + member = xml_node_child(roles, 0); + do + { + if (!xml_node_name_cmp(member, "RoleName")) + { + struct xml_string *content = xml_node_content(member); + response_role_name = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, (uint8_t*)response_role_name, xml_string_length(content)); + continue; + } + if (!xml_node_name_cmp(member, "Arn")) + { + struct xml_string *content = xml_node_content(member); + response_role_arn = ms3_cmalloc(xml_string_length(content) + 1); + xml_string_copy(content, (uint8_t*)response_role_arn, xml_string_length(content)); + continue; + } + } + while ((member = xml_node_child(roles, ++roles_it))); + if (!strcmp(response_role_name, role_name)) + { + ms3debug("Role Found ARN = %s",response_role_arn); + sprintf(arn, "%s", response_role_arn); + ms3_cfree(response_role_name); + ms3_cfree(response_role_arn); + xml_document_free(doc, false); + return MS3_ERR_NONE; + } + ms3_cfree(response_role_name); + ms3_cfree(response_role_arn); + } + while ((roles = xml_node_child(child, ++child_it))); + } + } + while ((child = xml_node_child(list_role_result, ++node_it))); + + xml_document_free(doc, false); + return MS3_ERR_NOT_FOUND; +} + +uint8_t parse_assume_role_response(const char *data, size_t length, char *assume_role_key, char *assume_role_secret, char *assume_role_token) +{ + struct xml_document *doc; + struct xml_node *root; + struct xml_node *assume_role_result; + struct xml_node *child; + struct xml_node *credentials; + uint64_t node_it = 0; + + // Empty list + if (!data || !length) + { + return 0; + } + + doc = xml_parse_document((uint8_t*)data, length); + + if (!doc) + { + return MS3_ERR_RESPONSE_PARSE; + } + + root = xml_document_root(doc); + // First node is AssumeRoleResponse + assume_role_result = xml_node_child(root, 0); + child = xml_node_child(assume_role_result, 0); + + do + { + if (!xml_node_name_cmp(child, "Credentials")) + { + uint64_t child_it = 0; + // Found contents + credentials = xml_node_child(child, 0); + do + { + if (!xml_node_name_cmp(credentials, "AccessKeyId")) + { + struct xml_string *content = xml_node_content(credentials); + size_t content_length = xml_string_length(content); + assume_role_key[0] = '\0'; + + if (content_length >= 128) + { + ms3debug("AccessKeyId error length = %zu", content_length); + xml_document_free(doc, false); + return MS3_ERR_AUTH_ROLE; + } + xml_string_copy(content, (uint8_t*)assume_role_key, content_length); + + continue; + } + if (!xml_node_name_cmp(credentials, "SecretAccessKey")) + { + struct xml_string *content = xml_node_content(credentials); + size_t content_length = xml_string_length(content); + assume_role_secret[0] = '\0'; + + if (content_length >= 1024) + { + ms3debug("SecretAccessKey error length = %zu", content_length); + xml_document_free(doc, false); + return MS3_ERR_AUTH_ROLE; + } + xml_string_copy(content, (uint8_t*)assume_role_secret, content_length); + + continue; + } + if (!xml_node_name_cmp(credentials, "SessionToken")) + { + struct xml_string *content = xml_node_content(credentials); + size_t content_length = xml_string_length(content); + assume_role_token[0] = '\0'; + + if (content_length >= 2048) + { + ms3debug("SessionToken error length = %zu", content_length); + xml_document_free(doc, false); + return MS3_ERR_AUTH_ROLE; + } + xml_string_copy(content, (uint8_t*)assume_role_token, content_length); + + continue; + } + } + while ((credentials = xml_node_child(child, ++child_it))); + } + } + while ((child = xml_node_child(assume_role_result, ++node_it))); + + xml_document_free(doc, false); + + return MS3_ERR_NONE; +} diff --git a/storage/maria/libmarias3/src/response.h b/storage/maria/libmarias3/src/response.h new file mode 100644 index 00000000..1d8dd907 --- /dev/null +++ b/storage/maria/libmarias3/src/response.h @@ -0,0 +1,32 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#pragma once + +#include "config.h" +#include + +char *parse_error_message(const char *data, size_t length); + +uint8_t parse_list_response(const char *data, size_t length, + struct ms3_list_container_st *list_container, uint8_t list_version, char **continuation); + +uint8_t parse_role_list_response(const char *data, size_t length, char *role_name, char* arn, char **continuation); + +uint8_t parse_assume_role_response(const char *data, size_t length, char *assume_role_key, char *assume_role_secret, char *assume_role_token); diff --git a/storage/maria/libmarias3/src/sha256-internal.c b/storage/maria/libmarias3/src/sha256-internal.c new file mode 100644 index 00000000..3be8aa3c --- /dev/null +++ b/storage/maria/libmarias3/src/sha256-internal.c @@ -0,0 +1,251 @@ +/* + * SHA-256 hash implementation and interface functions + * Copyright (c) 2003-2011, Jouni Malinen + * + * This software may be distributed under the terms of the BSD license. + * See README for more details. + */ + +#include "sha256.h" +#include "sha256_i.h" + + +/** + * sha256_vector - SHA256 hash for data vector + * @num_elem: Number of elements in the data vector + * @addr: Pointers to the data areas + * @len: Lengths of the data blocks + * @mac: Buffer for the hash + * Returns: 0 on success, -1 of failure + */ +int sha256_vector(size_t num_elem, const uint8_t *addr[], const size_t *len, + uint8_t *mac) +{ + struct sha256_state ctx; + size_t i; + + sha256_init(&ctx); + + for (i = 0; i < num_elem; i++) + if (sha256_process(&ctx, addr[i], len[i])) + return -1; + + if (sha256_done(&ctx, mac)) + return -1; + + return 0; +} + + +/* ===== start - public domain SHA256 implementation ===== */ + +/* This is based on SHA256 implementation in LibTomCrypt that was released into + * public domain by Tom St Denis. */ + +/* the K array */ +static const uint32_t K[64] = +{ + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, + 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, + 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, + 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, + 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, + 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, + 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, + 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, + 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, + 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; + + +/* Various logical functions */ +#define RORc(x, y) \ +( ((((uint32_t) (x) & 0xFFFFFFFFUL) >> (uint32_t) ((y) & 31)) | \ + ((uint32_t) (x) << (uint32_t) (32 - ((y) & 31)))) & 0xFFFFFFFFUL) +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) RORc((x), (n)) +#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) +#ifndef MIN +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) +#endif + +/* compress 512-bits */ +static int sha256_compress(struct sha256_state *md, unsigned char *buf) +{ + uint32_t S[8], W[64]; + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) + { + S[i] = md->state[i]; + } + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) + W[i] = WPA_GET_BE32(buf + (4 * i)); + + /* fill W[16..63] */ + for (i = 16; i < 64; i++) + { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + + W[i - 16]; + } + + /* Compress */ +#define RND(a,b,c,d,e,f,g,h,i) \ + uint32_t t0, t1; \ + t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + for (i = 0; i < 64; ++i) + { + uint32_t t; + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], i); + t = S[7]; + S[7] = S[6]; + S[6] = S[5]; + S[5] = S[4]; + S[4] = S[3]; + S[3] = S[2]; + S[2] = S[1]; + S[1] = S[0]; + S[0] = t; + } + + /* feedback */ + for (i = 0; i < 8; i++) + { + md->state[i] = md->state[i] + S[i]; + } + + return 0; +} + + +/* Initialize the hash state */ +void sha256_init(struct sha256_state *md) +{ + md->curlen = 0; + md->length = 0; + md->state[0] = 0x6A09E667UL; + md->state[1] = 0xBB67AE85UL; + md->state[2] = 0x3C6EF372UL; + md->state[3] = 0xA54FF53AUL; + md->state[4] = 0x510E527FUL; + md->state[5] = 0x9B05688CUL; + md->state[6] = 0x1F83D9ABUL; + md->state[7] = 0x5BE0CD19UL; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +int sha256_process(struct sha256_state *md, const unsigned char *in, + unsigned long inlen) +{ + unsigned long n; + + if (md->curlen >= sizeof(md->buf)) + return -1; + + while (inlen > 0) + { + if (md->curlen == 0 && inlen >= SHA256_BLOCK_SIZE) + { + if (sha256_compress(md, (unsigned char *) in) < 0) + return -1; + + md->length += SHA256_BLOCK_SIZE * 8; + in += SHA256_BLOCK_SIZE; + inlen -= SHA256_BLOCK_SIZE; + } + else + { + n = MIN(inlen, (SHA256_BLOCK_SIZE - md->curlen)); + memcpy(md->buf + md->curlen, in, n); + md->curlen += n; + in += n; + inlen -= n; + + if (md->curlen == SHA256_BLOCK_SIZE) + { + if (sha256_compress(md, md->buf) < 0) + return -1; + + md->length += 8 * SHA256_BLOCK_SIZE; + md->curlen = 0; + } + } + } + + return 0; +} + + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (32 bytes) + @return CRYPT_OK if successful +*/ +int sha256_done(struct sha256_state *md, unsigned char *out) +{ + int i; + + if (md->curlen >= sizeof(md->buf)) + return -1; + + /* increase the length of the message */ + md->length += md->curlen * 8; + + /* append the '1' bit */ + md->buf[md->curlen++] = (unsigned char) 0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->curlen > 56) + { + while (md->curlen < SHA256_BLOCK_SIZE) + { + md->buf[md->curlen++] = (unsigned char) 0; + } + + sha256_compress(md, md->buf); + md->curlen = 0; + } + + /* pad up to 56 bytes of zeroes */ + while (md->curlen < 56) + { + md->buf[md->curlen++] = (unsigned char) 0; + } + + /* store length */ + WPA_PUT_BE64(md->buf + 56, md->length); + sha256_compress(md, md->buf); + + /* copy output */ + for (i = 0; i < 8; i++) + WPA_PUT_BE32(out + (4 * i), md->state[i]); + + return 0; +} + +/* ===== end - public domain SHA256 implementation ===== */ diff --git a/storage/maria/libmarias3/src/sha256.c b/storage/maria/libmarias3/src/sha256.c new file mode 100644 index 00000000..8a28f906 --- /dev/null +++ b/storage/maria/libmarias3/src/sha256.c @@ -0,0 +1,131 @@ +/* + * SHA-256 hash implementation and interface functions + * Copyright (c) 2003-2012, Jouni Malinen + * + * This software may be distributed under the terms of the BSD license. + * See README for more details. + */ + +#include "sha256.h" +#include "sha256_i.h" + +/** + * sha256 - SHA256 hash for data vector + * @num_elem: Number of elements in the data vector + * @addr: Pointer to the data areas + * @len: Length of the data blocks + * @mac: Buffer for the hash + * Returns: 0 on success, -1 of failure + */ +int sha256(const uint8_t *addr, const size_t len, uint8_t *mac) +{ + struct sha256_state ctx; + + sha256_init(&ctx); + + if (sha256_process(&ctx, addr, len)) + return -1; + + if (sha256_done(&ctx, mac)) + return -1; + + return 0; +} + +/** + * hmac_sha256_vector - HMAC-SHA256 over data vector (RFC 2104) + * @key: Key for HMAC operations + * @key_len: Length of the key in bytes + * @num_elem: Number of elements in the data vector + * @addr: Pointers to the data areas + * @len: Lengths of the data blocks + * @mac: Buffer for the hash (32 bytes) + * Returns: 0 on success, -1 on failure + */ +int hmac_sha256_vector(const uint8_t *key, size_t key_len, size_t num_elem, + const uint8_t *addr[], const size_t *len, uint8_t *mac) +{ + unsigned char k_pad[64]; /* padding - key XORd with ipad/opad */ + unsigned char tk[32]; + const uint8_t *_addr[6]; + size_t _len[6], i; + + if (num_elem > 5) + { + /* + * Fixed limit on the number of fragments to avoid having to + * allocate memory (which could fail). + */ + return -1; + } + + /* if key is longer than 64 bytes reset it to key = SHA256(key) */ + if (key_len > 64) + { + if (sha256_vector(1, &key, &key_len, tk) < 0) + return -1; + + key = tk; + key_len = 32; + } + + /* the HMAC_SHA256 transform looks like: + * + * SHA256(K XOR opad, SHA256(K XOR ipad, text)) + * + * where K is an n byte key + * ipad is the byte 0x36 repeated 64 times + * opad is the byte 0x5c repeated 64 times + * and text is the data being protected */ + + /* start out by storing key in ipad */ + memset(k_pad, 0, sizeof(k_pad)); + memcpy(k_pad, key, key_len); + + /* XOR key with ipad values */ + for (i = 0; i < 64; i++) + k_pad[i] ^= 0x36; + + /* perform inner SHA256 */ + _addr[0] = k_pad; + _len[0] = 64; + + for (i = 0; i < num_elem; i++) + { + _addr[i + 1] = addr[i]; + _len[i + 1] = len[i]; + } + + if (sha256_vector(1 + num_elem, _addr, _len, mac) < 0) + return -1; + + memset(k_pad, 0, sizeof(k_pad)); + memcpy(k_pad, key, key_len); + + /* XOR key with opad values */ + for (i = 0; i < 64; i++) + k_pad[i] ^= 0x5c; + + /* perform outer SHA256 */ + _addr[0] = k_pad; + _len[0] = 64; + _addr[1] = mac; + _len[1] = SHA256_MAC_LEN; + return sha256_vector(2, _addr, _len, mac); +} + + +/** + * hmac_sha256 - HMAC-SHA256 over data buffer (RFC 2104) + * @key: Key for HMAC operations + * @key_len: Length of the key in bytes + * @data: Pointers to the data area + * @data_len: Length of the data area + * @mac: Buffer for the hash (32 bytes) + * Returns: 0 on success, -1 on failure + */ +int hmac_sha256(const uint8_t *key, size_t key_len, const uint8_t *data, + size_t data_len, uint8_t *mac) +{ + return hmac_sha256_vector(key, key_len, 1, &data, &data_len, mac); +} diff --git a/storage/maria/libmarias3/src/sha256.h b/storage/maria/libmarias3/src/sha256.h new file mode 100644 index 00000000..45457b9b --- /dev/null +++ b/storage/maria/libmarias3/src/sha256.h @@ -0,0 +1,25 @@ +/* + * SHA256 hash implementation and interface functions + * Copyright (c) 2003-2016, Jouni Malinen + * + * This software may be distributed under the terms of the BSD license. + * See README for more details. + */ + +#ifndef SHA256_H +#define SHA256_H + +#define SHA256_MAC_LEN 32 + +#include +#include +#include + +int hmac_sha256_vector(const uint8_t *key, size_t key_len, size_t num_elem, + const uint8_t *addr[], const size_t *len, uint8_t *mac); +int hmac_sha256(const uint8_t *key, size_t key_len, const uint8_t *data, + size_t data_len, uint8_t *mac); + +int sha256(const uint8_t *addr, const size_t len, uint8_t *mac); + +#endif /* SHA256_H */ diff --git a/storage/maria/libmarias3/src/sha256_i.h b/storage/maria/libmarias3/src/sha256_i.h new file mode 100644 index 00000000..9b8aa2b7 --- /dev/null +++ b/storage/maria/libmarias3/src/sha256_i.h @@ -0,0 +1,66 @@ +/* + * SHA-256 internal definitions + * Copyright (c) 2003-2011, Jouni Malinen + * + * This software may be distributed under the terms of the BSD license. + * See README for more details. + */ + +#ifndef SHA256_I_H +#define SHA256_I_H + +#define SHA256_BLOCK_SIZE 64 + +#include + +struct sha256_state +{ + uint64_t length; + uint32_t state[8], curlen; + uint8_t buf[SHA256_BLOCK_SIZE]; +}; + +void sha256_init(struct sha256_state *md); +int sha256_process(struct sha256_state *md, const unsigned char *in, + unsigned long inlen); +int sha256_done(struct sha256_state *md, unsigned char *out); + +/** + * sha256_vector - SHA256 hash for data vector + * @num_elem: Number of elements in the data vector + * @addr: Pointers to the data areas + * @len: Lengths of the data blocks + * @mac: Buffer for the hash + * Returns: 0 on success, -1 on failure + */ +int sha256_vector(size_t num_elem, const uint8_t *addr[], const size_t *len, + uint8_t *mac); + +static inline void WPA_PUT_BE64(uint8_t *a, uint64_t val) +{ + a[0] = val >> 56; + a[1] = val >> 48; + a[2] = val >> 40; + a[3] = val >> 32; + a[4] = val >> 24; + a[5] = val >> 16; + a[6] = val >> 8; + a[7] = val & 0xff; +} + + +static inline uint32_t WPA_GET_BE32(const uint8_t *a) +{ + return ((uint32_t) a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]; +} + +static inline void WPA_PUT_BE32(uint8_t *a, uint32_t val) +{ + a[0] = (val >> 24) & 0xff; + a[1] = (val >> 16) & 0xff; + a[2] = (val >> 8) & 0xff; + a[3] = val & 0xff; +} + + +#endif /* SHA256_I_H */ diff --git a/storage/maria/libmarias3/src/structs.h b/storage/maria/libmarias3/src/structs.h new file mode 100644 index 00000000..34cbd817 --- /dev/null +++ b/storage/maria/libmarias3/src/structs.h @@ -0,0 +1,83 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * Copyright 2019 MariaDB Corporation Ab. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#pragma once + +#include "config.h" + +struct ms3_pool_alloc_list_st +{ + struct ms3_list_st *pool; + struct ms3_pool_alloc_list_st *prev; +}; + +struct ms3_list_container_st +{ + struct ms3_list_st *pool; + struct ms3_list_st *start; + struct ms3_pool_alloc_list_st *pool_list; + struct ms3_list_st *next; + size_t pool_free; +}; + +struct ms3_st +{ + char *s3key; + char *s3secret; + char *region; + char *base_domain; + int port; // 0 means "Use default" + + char *sts_endpoint; + char *sts_region; + char *iam_endpoint; + char *iam_role; + char *role_key; + char *role_secret; + char *role_session_token; + char *iam_role_arn; + size_t role_session_duration; + + size_t buffer_chunk_size; + CURL *curl; + char *last_error; + bool use_http; + bool disable_verification; + uint8_t list_version; + uint8_t protocol_version; + bool first_run; + char *path_buffer; + char *query_buffer; + struct ms3_list_container_st list_container; +}; + +struct memory_buffer_st +{ + uint8_t *data; + size_t length; + size_t alloced; + size_t buffer_chunk_size; +}; + +struct put_buffer_st +{ + const uint8_t *data; + size_t length; + size_t offset; +}; diff --git a/storage/maria/libmarias3/src/xml.c b/storage/maria/libmarias3/src/xml.c new file mode 100644 index 00000000..2c48a4ea --- /dev/null +++ b/storage/maria/libmarias3/src/xml.c @@ -0,0 +1,1157 @@ +/** + * Copyright (c) 2012 ooxi/xml.c + * https://github.com/ooxi/xml.c + * + * This software is provided 'as-is', without any express or implied warranty. + * In no event will the authors be held liable for any damages arising from the + * use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software in a + * product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source distribution. + */ + +#include "config.h" + +#ifdef XML_PARSER_VERBOSE +#include +#endif + +#include +#include +#include +#include +#include + +#include "common.h" +#include "xml.h" + + +/* + * public domain strtok_r() by Charlie Gordon + * + * from comp.lang.c 9/14/2007 + * + * http://groups.google.com/group/comp.lang.c/msg/2ab1ecbb86646684 + * + * (Declaration that it's public domain): + * http://groups.google.com/group/comp.lang.c/msg/7c7b39328fefab9c + */ +static char* xml_strtok_r(char *str, const char *delim, char **nextp) { + char *ret; + + if (str == NULL) { + str = *nextp; + } + + str += strspn(str, delim); + + if (*str == '\0') { + return NULL; + } + + ret = str; + + str += strcspn(str, delim); + + if (*str) { + *str++ = '\0'; + } + + *nextp = str; + + return ret; +} + + + + + + +/** + * [OPAQUE API] + * + * UTF-8 text + */ +struct xml_string { + uint8_t const* buffer; + size_t length; +}; + +/** + * [OPAQUE API] + * + * An xml_attribute may contain text content. + */ +struct xml_attribute { + struct xml_string* name; + struct xml_string* content; +}; + +/** + * [OPAQUE API] + * + * An xml_node will always contain a tag name, a 0-terminated list of attributes + * and a 0-terminated list of children. Moreover it may contain text content. + */ +struct xml_node { + struct xml_string* name; + struct xml_string* content; + struct xml_attribute** attributes; + struct xml_node** children; +}; + +/** + * [OPAQUE API] + * + * An xml_document simply contains the root node and the underlying buffer + */ +struct xml_document { + struct { + uint8_t* buffer; + size_t length; + } buffer; + + struct xml_node* root; +}; + + + + + +/** + * [PRIVATE] + * + * Parser context + */ +struct xml_parser { + uint8_t* buffer; + size_t position; + size_t length; +}; + +/** + * [PRIVATE] + * + * Character offsets + */ +enum xml_parser_offset { + NO_CHARACTER = -1, + CURRENT_CHARACTER = 0, + NEXT_CHARACTER = 1, +}; + + + + + +/** + * [PRIVATE] + * + * @return Number of attributes in 0-terminated array + */ +static size_t get_zero_terminated_array_attributes(struct xml_attribute** attributes) { + size_t elements = 0; + + while (attributes[elements]) { + ++elements; + } + + return elements; +} + + + +/** + * [PRIVATE] + * + * @return Number of nodes in 0-terminated array + */ +static size_t get_zero_terminated_array_nodes(struct xml_node** nodes) { + size_t elements = 0; + + while (nodes[elements]) { + ++elements; + } + + return elements; +} + + + +/** + * [PRIVATE] + * + * @warning No UTF conversions will be attempted + * + * @return true iff a == b + */ +static _Bool xml_string_equals(struct xml_string* a, struct xml_string* b) { + + size_t i = 0; + if (a->length != b->length) { + return false; + } + + for (; i < a->length; ++i) { + if (a->buffer[i] != b->buffer[i]) { + return false; + } + } + + return true; +} + + + +/** + * [PRIVATE] + */ +static uint8_t* xml_string_clone(struct xml_string* s) { + uint8_t* clone; + if (!s) { + return 0; + } + + clone = ms3_ccalloc(s->length + 1, sizeof(uint8_t)); + + xml_string_copy(s, clone, s->length); + clone[s->length] = 0; + + return clone; +} + + + +/** + * [PRIVATE] + * + * Frees the resources allocated by the string + * + * @warning `buffer` must _not_ be freed, since it is a reference to the + * document's buffer + */ +static void xml_string_free(struct xml_string* string) { + ms3_cfree(string); +} + + + +/** + * [PRIVATE] + * + * Frees the resources allocated by the attribute + */ +static void xml_attribute_free(struct xml_attribute* attribute) { + if(attribute->name) { + xml_string_free(attribute->name); + } + if(attribute->content) { + xml_string_free(attribute->content); + } + ms3_cfree(attribute); +} + +/** + * [PRIVATE] + * + * Frees the resources allocated by the node + */ +static void xml_node_free(struct xml_node* node) { + struct xml_attribute** at; + struct xml_node** it; + + xml_string_free(node->name); + + if (node->content) { + xml_string_free(node->content); + } + + at = node->attributes; + while(*at) { + xml_attribute_free(*at); + ++at; + } + ms3_cfree(node->attributes); + + it = node->children; + while (*it) { + xml_node_free(*it); + ++it; + } + ms3_cfree(node->children); + + ms3_cfree(node); +} + + + +/** + * [PRIVATE] + * + * Echos the parsers call stack for debugging purposes + */ +#ifdef XML_PARSER_VERBOSE +static void xml_parser_info(struct xml_parser* parser, char const* message) { + fprintf(stdout, "xml_parser_info %s\n", message); +} +#else +#define xml_parser_info(parser, message) {} +#endif + + + +/** + * [PRIVATE] + * + * Echos an error regarding the parser's source to the console + */ + +#define tmp_min(X,Y) ((X) < (Y) ? (X) : (Y)) + + +static void xml_parser_error(struct xml_parser* parser, enum xml_parser_offset offset, char const* message) { + int row = 0; + int column = 0; + + size_t character = tmp_min(parser->length, parser->position + offset); + size_t position = 0; for (; position < character; ++position) { + column++; + + if ('\n' == parser->buffer[position]) { + row++; + column = 0; + } + } + + if (NO_CHARACTER != offset) { + fprintf(stderr, "xml_parser_error at %i:%i (is %c): %s\n", + row + 1, column, parser->buffer[character], message + ); + } else { + fprintf(stderr, "xml_parser_error at %i:%i: %s\n", + row + 1, column, message + ); + } +} + + + +/** + * [PRIVATE] + * + * Returns the n-th not-whitespace byte in parser and 0 if such a byte does not + * exist + */ +static uint8_t xml_parser_peek(struct xml_parser* parser, size_t n) { + size_t position = parser->position; + + while (position < parser->length) { + if (!isspace(parser->buffer[position])) { + if (n == 0) { + return parser->buffer[position]; + } else { + --n; + } + } + + position++; + } + + return 0; +} + + + +/** + * [PRIVATE] + * + * Moves the parser's position n bytes. If the new position would be out of + * bounds, it will be converted to the bounds itself + */ +static void xml_parser_consume(struct xml_parser* parser, size_t n) { + + /* Debug information + */ + #ifdef XML_PARSER_VERBOSE + #define min(X,Y) ((X) < (Y) ? (X) : (Y)) + char* consumed = alloca((n + 1) * sizeof(char)); + memcpy(consumed, &parser->buffer[parser->position], min(n, parser->length - parser->position)); + consumed[n] = 0; + #undef min + + size_t message_buffer_length = 512; + char* message_buffer = alloca(512 * sizeof(char)); + snprintf(message_buffer, message_buffer_length, "Consuming %li bytes \"%s\"", (long)n, consumed); + message_buffer[message_buffer_length - 1] = 0; + + xml_parser_info(parser, message_buffer); + #endif + + + /* Move the position forward + */ + parser->position += n; + + /* Don't go too far + * + * @warning Valid because parser->length must be greater than 0 + */ + if (parser->position >= parser->length) { + parser->position = parser->length - 1; + } +} + + + +/** + * [PRIVATE] + * + * Skips to the next non-whitespace character + */ +static void xml_skip_whitespace(struct xml_parser* parser) { + xml_parser_info(parser, "whitespace"); + + while (isspace(parser->buffer[parser->position])) { + if (parser->position + 1 >= parser->length) { + return; + } else { + parser->position++; + } + } +} + + + +/** + * [PRIVATE] + * + * Finds and creates all attributes on the given node. + * + * @author Blake Felt + * @see https://github.com/Molorius + */ +static struct xml_attribute** xml_find_attributes(struct xml_parser* parser, struct xml_string* tag_open) { + char* tmp; + char* rest = NULL; + char* token; + char* str_name; + char* str_content; + const unsigned char* start_name; + const unsigned char* start_content; + size_t old_elements; + size_t new_elements; + struct xml_attribute* new_attribute; + struct xml_attribute** attributes; + long position; + + (void) parser; // clang for some reason thinks this isn't used + xml_parser_info(parser, "find_attributes"); + attributes = ms3_ccalloc(1, sizeof(struct xml_attribute*)); + attributes[0] = 0; + + tmp = (char*) xml_string_clone(tag_open); + + token = xml_strtok_r(tmp, " ", &rest); // skip the first value + if(token == NULL) { + goto cleanup; + } + tag_open->length = strlen(token); + + for(token=xml_strtok_r(NULL," ", &rest); token!=NULL; token=xml_strtok_r(NULL," ", &rest)) { + str_name = ms3_cmalloc(strlen(token)+1); + str_content = ms3_cmalloc(strlen(token)+1); + // %s=\"%s\" wasn't working for some reason, ugly hack to make it work + if(sscanf(token, "%[^=]=\"%[^\"]", str_name, str_content) != 2) { + if(sscanf(token, "%[^=]=\'%[^\']", str_name, str_content) != 2) { + ms3_cfree(str_name); + ms3_cfree(str_content); + continue; + } + } + position = token-tmp; + start_name = &tag_open->buffer[position]; + start_content = &tag_open->buffer[position + strlen(str_name) + 2]; + + new_attribute = ms3_cmalloc(sizeof(struct xml_attribute)); + new_attribute->name = ms3_cmalloc(sizeof(struct xml_string)); + new_attribute->name->buffer = (unsigned char*)start_name; + new_attribute->name->length = strlen(str_name); + new_attribute->content = ms3_cmalloc(sizeof(struct xml_string)); + new_attribute->content->buffer = (unsigned char*)start_content; + new_attribute->content->length = strlen(str_content); + + old_elements = get_zero_terminated_array_attributes(attributes); + new_elements = old_elements + 1; + attributes = ms3_crealloc(attributes, (new_elements+1)*sizeof(struct xml_attributes*)); + + attributes[new_elements-1] = new_attribute; + attributes[new_elements] = 0; + + + ms3_cfree(str_name); + ms3_cfree(str_content); + } + +cleanup: + ms3_cfree(tmp); + return attributes; +} + + + +/** + * [PRIVATE] + * + * Parses the name out of the an XML tag's ending + * + * ---( Example )--- + * tag_name> + * --- + */ +static struct xml_string* xml_parse_tag_end(struct xml_parser* parser) { + size_t start; + size_t length = 0; + struct xml_string* name; + + xml_parser_info(parser, "tag_end"); + start = parser->position; + + /* Parse until `>' or a whitespace is reached + */ + while (start + length < parser->length) { + uint8_t current = xml_parser_peek(parser, CURRENT_CHARACTER); + + if (('>' == current) || isspace(current)) { + break; + } else { + xml_parser_consume(parser, 1); + length++; + } + } + + /* Consume `>' + */ + if ('>' != xml_parser_peek(parser, CURRENT_CHARACTER)) { + xml_parser_error(parser, CURRENT_CHARACTER, "xml_parse_tag_end::expected tag end"); + return 0; + } + xml_parser_consume(parser, 1); + + /* Return parsed tag name + */ + name = ms3_cmalloc(sizeof(struct xml_string)); + name->buffer = &parser->buffer[start]; + name->length = length; + return name; +} + +/** + * [PRIVATE] + * + * Parses an opening XML tag without attributes + * + * ---( Example )--- + * + * --- + */ +static struct xml_string* xml_parse_tag_open(struct xml_parser* parser) { + xml_parser_info(parser, "tag_open"); + xml_skip_whitespace(parser); + + /* Consume `<' + */ + if ('<' != xml_parser_peek(parser, CURRENT_CHARACTER)) { + xml_parser_error(parser, CURRENT_CHARACTER, "xml_parse_tag_open::expected opening tag"); + return 0; + } + xml_parser_consume(parser, 1); + + /* Consume tag name + */ + return xml_parse_tag_end(parser); +} + + + +/** + * [PRIVATE] + * + * Parses an closing XML tag without attributes + * + * ---( Example )--- + * + * --- + */ +static struct xml_string* xml_parse_tag_close(struct xml_parser* parser) { + xml_parser_info(parser, "tag_close"); + xml_skip_whitespace(parser); + + /* Consume `position; + + /* Consume until `<' is reached + */ + while (start + length < parser->length) { + uint8_t current = xml_parser_peek(parser, CURRENT_CHARACTER); + + if ('<' == current) { + break; + } else { + xml_parser_consume(parser, 1); + length++; + } + } + + /* Next character must be an `<' or we have reached end of file + */ + if ('<' != xml_parser_peek(parser, CURRENT_CHARACTER)) { + xml_parser_error(parser, CURRENT_CHARACTER, "xml_parse_content::expected <"); + return 0; + } + + /* Ignore tailing whitespace + */ + while ((length > 0) && isspace(parser->buffer[start + length - 1])) { + length--; + } + + /* Return text + */ + content = ms3_cmalloc(sizeof(struct xml_string)); + content->buffer = &parser->buffer[start]; + content->length = length; + return content; +} + + + +/** + * [PRIVATE] + * + * Parses an XML fragment node + * + * ---( Example without children )--- + * Text + * --- + * + * ---( Example with children )--- + * + * Text + * Text + * Content + * + * --- + */ +static struct xml_node* xml_parse_node(struct xml_parser* parser) { + /* Setup variables + */ + struct xml_string* tag_open = 0; + struct xml_string* tag_close = 0; + struct xml_string* content = 0; + struct xml_node* node; + struct xml_node** it; + size_t original_length; + struct xml_attribute** attributes; + struct xml_node** children = ms3_ccalloc(1, sizeof(struct xml_node*)); + children[0] = 0; + + xml_parser_info(parser, "node"); + + /* Parse open tag + */ + tag_open = xml_parse_tag_open(parser); + if (!tag_open) { + xml_parser_error(parser, NO_CHARACTER, "xml_parse_node::tag_open"); + goto exit_failure; + } + + original_length = tag_open->length; + attributes = xml_find_attributes(parser, tag_open); + + /* If tag ends with `/' it's self closing, skip content lookup */ + if (tag_open->length > 0 && '/' == tag_open->buffer[original_length - 1]) { + /* Drop `/' + */ + goto node_creation; + } + + /* If the content does not start with '<', a text content is assumed + */ + if ('<' != xml_parser_peek(parser, CURRENT_CHARACTER)) { + content = xml_parse_content(parser); + + if (!content) { + xml_parser_error(parser, 0, "xml_parse_node::content"); + goto exit_failure; + } + + + /* Otherwise children are to be expected + */ + } else while ('/' != xml_parser_peek(parser, NEXT_CHARACTER)) { + + /* Parse child node + */ + struct xml_node* child = xml_parse_node(parser); + size_t old_elements, new_elements; + + if (!child) { + xml_parser_error(parser, NEXT_CHARACTER, "xml_parse_node::child"); + goto exit_failure; + } + + /* Grow child array :) + */ + old_elements = get_zero_terminated_array_nodes(children); + new_elements = old_elements + 1; + children = ms3_crealloc(children, (new_elements + 1) * sizeof(struct xml_node*)); + + /* Save child + */ + children[new_elements - 1] = child; + children[new_elements] = 0; + } + + + /* Parse close tag + */ + tag_close = xml_parse_tag_close(parser); + if (!tag_close) { + xml_parser_error(parser, NO_CHARACTER, "xml_parse_node::tag_close"); + goto exit_failure; + } + + + /* Close tag has to match open tag + */ + if (!xml_string_equals(tag_open, tag_close)) { + xml_parser_error(parser, NO_CHARACTER, "xml_parse_node::tag mismatch"); + goto exit_failure; + } + + + /* Return parsed node + */ + xml_string_free(tag_close); + +node_creation:; + node = ms3_cmalloc(sizeof(struct xml_node)); + node->name = tag_open; + node->content = content; + node->attributes = attributes; + node->children = children; + return node; + + + /* A failure occured, so free all allocalted resources + */ +exit_failure: + if (tag_open) { + xml_string_free(tag_open); + } + if (tag_close) { + xml_string_free(tag_close); + } + if (content) { + xml_string_free(content); + } + + it = children; + while (*it) { + xml_node_free(*it); + ++it; + } + ms3_cfree(children); + + return 0; +} + + +/** + * [PRIVATE] + * Skips XML headers in format + */ +static void xml_parse_skip_meta(struct xml_parser* parser) { + if ('<' == xml_parser_peek(parser, CURRENT_CHARACTER) && + '?' == xml_parser_peek(parser, NEXT_CHARACTER)) { + size_t pos = parser->position; + while (pos < parser->length) { + if ('?' == parser->buffer[pos] && + '>' == parser->buffer[pos + 1]) { + parser->position = pos + 2; + return; + } + pos++; + } + } +} + +/** + * [PUBLIC API] + */ +struct xml_document* xml_parse_document(uint8_t* buffer, size_t length) { + + /* Initialize parser + */ + struct xml_parser parser = { + .buffer = buffer, + .position = 0, + .length = length + }; + struct xml_node* root; + struct xml_document* document; + + /* An empty buffer can never contain a valid document + */ + if (!length) { + xml_parser_error(&parser, NO_CHARACTER, "xml_parse_document::length equals zero"); + return 0; + } + + /* Parse the root node + */ + xml_parse_skip_meta(&parser); + root = xml_parse_node(&parser); + if (!root) { + xml_parser_error(&parser, NO_CHARACTER, "xml_parse_document::parsing document failed"); + return 0; + } + + /* Return parsed document + */ + document = ms3_cmalloc(sizeof(struct xml_document)); + document->buffer.buffer = buffer; + document->buffer.length = length; + document->root = root; + + return document; +} + + + +/** + * [PUBLIC API] + */ +struct xml_document* xml_open_document(FILE* source) { + + /* Prepare buffer + */ + size_t const read_chunk = 1; // TODO 4096; + + size_t document_length = 0; + size_t buffer_size = 1; // TODO 4069 + struct xml_document* document; + uint8_t* buffer = ms3_cmalloc(buffer_size * sizeof(uint8_t)); + + + /* Read hole file into buffer + */ + while (!feof(source)) { + size_t read; + /* Reallocate buffer + */ + if (buffer_size - document_length < read_chunk) { + buffer = ms3_crealloc(buffer, buffer_size + 2 * read_chunk); + buffer_size += 2 * read_chunk; + } + + read = fread(&buffer[document_length], + sizeof(uint8_t), read_chunk, + source + ); + + document_length += read; + } + fclose(source); + + /* Try to parse buffer + */ + document = xml_parse_document(buffer, document_length); + + if (!document) { + ms3_cfree(buffer); + return 0; + } + return document; +} + + + +/** + * [PUBLIC API] + */ +void xml_document_free(struct xml_document* document, bool free_buffer) { + xml_node_free(document->root); + + if (free_buffer) { + ms3_cfree(document->buffer.buffer); + } + ms3_cfree(document); +} + + + +/** + * [PUBLIC API] + */ +struct xml_node* xml_document_root(struct xml_document* document) { + return document->root; +} + + + +/** + * [PUBLIC API] + */ +struct xml_string* xml_node_name(struct xml_node* node) { + return node->name; +} + +int xml_node_name_cmp(struct xml_node* node, const char *name) { + return strncmp((char*)node->name->buffer, name, node->name->length); +} + +/** + * [PUBLIC API] + */ +struct xml_string* xml_node_content(struct xml_node* node) { + return node->content; +} + + + +/** + * [PUBLIC API] + * + * @warning O(n) + */ +size_t xml_node_children(struct xml_node* node) { + return get_zero_terminated_array_nodes(node->children); +} + + + +/** + * [PUBLIC API] + */ +struct xml_node* xml_node_child(struct xml_node* node, size_t child) { + if (child >= xml_node_children(node)) { + return 0; + } + + return node->children[child]; +} + + + +/** + * [PUBLIC API] + */ +size_t xml_node_attributes(struct xml_node* node) { + return get_zero_terminated_array_attributes(node->attributes); +} + + + +/** + * [PUBLIC API] + */ +struct xml_string* xml_node_attribute_name(struct xml_node* node, size_t attribute) { + if(attribute >= xml_node_attributes(node)) { + return 0; + } + + return node->attributes[attribute]->name; +} + + + +/** + * [PUBLIC API] + */ +struct xml_string* xml_node_attribute_content(struct xml_node* node, size_t attribute) { + if(attribute >= xml_node_attributes(node)) { + return 0; + } + + return node->attributes[attribute]->content; +} + + + +/** + * [PUBLIC API] + */ +struct xml_node* xml_easy_child(struct xml_node* node, uint8_t const* child_name, ...) { + + /* Find children, one by one + */ + struct xml_node* current = node; + + va_list arguments; + va_start(arguments, child_name); + + + /* Descent to current.child + */ + while (child_name) { + + /* Convert child_name to xml_string for easy comparison + */ + struct xml_string cn = { + .buffer = child_name, + .length = strlen((const char*)child_name) + }; + + /* Interate through all children + */ + struct xml_node* next = 0; + + size_t i = 0; for (; i < xml_node_children(current); ++i) { + struct xml_node* child = xml_node_child(current, i); + + if (xml_string_equals(xml_node_name(child), &cn)) { + if (!next) { + next = child; + + /* Two children with the same name + */ + } else { + va_end(arguments); + return 0; + } + } + } + + /* No child with that name found + */ + if (!next) { + va_end(arguments); + return 0; + } + current = next; + + /* Find name of next child + */ + child_name = va_arg(arguments, uint8_t const*); + } + va_end(arguments); + + + /* Return current element + */ + return current; +} + + + +/** + * [PUBLIC API] + */ +uint8_t* xml_easy_name(struct xml_node* node) { + if (!node) { + return 0; + } + + return xml_string_clone(xml_node_name(node)); +} + + + +/** + * [PUBLIC API] + */ +uint8_t* xml_easy_content(struct xml_node* node) { + if (!node) { + return 0; + } + + return xml_string_clone(xml_node_content(node)); +} + + + +/** + * [PUBLIC API] + */ +size_t xml_string_length(struct xml_string* string) { + if (!string) { + return 0; + } + return string->length; +} + + + +/** + * [PUBLIC API] + */ +void xml_string_copy(struct xml_string* string, uint8_t* buffer, size_t length) { + if (!string) { + return; + } + + #define min(X,Y) ((X) < (Y) ? (X) : (Y)) + length = min(length, string->length); + #undef min + + memcpy(buffer, string->buffer, length); + buffer[length]= '\0'; +} diff --git a/storage/maria/libmarias3/src/xml.h b/storage/maria/libmarias3/src/xml.h new file mode 100644 index 00000000..9574b2bc --- /dev/null +++ b/storage/maria/libmarias3/src/xml.h @@ -0,0 +1,197 @@ +/** + * Copyright (c) 2012 ooxi/xml.c + * https://github.com/ooxi/xml.c + * + * This software is provided 'as-is', without any express or implied warranty. + * In no event will the authors be held liable for any damages arising from the + * use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software in a + * product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source distribution. + */ +#ifndef HEADER_XML +#define HEADER_XML + + +/** + * Includes + */ +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Opaque structure holding the parsed xml document + */ +struct xml_document; +struct xml_node; +struct xml_attribute; + +/** + * Internal character sequence representation + */ +struct xml_string; + + + +/** + * Tries to parse the XML fragment in buffer + * + * @param buffer Chunk to parse + * @param length Size of the buffer + * + * @warning `buffer` will be referenced by the document, you may not free it + * until you free the xml_document + * @warning You have to call xml_document_free after you finished using the + * document + * + * @return The parsed xml fragment iff parsing was successful, 0 otherwise + */ +struct xml_document* xml_parse_document(uint8_t* buffer, size_t length); + + + +/** + * Tries to read an XML document from disk + * + * @param source File that will be read into an xml document. Will be closed + * + * @warning You have to call xml_document_free with free_buffer = true after you + * finished using the document + * + * @return The parsed xml fragment iff parsing was successful, 0 otherwise + */ +struct xml_document* xml_open_document(FILE* source); + + + +/** + * Frees all resources associated with the document. All xml_node and xml_string + * references obtained through the document will be invalidated + * + * @param document xml_document to free + * @param free_buffer iff true the internal buffer supplied via xml_parse_buffer + * will be freed with the `free` system call + */ +void xml_document_free(struct xml_document* document, bool free_buffer); + + +/** + * @return xml_node representing the document root + */ +struct xml_node* xml_document_root(struct xml_document* document); + + + +/** + * @return The xml_node's tag name + */ +struct xml_string* xml_node_name(struct xml_node* node); + + + +/** + * @return The xml_node's string content (if available, otherwise NULL) + */ +struct xml_string* xml_node_content(struct xml_node* node); + + + +/** + * @return Number of child nodes + */ +size_t xml_node_children(struct xml_node* node); + + + +/** + * @return The n-th child or 0 if out of range + */ +struct xml_node* xml_node_child(struct xml_node* node, size_t child); + + + +/** + * @return Number of attribute nodes + */ +size_t xml_node_attributes(struct xml_node* node); + + + +/** + * @return the n-th attribute name or 0 if out of range + */ +struct xml_string* xml_node_attribute_name(struct xml_node* node, size_t attribute); + + + +/** + * @return the n-th attribute content or 0 if out of range + */ +struct xml_string* xml_node_attribute_content(struct xml_node* node, size_t attribute); + + + +/** + * @return The node described by the path or 0 if child cannot be found + * @warning Each element on the way must be unique + * @warning Last argument must be 0 + */ +struct xml_node* xml_easy_child(struct xml_node* node, uint8_t const* child, ...); + + + +/** + * @return 0-terminated copy of node name + * @warning User must free the result + */ +uint8_t* xml_easy_name(struct xml_node* node); + + + +/** + * @return 0-terminated copy of node content + * @warning User must free the result + */ +uint8_t* xml_easy_content(struct xml_node* node); + + + +/** + * @return Length of the string + */ +size_t xml_string_length(struct xml_string* string); + + + +/** + * Copies the string into the supplied buffer + * + * @warning String will not be 0-terminated + * @warning Will write at most length bytes, even if the string is longer + */ +void xml_string_copy(struct xml_string* string, uint8_t* buffer, size_t length); + +int xml_node_name_cmp(struct xml_node* node, const char *name); + +#ifdef __cplusplus +} +#endif + +#endif + -- cgit v1.2.3