summaryrefslogtreecommitdiffstats
path: root/extra/mariabackup/xbcloud.cc
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-04 18:00:34 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-04 18:00:34 +0000
commit3f619478f796eddbba6e39502fe941b285dd97b1 (patch)
treee2c7b5777f728320e5b5542b6213fd3591ba51e2 /extra/mariabackup/xbcloud.cc
parentInitial commit. (diff)
downloadmariadb-3f619478f796eddbba6e39502fe941b285dd97b1.tar.xz
mariadb-3f619478f796eddbba6e39502fe941b285dd97b1.zip
Adding upstream version 1:10.11.6.upstream/1%10.11.6upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'extra/mariabackup/xbcloud.cc')
-rw-r--r--extra/mariabackup/xbcloud.cc2722
1 files changed, 2722 insertions, 0 deletions
diff --git a/extra/mariabackup/xbcloud.cc b/extra/mariabackup/xbcloud.cc
new file mode 100644
index 00000000..588a15eb
--- /dev/null
+++ b/extra/mariabackup/xbcloud.cc
@@ -0,0 +1,2722 @@
+/******************************************************
+Copyright (c) 2014 Percona LLC and/or its affiliates.
+
+The xbstream utility: serialize/deserialize files in the XBSTREAM format.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
+
+*******************************************************/
+
+#include <my_global.h>
+#include <my_default.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <curl/curl.h>
+#include <ev.h>
+#include <unistd.h>
+#include <errno.h>
+#include <gcrypt.h>
+#include <assert.h>
+#include <my_sys.h>
+#include <my_dir.h>
+#include <my_getopt.h>
+#include <algorithm>
+#include <map>
+#include <string>
+#include <jsmn.h>
+#include "xbstream.h"
+
+using std::min;
+using std::max;
+using std::map;
+using std::string;
+
+#define XBCLOUD_VERSION "1.0"
+
+#define SWIFT_MAX_URL_SIZE 8192
+#define SWIFT_MAX_HDR_SIZE 8192
+
+#define SWIFT_CHUNK_SIZE 11 * 1024 * 1024
+
+#if ((LIBCURL_VERSION_MAJOR >= 7) && (LIBCURL_VERSION_MINOR >= 16))
+#define OLD_CURL_MULTI 0
+#else
+#define OLD_CURL_MULTI 1
+#endif
+
+/*****************************************************************************/
+
+typedef struct swift_auth_info_struct swift_auth_info;
+typedef struct connection_info_struct connection_info;
+typedef struct socket_info_struct socket_info;
+typedef struct global_io_info_struct global_io_info;
+typedef struct slo_chunk_struct slo_chunk;
+typedef struct container_list_struct container_list;
+typedef struct object_info_struct object_info;
+
+struct swift_auth_info_struct {
+ char url[SWIFT_MAX_URL_SIZE];
+ char token[SWIFT_MAX_HDR_SIZE];
+};
+
+struct global_io_info_struct {
+ struct ev_loop *loop;
+ struct ev_io input_event;
+ struct ev_timer timer_event;
+ CURLM *multi;
+ int still_running;
+ int eof;
+ curl_socket_t input_fd;
+ connection_info **connections;
+ long chunk_no;
+ connection_info *current_connection;
+ const char *url;
+ const char *container;
+ const char *token;
+ const char *backup_name;
+};
+
+struct socket_info_struct {
+ curl_socket_t sockfd;
+ CURL *easy;
+ int action;
+ long timeout;
+ struct ev_io ev;
+ int evset;
+ global_io_info *global;
+};
+
+struct connection_info_struct {
+ CURL *easy;
+ global_io_info *global;
+ char *buffer;
+ size_t buffer_size;
+ size_t filled_size;
+ size_t upload_size;
+ bool chunk_uploaded;
+ bool chunk_acked;
+ char error[CURL_ERROR_SIZE];
+ struct curl_slist *slist;
+ char *name;
+ size_t name_len;
+ char hash[33];
+ size_t chunk_no;
+ bool magic_verified;
+ size_t chunk_path_len;
+ xb_chunk_type_t chunk_type;
+ size_t payload_size;
+ size_t chunk_size;
+ int retry_count;
+ bool upload_started;
+ ulong global_idx;
+};
+
+struct slo_chunk_struct {
+ char name[SWIFT_MAX_URL_SIZE];
+ char md5[33];
+ int idx;
+ size_t size;
+};
+
+struct object_info_struct {
+ char hash[33];
+ char name[SWIFT_MAX_URL_SIZE];
+ size_t bytes;
+};
+
+struct container_list_struct {
+ size_t content_length;
+ size_t content_bufsize;
+ char *content_json;
+ size_t object_count;
+ size_t idx;
+ object_info *objects;
+ bool final;
+};
+
+enum {SWIFT, S3};
+const char *storage_names[] =
+{ "SWIFT", "S3", NullS};
+
+static my_bool opt_verbose = 0;
+static ulong opt_storage = SWIFT;
+static const char *opt_swift_user = NULL;
+static const char *opt_swift_user_id = NULL;
+static const char *opt_swift_password = NULL;
+static const char *opt_swift_tenant = NULL;
+static const char *opt_swift_tenant_id = NULL;
+static const char *opt_swift_project = NULL;
+static const char *opt_swift_project_id = NULL;
+static const char *opt_swift_domain = NULL;
+static const char *opt_swift_domain_id = NULL;
+static const char *opt_swift_region = NULL;
+static const char *opt_swift_container = NULL;
+static const char *opt_swift_storage_url = NULL;
+static const char *opt_swift_auth_url = NULL;
+static const char *opt_swift_key = NULL;
+static const char *opt_swift_auth_version = NULL;
+static const char *opt_name = NULL;
+static const char *opt_cacert = NULL;
+static ulong opt_parallel = 1;
+static my_bool opt_insecure = 0;
+static enum {MODE_GET, MODE_PUT, MODE_DELETE} opt_mode;
+
+static char **file_list = NULL;
+static int file_list_size = 0;
+
+TYPELIB storage_typelib =
+{array_elements(storage_names)-1, "", storage_names, NULL};
+
+enum {
+ OPT_STORAGE = 256,
+ OPT_SWIFT_CONTAINER,
+ OPT_SWIFT_AUTH_URL,
+ OPT_SWIFT_KEY,
+ OPT_SWIFT_USER,
+ OPT_SWIFT_USER_ID,
+ OPT_SWIFT_PASSWORD,
+ OPT_SWIFT_TENANT,
+ OPT_SWIFT_TENANT_ID,
+ OPT_SWIFT_PROJECT,
+ OPT_SWIFT_PROJECT_ID,
+ OPT_SWIFT_DOMAIN,
+ OPT_SWIFT_DOMAIN_ID,
+ OPT_SWIFT_REGION,
+ OPT_SWIFT_STORAGE_URL,
+ OPT_SWIFT_AUTH_VERSION,
+ OPT_PARALLEL,
+ OPT_CACERT,
+ OPT_INSECURE,
+ OPT_VERBOSE
+};
+
+
+static struct my_option my_long_options[] =
+{
+ {"help", '?', "Display this help and exit.",
+ 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"storage", OPT_STORAGE, "Specify storage type S3/SWIFT.",
+ &opt_storage, &opt_storage, &storage_typelib,
+ GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"swift-auth-version", OPT_SWIFT_AUTH_VERSION,
+ "Swift authentication verison to use.",
+ &opt_swift_auth_version, &opt_swift_auth_version, 0,
+ GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-container", OPT_SWIFT_CONTAINER,
+ "Swift container to store backups into.",
+ &opt_swift_container, &opt_swift_container, 0,
+ GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-user", OPT_SWIFT_USER,
+ "Swift user name.",
+ &opt_swift_user, &opt_swift_user, 0, GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-user-id", OPT_SWIFT_USER_ID,
+ "Swift user ID.",
+ &opt_swift_user_id, &opt_swift_user_id, 0, GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-auth-url", OPT_SWIFT_AUTH_URL,
+ "Base URL of SWIFT authentication service.",
+ &opt_swift_auth_url, &opt_swift_auth_url, 0,
+ GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-storage-url", OPT_SWIFT_STORAGE_URL,
+ "URL of object-store endpoint. Usually received from authentication "
+ "service. Specify to override this value.",
+ &opt_swift_storage_url, &opt_swift_storage_url, 0,
+ GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-key", OPT_SWIFT_KEY,
+ "Swift key.",
+ &opt_swift_key, &opt_swift_key, 0, GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-tenant", OPT_SWIFT_TENANT,
+ "The tenant name. Both the --swift-tenant and --swift-tenant-id "
+ "options are optional, but should not be specified together.",
+ &opt_swift_tenant, &opt_swift_tenant, 0, GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-tenant-id", OPT_SWIFT_TENANT_ID,
+ "The tenant ID. Both the --swift-tenant and --swift-tenant-id "
+ "options are optional, but should not be specified together.",
+ &opt_swift_tenant_id, &opt_swift_tenant_id, 0,
+ GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-project", OPT_SWIFT_PROJECT,
+ "The project name.",
+ &opt_swift_project, &opt_swift_project, 0, GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-project-id", OPT_SWIFT_PROJECT_ID,
+ "The project ID.",
+ &opt_swift_project_id, &opt_swift_project_id, 0,
+ GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-domain", OPT_SWIFT_DOMAIN,
+ "The domain name.",
+ &opt_swift_domain, &opt_swift_domain, 0, GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-domain-id", OPT_SWIFT_DOMAIN_ID,
+ "The domain ID.",
+ &opt_swift_domain_id, &opt_swift_domain_id, 0,
+ GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-password", OPT_SWIFT_PASSWORD,
+ "The password of the user.",
+ &opt_swift_password, &opt_swift_password, 0,
+ GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"swift-region", OPT_SWIFT_REGION,
+ "The region object-store endpoint.",
+ &opt_swift_region, &opt_swift_region, 0,
+ GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"parallel", OPT_PARALLEL,
+ "Number of parallel chunk uploads.",
+ &opt_parallel, &opt_parallel, 0, GET_ULONG, REQUIRED_ARG,
+ 1, 0, 0, 0, 0, 0},
+
+ {"cacert", OPT_CACERT,
+ "CA certificate file.",
+ &opt_cacert, &opt_cacert, 0, GET_STR_ALLOC, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"insecure", OPT_INSECURE,
+ "Do not verify server SSL certificate.",
+ &opt_insecure, &opt_insecure, 0, GET_BOOL, NO_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"verbose", OPT_VERBOSE,
+ "Turn ON cURL tracing.",
+ &opt_verbose, &opt_verbose, 0, GET_BOOL, NO_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
+};
+
+/* The values of these arguments should be masked
+ on the command line */
+static const char * const masked_args[] = {
+ "--swift-password",
+ "--swift-key",
+ "--swift-auth-url",
+ "--swift-storage-url",
+ "--swift-container",
+ "--swift-user",
+ "--swift-tenant",
+ "--swift-user-id",
+ "--swift-tenant-id",
+ 0
+};
+
+static map<string, ulonglong> file_chunk_count;
+
+static
+void
+print_version()
+{
+ printf("%s Ver %s for %s (%s)\n", my_progname, XBCLOUD_VERSION,
+ SYSTEM_TYPE, MACHINE_TYPE);
+}
+
+static
+void
+usage()
+{
+ print_version();
+ puts("Copyright (C) 2015 Percona LLC and/or its affiliates.");
+ puts("This software comes with ABSOLUTELY NO WARRANTY. "
+ "This is free software,\nand you are welcome to modify and "
+ "redistribute it under the GPL license.\n");
+
+ puts("Manage backups on Cloud services.\n");
+
+ puts("Usage: ");
+ printf(" %s -c put [OPTIONS...] <NAME> upload backup from STDIN into "
+ "the cloud service with given name.\n", my_progname);
+ printf(" %s -c get [OPTIONS...] <NAME> [FILES...] stream specified "
+ "backup or individual files from cloud service into STDOUT.\n",
+ my_progname);
+
+ puts("\nOptions:");
+ my_print_help(my_long_options);
+}
+
+static
+my_bool
+get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
+ char *argument __attribute__((unused)))
+{
+ switch (optid) {
+ case '?':
+ usage();
+ exit(0);
+ }
+
+ return(FALSE);
+}
+
+static const char *load_default_groups[]=
+ { "xbcloud", 0 };
+
+/*********************************************************************//**
+mask sensitive values on the command line */
+static
+void
+mask_args(int argc, char **argv)
+{
+ int i;
+ for (i = 0; i < argc-1; i++) {
+ int j = 0;
+ if (argv[i]) while (masked_args[j]) {
+ char *p;
+ if ((p = strstr(argv[i], masked_args[j]))) {
+ p += strlen(masked_args[j]);
+ while (*p && *p != '=') {
+ p++;
+ }
+ if (*p == '=') {
+ p++;
+ while (*p) {
+ *p++ = 'x';
+ }
+ }
+ }
+ j++;
+ }
+ }
+}
+
+static
+int parse_args(int argc, char **argv)
+{
+ const char *command;
+
+ if (argc < 2) {
+ fprintf(stderr, "Command isn't specified. "
+ "Supported commands are put and get\n");
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ command = argv[1];
+ argc--; argv++;
+
+ if (strcasecmp(command, "put") == 0) {
+ opt_mode = MODE_PUT;
+ } else if (strcasecmp(command, "get") == 0) {
+ opt_mode = MODE_GET;
+ } else if (strcasecmp(command, "delete") == 0) {
+ opt_mode = MODE_DELETE;
+ } else {
+ fprintf(stderr, "Unknown command %s. "
+ "Supported commands are put and get\n", command);
+ usage();
+ exit(EXIT_FAILURE);
+ }
+
+ load_defaults_or_exit("my", load_default_groups, &argc, &argv);
+
+ if (handle_options(&argc, &argv, my_long_options, get_one_option)) {
+ exit(EXIT_FAILURE);
+ }
+
+ /* make sure name is specified */
+ if (argc < 1) {
+ fprintf(stderr, "Backup name is required argument\n");
+ exit(EXIT_FAILURE);
+ }
+ opt_name = argv[0];
+ argc--; argv++;
+
+ /* validate arguments */
+ if (opt_storage == SWIFT) {
+ if (opt_swift_user == NULL) {
+ fprintf(stderr, "Swift user is not specified\n");
+ exit(EXIT_FAILURE);
+ }
+ if (opt_swift_container == NULL) {
+ fprintf(stderr,
+ "Swift container is not specified\n");
+ exit(EXIT_FAILURE);
+ }
+ if (opt_swift_auth_url == NULL) {
+ fprintf(stderr, "Swift auth URL is not specified\n");
+ exit(EXIT_FAILURE);
+ }
+ } else {
+ fprintf(stderr, "Swift is only supported storage API\n");
+ }
+
+ if (argc > 0) {
+ file_list = argv;
+ file_list_size = argc;
+ }
+
+ return(0);
+}
+
+static char *hex_md5(const unsigned char *hash, char *out)
+{
+ enum { hash_len = 16 };
+ char *p;
+ int i;
+
+ for (i = 0, p = out; i < hash_len; i++, p+=2) {
+ sprintf(p, "%02x", hash[i]);
+ }
+
+ return out;
+}
+
+/* If header starts with prefix it's value will be copied into output buffer */
+static
+int get_http_header(const char *prefix, const char *buffer,
+ char *out, size_t out_size)
+{
+ const char *beg, *end;
+ size_t len, prefix_len;
+
+ prefix_len = strlen(prefix);
+
+ if (strncasecmp(buffer, prefix, prefix_len) == 0) {
+ beg = buffer + prefix_len;
+ end = strchr(beg, '\r');
+
+ len = min<size_t>(end - beg, out_size - 1);
+
+ strncpy(out, beg, len);
+
+ out[len] = 0;
+
+ return 1;
+ }
+
+ return 0;
+}
+
+static
+size_t swift_auth_header_read_cb(char *ptr, size_t size, size_t nmemb,
+ void *data)
+{
+ swift_auth_info *info = (swift_auth_info*)(data);
+
+ get_http_header("X-Storage-Url: ", ptr,
+ info->url, array_elements(info->url));
+ get_http_header("X-Auth-Token: ", ptr,
+ info->token, array_elements(info->token));
+
+ return nmemb * size;
+}
+
+/*********************************************************************//**
+Authenticate against Swift TempAuth. Fills swift_auth_info struct.
+Uses creadentials privided as global variables.
+@returns true if access is granted and token received. */
+static
+bool
+swift_temp_auth(const char *auth_url, swift_auth_info *info)
+{
+ CURL *curl;
+ CURLcode res;
+ long http_code;
+ char *hdr_buf = NULL;
+ struct curl_slist *slist = NULL;
+
+ if (opt_swift_user == NULL) {
+ fprintf(stderr, "Swift user must be specified for TempAuth.\n");
+ return(false);
+ }
+
+ if (opt_swift_key == NULL) {
+ fprintf(stderr, "Swift key must be specified for TempAuth.\n");
+ return(false);
+ }
+
+ curl = curl_easy_init();
+
+ if (curl != NULL) {
+
+ hdr_buf = (char *)(calloc(14 + max(strlen(opt_swift_user),
+ strlen(opt_swift_key)), 1));
+
+ if (!hdr_buf) {
+ res = CURLE_FAILED_INIT;
+ goto cleanup;
+ }
+
+ sprintf(hdr_buf, "X-Auth-User: %s", opt_swift_user);
+ slist = curl_slist_append(slist, hdr_buf);
+
+ sprintf(hdr_buf, "X-Auth-Key: %s", opt_swift_key);
+ slist = curl_slist_append(slist, hdr_buf);
+
+ curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
+ curl_easy_setopt(curl, CURLOPT_URL, auth_url);
+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION,
+ swift_auth_header_read_cb);
+ curl_easy_setopt(curl, CURLOPT_HEADERDATA, info);
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+ if (opt_cacert != NULL)
+ curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
+ if (opt_insecure)
+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
+
+ res = curl_easy_perform(curl);
+
+ if (res != CURLE_OK) {
+ fprintf(stderr, "error: authentication failed: "
+ "curl_easy_perform(): %s\n",
+ curl_easy_strerror(res));
+ goto cleanup;
+ }
+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+ if (http_code != 200 &&
+ http_code != 204) {
+ fprintf(stderr, "error: authentication failed "
+ "with response code: %ld\n", http_code);
+ res = CURLE_LOGIN_DENIED;
+ goto cleanup;
+ }
+ } else {
+ res = CURLE_FAILED_INIT;
+ fprintf(stderr, "error: curl_easy_init() failed\n");
+ goto cleanup;
+ }
+
+cleanup:
+ if (hdr_buf) {
+ free(hdr_buf);
+ }
+ if (slist) {
+ curl_slist_free_all(slist);
+ }
+ if (curl) {
+ curl_easy_cleanup(curl);
+ }
+
+ if (res == CURLE_OK) {
+ /* check that we received token and storage URL */
+ if (*info->url == 0) {
+ fprintf(stderr, "error: malformed response: "
+ "X-Storage-Url is missing\n");
+ return(false);
+ }
+ if (*info->token == 0) {
+ fprintf(stderr, "error: malformed response: "
+ "X-Auth-Token is missing\n");
+ return(false);
+ }
+ return(true);
+ }
+
+ return(false);
+}
+
+static
+size_t
+write_null_cb(char *buffer, size_t size, size_t nmemb, void *stream)
+{
+ return fwrite(buffer, size, nmemb, stderr);
+}
+
+
+static
+size_t
+read_null_cb(char *ptr, size_t size, size_t nmemb, void *data)
+{
+ return 0;
+}
+
+
+static
+int
+swift_create_container(swift_auth_info *info, const char *name)
+{
+ char url[SWIFT_MAX_URL_SIZE];
+ char auth_token[SWIFT_MAX_HDR_SIZE];
+ CURLcode res;
+ long http_code;
+ CURL *curl;
+ struct curl_slist *slist = NULL;
+
+ snprintf(url, array_elements(url), "%s/%s", info->url, name);
+ snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s",
+ info->token);
+
+ curl = curl_easy_init();
+
+ if (curl != NULL) {
+ slist = curl_slist_append(slist, auth_token);
+ slist = curl_slist_append(slist, "Content-Length: 0");
+
+ curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
+ curl_easy_setopt(curl, CURLOPT_URL, url);
+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_null_cb);
+ curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_null_cb);
+ curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0L);
+ curl_easy_setopt(curl, CURLOPT_PUT, 1L);
+ if (opt_cacert != NULL)
+ curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
+ if (opt_insecure)
+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
+
+ res = curl_easy_perform(curl);
+
+ if (res != CURLE_OK) {
+ fprintf(stderr,
+ "error: curl_easy_perform() failed: %s\n",
+ curl_easy_strerror(res));
+ goto cleanup;
+ }
+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+ if (http_code != 201 && /* created */
+ http_code != 202 /* accepted (already exists) */) {
+ fprintf(stderr, "error: request failed "
+ "with response code: %ld\n", http_code);
+ res = CURLE_LOGIN_DENIED;
+ goto cleanup;
+ }
+ } else {
+ res = CURLE_FAILED_INIT;
+ fprintf(stderr, "error: curl_easy_init() failed\n");
+ goto cleanup;
+ }
+
+cleanup:
+ if (slist) {
+ curl_slist_free_all(slist);
+ }
+ if (curl) {
+ curl_easy_cleanup(curl);
+ }
+
+ return res;
+}
+
+
+/*********************************************************************//**
+Delete object with given url.
+@returns true if object deleted successfully. */
+static
+bool
+swift_delete_object(swift_auth_info *info, const char *url)
+{
+ char auth_token[SWIFT_MAX_HDR_SIZE];
+ CURLcode res;
+ long http_code;
+ CURL *curl;
+ struct curl_slist *slist = NULL;
+ bool ret = false;
+
+ snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s",
+ info->token);
+
+ curl = curl_easy_init();
+
+ if (curl != NULL) {
+ slist = curl_slist_append(slist, auth_token);
+
+ curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
+ curl_easy_setopt(curl, CURLOPT_URL, url);
+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+ curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
+ if (opt_cacert != NULL)
+ curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
+ if (opt_insecure)
+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
+
+ res = curl_easy_perform(curl);
+
+ if (res != CURLE_OK) {
+ fprintf(stderr,
+ "error: curl_easy_perform() failed: %s\n",
+ curl_easy_strerror(res));
+ goto cleanup;
+ }
+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+ if (http_code != 200 && /* OK */
+ http_code != 204 /* no content */) {
+ fprintf(stderr, "error: request failed "
+ "with response code: %ld\n", http_code);
+ goto cleanup;
+ }
+ ret = true;
+ } else {
+ fprintf(stderr, "error: curl_easy_init() failed\n");
+ goto cleanup;
+ }
+
+cleanup:
+ if (slist) {
+ curl_slist_free_all(slist);
+ }
+ if (curl) {
+ curl_easy_cleanup(curl);
+ }
+
+ return ret;
+}
+
+static int conn_upload_init(connection_info *conn);
+static void conn_buffer_updated(connection_info *conn);
+static connection_info *conn_new(global_io_info *global, ulong global_idx);
+static void conn_cleanup(connection_info *conn);
+static void conn_upload_retry(connection_info *conn);
+
+/* Check for completed transfers, and remove their easy handles */
+static void check_multi_info(global_io_info *g)
+{
+ char *eff_url;
+ CURLMsg *msg;
+ int msgs_left;
+ connection_info *conn;
+ CURL *easy;
+
+ while ((msg = curl_multi_info_read(g->multi, &msgs_left))) {
+ if (msg->msg == CURLMSG_DONE) {
+ easy = msg->easy_handle;
+ curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
+ curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL,
+ &eff_url);
+ curl_multi_remove_handle(g->multi, easy);
+ curl_easy_cleanup(easy);
+ conn->easy = NULL;
+ if (conn->chunk_acked) {
+ conn->chunk_uploaded = true;
+ fprintf(stderr, "%s is done\n", conn->hash);
+ } else {
+ fprintf(stderr, "error: chunk %zu '%s' %s "
+ "is not uploaded, but socket closed "
+ "(%zu bytes of %zu left to upload)\n",
+ conn->chunk_no,
+ conn->name,
+ conn->hash,
+ conn->chunk_size - conn->upload_size,
+ conn->chunk_size);
+ conn_upload_retry(conn);
+ }
+ }
+ }
+}
+
+/* Die if we get a bad CURLMcode somewhere */
+static void mcode_or_die(const char *where, CURLMcode code)
+{
+ if (code != CURLM_OK)
+ {
+ const char *s;
+ switch (code)
+ {
+ case CURLM_BAD_HANDLE:
+ s = "CURLM_BAD_HANDLE";
+ break;
+ case CURLM_BAD_EASY_HANDLE:
+ s = "CURLM_BAD_EASY_HANDLE";
+ break;
+ case CURLM_OUT_OF_MEMORY:
+ s = "CURLM_OUT_OF_MEMORY";
+ break;
+ case CURLM_INTERNAL_ERROR:
+ s = "CURLM_INTERNAL_ERROR";
+ break;
+ case CURLM_UNKNOWN_OPTION:
+ s = "CURLM_UNKNOWN_OPTION";
+ break;
+ case CURLM_LAST:
+ s = "CURLM_LAST";
+ break;
+ default:
+ s = "CURLM_unknown";
+ break;
+ case CURLM_BAD_SOCKET:
+ s = "CURLM_BAD_SOCKET";
+ fprintf(stderr, "error: %s returns (%d) %s\n",
+ where, code, s);
+ /* ignore this error */
+ return;
+ }
+ fprintf(stderr, "error: %s returns (%d) %s\n",
+ where, code, s);
+ assert(0);
+ }
+}
+
+/* Called by libev when we get action on a multi socket */
+static void event_cb(EV_P_ struct ev_io *w, int revents)
+{
+ global_io_info *global = (global_io_info*)(w->data);
+ CURLMcode rc;
+
+#if !(OLD_CURL_MULTI)
+ int action = (revents & EV_READ ? CURL_POLL_IN : 0) |
+ (revents & EV_WRITE ? CURL_POLL_OUT : 0);
+
+ do {
+ rc = curl_multi_socket_action(global->multi, w->fd, action,
+ &global->still_running);
+ } while (rc == CURLM_CALL_MULTI_PERFORM);
+#else
+ do {
+ rc = curl_multi_socket(global->multi, w->fd,
+ &global->still_running);
+ } while (rc == CURLM_CALL_MULTI_PERFORM);
+#endif
+ mcode_or_die("error: event_cb: curl_multi_socket_action", rc);
+ check_multi_info(global);
+ if (global->still_running <= 0) {
+ ev_timer_stop(global->loop, &global->timer_event);
+ }
+}
+
+static void remsock(curl_socket_t s, socket_info *fdp, global_io_info *global)
+{
+ if (fdp) {
+ if (fdp->evset) {
+ ev_io_stop(global->loop, &fdp->ev);
+ }
+ free(fdp);
+ }
+}
+
+static void setsock(socket_info *fdp, curl_socket_t s, CURL *easy, int action,
+ global_io_info *global)
+{
+ int kind = (action & CURL_POLL_IN ? (int)(EV_READ) : 0) |
+ (action & CURL_POLL_OUT ? (int)(EV_WRITE) : 0);
+
+ fdp->sockfd = s;
+ fdp->action = action;
+ fdp->easy = easy;
+ if (fdp->evset)
+ ev_io_stop(global->loop, &fdp->ev);
+ ev_io_init(&fdp->ev, event_cb, fdp->sockfd, kind);
+ fdp->ev.data = global;
+ fdp->evset = 1;
+ ev_io_start(global->loop, &fdp->ev);
+}
+
+static void addsock(curl_socket_t s, CURL *easy, int action,
+ global_io_info *global)
+{
+ socket_info *fdp = (socket_info *)(calloc(sizeof(socket_info), 1));
+
+ fdp->global = global;
+ setsock(fdp, s, easy, action, global);
+ curl_multi_assign(global->multi, s, fdp);
+}
+
+static int sock_cb(CURL *easy, curl_socket_t s, int what, void *cbp,
+ void *sockp)
+{
+ global_io_info *global = (global_io_info*)(cbp);
+ socket_info *fdp = (socket_info*)(sockp);
+
+ if (what == CURL_POLL_REMOVE) {
+ remsock(s, fdp, global);
+ } else {
+ if (!fdp) {
+ addsock(s, easy, what, global);
+ } else {
+ setsock(fdp, s, easy, what, global);
+ }
+ }
+ return 0;
+}
+
+/* Called by libev when our timeout expires */
+static void timer_cb(EV_P_ struct ev_timer *w, int revents)
+{
+ global_io_info *io_global = (global_io_info*)(w->data);
+ CURLMcode rc;
+
+#if !(OLD_CURL_MULTI)
+ do {
+ rc = curl_multi_socket_action(io_global->multi,
+ CURL_SOCKET_TIMEOUT, 0,
+ &io_global->still_running);
+ } while (rc == CURLM_CALL_MULTI_PERFORM);
+#else
+ do {
+ rc = curl_multi_socket_all(io_global->multi,
+ &io_global->still_running);
+ } while (rc == CURLM_CALL_MULTI_PERFORM);
+#endif
+ mcode_or_die("timer_cb: curl_multi_socket_action", rc);
+ check_multi_info(io_global);
+}
+
+static connection_info *get_current_connection(global_io_info *global)
+{
+ connection_info *conn = global->current_connection;
+ ulong i;
+
+ if (conn && conn->filled_size < conn->chunk_size)
+ return conn;
+
+ for (i = 0; i < opt_parallel; i++) {
+ conn = global->connections[i];
+ if (conn->chunk_uploaded || conn->filled_size == 0) {
+ global->current_connection = conn;
+ conn_upload_init(conn);
+ return conn;
+ }
+ }
+
+ return NULL;
+}
+
+/* This gets called whenever data is received from the input */
+static void input_cb(EV_P_ struct ev_io *w, int revents)
+{
+ global_io_info *io_global = (global_io_info *)(w->data);
+ connection_info *conn = get_current_connection(io_global);
+
+ if (conn == NULL)
+ return;
+
+ if (conn->filled_size < conn->chunk_size) {
+ if (revents & EV_READ) {
+ ssize_t nbytes = read(io_global->input_fd,
+ conn->buffer + conn->filled_size,
+ conn->chunk_size -
+ conn->filled_size);
+ if (nbytes > 0) {
+ conn->filled_size += nbytes;
+ conn_buffer_updated(conn);
+ } else if (nbytes < 0) {
+ if (errno != EAGAIN && errno != EINTR) {
+ char error[200];
+ my_strerror(error, sizeof(error),
+ errno);
+ fprintf(stderr, "error: failed to read "
+ "input stream (%s)\n", error);
+ /* failed to read input */
+ exit(1);
+ }
+ } else {
+ io_global->eof = 1;
+ ev_io_stop(io_global->loop, w);
+ }
+ }
+ }
+
+ assert(conn->filled_size <= conn->chunk_size);
+}
+
+static int swift_upload_read_cb(char *ptr, size_t size, size_t nmemb,
+ void *data)
+{
+ size_t realsize;
+
+ connection_info *conn = (connection_info*)(data);
+
+ if (conn->filled_size == conn->upload_size &&
+ conn->upload_size < conn->chunk_size && !conn->global->eof) {
+ ssize_t nbytes;
+ assert(conn->global->current_connection == conn);
+ do {
+ nbytes = read(conn->global->input_fd,
+ conn->buffer + conn->filled_size,
+ conn->chunk_size - conn->filled_size);
+ } while (nbytes == -1 && errno == EAGAIN);
+ if (nbytes > 0) {
+ conn->filled_size += nbytes;
+ conn_buffer_updated(conn);
+ } else {
+ conn->global->eof = 1;
+ }
+ }
+
+ realsize = min(size * nmemb, conn->filled_size - conn->upload_size);
+
+ memcpy(ptr, conn->buffer + conn->upload_size, realsize);
+ conn->upload_size += realsize;
+
+ assert(conn->filled_size <= conn->chunk_size);
+ assert(conn->upload_size <= conn->filled_size);
+
+ return realsize;
+}
+
+static
+size_t upload_header_read_cb(char *ptr, size_t size, size_t nmemb,
+ void *data)
+{
+ connection_info *conn = (connection_info *)(data);
+ char etag[33];
+
+ if (get_http_header("Etag: ", ptr, etag, array_elements(etag))) {
+ if (strcmp(conn->hash, etag) != 0) {
+ fprintf(stderr, "error: ETag mismatch\n");
+ exit(EXIT_FAILURE);
+ }
+ fprintf(stderr, "acked chunk %s\n", etag);
+ conn->chunk_acked = true;
+ }
+
+ return nmemb * size;
+}
+
+static int conn_upload_init(connection_info *conn)
+{
+ conn->filled_size = 0;
+ conn->upload_size = 0;
+ conn->chunk_uploaded = false;
+ conn->chunk_acked = false;
+ conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN;
+ conn->magic_verified = false;
+ conn->chunk_path_len = 0;
+ conn->chunk_type = XB_CHUNK_TYPE_UNKNOWN;
+ conn->payload_size = 0;
+ conn->upload_started = false;
+ conn->retry_count = 0;
+ if (conn->name != NULL) {
+ conn->name[0] = 0;
+ }
+
+ if (conn->easy != NULL) {
+ conn->easy = 0;
+ }
+
+ if (conn->slist != NULL) {
+ curl_slist_free_all(conn->slist);
+ conn->slist = NULL;
+ }
+
+ return 0;
+}
+
+static void conn_upload_prepare(connection_info *conn)
+{
+ gcry_md_hd_t md5;
+
+ gcry_md_open(&md5, GCRY_MD_MD5, 0);
+ gcry_md_write(md5, conn->buffer, conn->chunk_size);
+ hex_md5(gcry_md_read(md5, GCRY_MD_MD5), conn->hash);
+ gcry_md_close(md5);
+}
+
+static int conn_upload_start(connection_info *conn)
+{
+ char token_header[SWIFT_MAX_HDR_SIZE];
+ char object_url[SWIFT_MAX_URL_SIZE];
+ char content_len[200], etag[200];
+ global_io_info *global;
+ CURLMcode rc;
+
+ global = conn->global;
+
+ fprintf(stderr, "uploading chunk %s/%s/%s.%020zu "
+ "(md5: %s, size: %zu)\n",
+ global->container, global->backup_name, conn->name,
+ conn->chunk_no, conn->hash, conn->chunk_size);
+
+ snprintf(object_url, array_elements(object_url), "%s/%s/%s/%s.%020zu",
+ global->url, global->container, global->backup_name,
+ conn->name, conn->chunk_no);
+
+ snprintf(content_len, sizeof(content_len), "Content-Length: %lu",
+ (ulong)(conn->chunk_size));
+
+ snprintf(etag, sizeof(etag), "ETag: %s", conn->hash);
+
+ snprintf(token_header, array_elements(token_header),
+ "X-Auth-Token: %s", global->token);
+
+ conn->slist = curl_slist_append(conn->slist, token_header);
+ conn->slist = curl_slist_append(conn->slist,
+ "Connection: keep-alive");
+ conn->slist = curl_slist_append(conn->slist,
+ "Content-Type: "
+ "application/octet-stream");
+ conn->slist = curl_slist_append(conn->slist, content_len);
+ conn->slist = curl_slist_append(conn->slist, etag);
+
+ conn->easy = curl_easy_init();
+ if (!conn->easy) {
+ fprintf(stderr, "error: curl_easy_init() failed\n");
+ return 1;
+ }
+ curl_easy_setopt(conn->easy, CURLOPT_URL, object_url);
+ curl_easy_setopt(conn->easy, CURLOPT_READFUNCTION,
+ swift_upload_read_cb);
+ curl_easy_setopt(conn->easy, CURLOPT_READDATA, conn);
+ curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, opt_verbose);
+ curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error);
+ curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn);
+ curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 1L);
+ curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_TIME, 5L);
+ curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_LIMIT, 1024L);
+ curl_easy_setopt(conn->easy, CURLOPT_PUT, 1L);
+ curl_easy_setopt(conn->easy, CURLOPT_HTTPHEADER, conn->slist);
+ curl_easy_setopt(conn->easy, CURLOPT_HEADERFUNCTION,
+ upload_header_read_cb);
+ curl_easy_setopt(conn->easy, CURLOPT_HEADERDATA, conn);
+ curl_easy_setopt(conn->easy, CURLOPT_INFILESIZE,
+ (long) conn->chunk_size);
+ if (opt_cacert != NULL)
+ curl_easy_setopt(conn->easy, CURLOPT_CAINFO, opt_cacert);
+ if (opt_insecure)
+ curl_easy_setopt(conn->easy, CURLOPT_SSL_VERIFYPEER, FALSE);
+
+ rc = curl_multi_add_handle(conn->global->multi, conn->easy);
+ mcode_or_die("conn_upload_init: curl_multi_add_handle", rc);
+
+#if (OLD_CURL_MULTI)
+ do {
+ rc = curl_multi_socket_all(global->multi,
+ &global->still_running);
+ } while(rc == CURLM_CALL_MULTI_PERFORM);
+#endif
+
+ conn->upload_started = true;
+
+ return 0;
+}
+
+static void conn_cleanup(connection_info *conn)
+{
+ if (conn) {
+ free(conn->name);
+ free(conn->buffer);
+ if (conn->slist) {
+ curl_slist_free_all(conn->slist);
+ conn->slist = NULL;
+ }
+ if (conn->easy) {
+ curl_easy_cleanup(conn->easy);
+ conn->easy = NULL;
+ }
+ }
+ free(conn);
+}
+
+static void conn_upload_retry(connection_info *conn)
+{
+ /* already closed by cURL */
+ conn->easy = NULL;
+
+ if (conn->slist != NULL) {
+ curl_slist_free_all(conn->slist);
+ conn->slist = NULL;
+ }
+
+ if (conn->retry_count++ > 3) {
+ fprintf(stderr, "error: retry count limit reached\n");
+ exit(EXIT_FAILURE);
+ }
+
+ fprintf(stderr, "warning: retrying to upload chunk %zu of '%s'\n",
+ conn->chunk_no, conn->name);
+
+ conn->upload_size = 0;
+
+ conn_upload_start(conn);
+}
+
+static connection_info *conn_new(global_io_info *global, ulong global_idx)
+{
+ connection_info *conn;
+
+ conn = (connection_info *)(calloc(1, sizeof(connection_info)));
+ if (conn == NULL) {
+ goto error;
+ }
+
+ conn->global = global;
+ conn->global_idx = global_idx;
+ conn->buffer_size = SWIFT_CHUNK_SIZE;
+ if ((conn->buffer = (char *)(calloc(conn->buffer_size, 1))) ==
+ NULL) {
+ goto error;
+ }
+
+ return conn;
+
+error:
+ if (conn != NULL) {
+ conn_cleanup(conn);
+ }
+
+ fprintf(stderr, "error: out of memory\n");
+ exit(EXIT_FAILURE);
+
+ return NULL;
+}
+
+/*********************************************************************//**
+Handle input buffer updates. Parse chunk header and set appropriate
+buffer size. */
+static
+void
+conn_buffer_updated(connection_info *conn)
+{
+ bool ready_for_upload = false;
+
+ /* chunk header */
+ if (!conn->magic_verified &&
+ conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN) {
+ if (strncmp(XB_STREAM_CHUNK_MAGIC, conn->buffer,
+ sizeof(XB_STREAM_CHUNK_MAGIC) - 1) != 0) {
+
+ fprintf(stderr, "Error: magic expected\n");
+ exit(EXIT_FAILURE);
+ }
+ conn->magic_verified = true;
+ conn->chunk_path_len = uint4korr(conn->buffer
+ + PATH_LENGTH_OFFSET);
+ conn->chunk_type = (xb_chunk_type_t)
+ (conn->buffer[CHUNK_TYPE_OFFSET]);
+ conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN +
+ conn->chunk_path_len;
+ if (conn->chunk_type != XB_CHUNK_TYPE_EOF) {
+ conn->chunk_size += 16;
+ }
+ }
+
+ /* ordinary chunk */
+ if (conn->magic_verified &&
+ conn->payload_size == 0 &&
+ conn->chunk_type != XB_CHUNK_TYPE_EOF &&
+ conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN
+ + conn->chunk_path_len + 16) {
+
+ conn->payload_size = uint8korr(conn->buffer +
+ CHUNK_HEADER_CONSTANT_LEN +
+ conn->chunk_path_len);
+
+ conn->chunk_size = conn->payload_size + 4 + 16 +
+ conn->chunk_path_len +
+ CHUNK_HEADER_CONSTANT_LEN;
+
+ if (conn->name == NULL) {
+ conn->name = (char*)(malloc(conn->chunk_path_len + 1));
+ } else if (conn->name_len < conn->chunk_path_len + 1) {
+ conn->name = (char*)(realloc(conn->name,
+ conn->chunk_path_len + 1));
+ }
+ conn->name_len = conn->chunk_path_len + 1;
+
+ memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN,
+ conn->chunk_path_len);
+ conn->name[conn->chunk_path_len] = 0;
+
+ if (conn->buffer_size < conn->chunk_size) {
+ conn->buffer =
+ (char *)(realloc(conn->buffer, conn->chunk_size));
+ conn->buffer_size = conn->chunk_size;
+ }
+ }
+
+ /* EOF chunk has no payload */
+ if (conn->magic_verified &&
+ conn->chunk_type == XB_CHUNK_TYPE_EOF &&
+ conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN
+ + conn->chunk_path_len) {
+
+ if (conn->name == NULL) {
+ conn->name = (char*)(malloc(conn->chunk_path_len + 1));
+ } else if (conn->name_len < conn->chunk_path_len + 1) {
+ conn->name = (char*)(realloc(conn->name,
+ conn->chunk_path_len + 1));
+ }
+ conn->name_len = conn->chunk_path_len + 1;
+
+ memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN,
+ conn->chunk_path_len);
+ conn->name[conn->chunk_path_len] = 0;
+ }
+
+ if (conn->filled_size > 0 && conn->filled_size == conn->chunk_size) {
+ ready_for_upload = true;
+ }
+
+ /* start upload once recieved the size of the chunk */
+ if (!conn->upload_started && ready_for_upload) {
+ conn->chunk_no = file_chunk_count[conn->name]++;
+ conn_upload_prepare(conn);
+ conn_upload_start(conn);
+ }
+}
+
+static int init_input(global_io_info *io_global)
+{
+ ev_io_init(&io_global->input_event, input_cb, STDIN_FILENO, EV_READ);
+ io_global->input_event.data = io_global;
+ ev_io_start(io_global->loop, &io_global->input_event);
+
+ return 0;
+}
+
+/* Update the event timer after curl_multi library calls */
+static int multi_timer_cb(CURLM *multi, long timeout_ms, global_io_info *global)
+{
+ ev_timer_stop(global->loop, &global->timer_event);
+ if (timeout_ms > 0) {
+ double t = timeout_ms / 1000.0;
+ ev_timer_init(&global->timer_event, timer_cb, t, 0.);
+ ev_timer_start(global->loop, &global->timer_event);
+ } else {
+ timer_cb(global->loop, &global->timer_event, 0);
+ }
+ return 0;
+}
+
+static
+int swift_upload_parts(swift_auth_info *auth, const char *container,
+ const char *name)
+{
+ global_io_info io_global;
+ ulong i;
+#if (OLD_CURL_MULTI)
+ long timeout;
+#endif
+ CURLMcode rc;
+ int n_dirty_buffers;
+
+ memset(&io_global, 0, sizeof(io_global));
+
+ io_global.loop = ev_default_loop(0);
+ init_input(&io_global);
+ io_global.multi = curl_multi_init();
+ ev_timer_init(&io_global.timer_event, timer_cb, 0., 0.);
+ io_global.timer_event.data = &io_global;
+ io_global.connections = (connection_info **)
+ (calloc(opt_parallel, sizeof(connection_info)));
+ io_global.url = auth->url;
+ io_global.container = container;
+ io_global.backup_name = name;
+ io_global.token = auth->token;
+ for (i = 0; i < opt_parallel; i++) {
+ io_global.connections[i] = conn_new(&io_global, i);
+ }
+
+ /* setup the generic multi interface options we want */
+ curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
+ curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETDATA, &io_global);
+#if !(OLD_CURL_MULTI)
+ curl_multi_setopt(io_global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
+ curl_multi_setopt(io_global.multi, CURLMOPT_TIMERDATA, &io_global);
+ do {
+ rc = curl_multi_socket_action(io_global.multi,
+ CURL_SOCKET_TIMEOUT, 0,
+ &io_global.still_running);
+ } while (rc == CURLM_CALL_MULTI_PERFORM);
+#else
+ curl_multi_timeout(io_global.multi, &timeout);
+ if (timeout >= 0) {
+ multi_timer_cb(io_global.multi, timeout, &io_global);
+ }
+ do {
+ rc = curl_multi_socket_all(io_global.multi, &io_global.still_running);
+ } while(rc == CURLM_CALL_MULTI_PERFORM);
+#endif
+
+ ev_loop(io_global.loop, 0);
+ check_multi_info(&io_global);
+ curl_multi_cleanup(io_global.multi);
+
+ n_dirty_buffers = 0;
+ for (i = 0; i < opt_parallel; i++) {
+ connection_info *conn = io_global.connections[i];
+ if (conn && conn->upload_size != conn->filled_size) {
+ fprintf(stderr, "error: upload failed: %lu bytes left "
+ "in the buffer %s (uploaded = %d)\n",
+ (ulong)(conn->filled_size - conn->upload_size),
+ conn->name, conn->chunk_uploaded);
+ ++n_dirty_buffers;
+ }
+ }
+
+ for (i = 0; i < opt_parallel; i++) {
+ if (io_global.connections[i] != NULL) {
+ conn_cleanup(io_global.connections[i]);
+ }
+ }
+ free(io_global.connections);
+
+ if (n_dirty_buffers > 0) {
+ return(EXIT_FAILURE);
+ }
+
+ return 0;
+}
+
+struct download_buffer_info {
+ off_t offset;
+ size_t size;
+ size_t result_len;
+ char *buf;
+ curl_read_callback custom_header_callback;
+ void *custom_header_callback_data;
+};
+
+/*********************************************************************//**
+Callback to parse header of GET request on swift contaier. */
+static
+size_t fetch_buffer_header_cb(char *ptr, size_t size, size_t nmemb,
+ void *data)
+{
+ download_buffer_info *buffer_info = (download_buffer_info*)(data);
+ size_t buf_size;
+ char content_length_str[100];
+ char *endptr;
+
+ if (get_http_header("Content-Length: ", ptr,
+ content_length_str, sizeof(content_length_str))) {
+
+ buf_size = strtoull(content_length_str, &endptr, 10);
+
+ if (buffer_info->buf == NULL) {
+ buffer_info->buf = (char*)(malloc(buf_size));
+ buffer_info->size = buf_size;
+ }
+
+ if (buf_size > buffer_info->size) {
+ buffer_info->buf = (char*)
+ (realloc(buffer_info->buf, buf_size));
+ buffer_info->size = buf_size;
+ }
+
+ buffer_info->result_len = buf_size;
+ }
+
+ if (buffer_info->custom_header_callback) {
+ buffer_info->custom_header_callback(ptr, size, nmemb,
+ buffer_info->custom_header_callback_data);
+ }
+
+ return nmemb * size;
+}
+
+/*********************************************************************//**
+Write contents into string buffer */
+static
+size_t
+fetch_buffer_cb(char *buffer, size_t size, size_t nmemb, void *out_buffer)
+{
+ download_buffer_info *buffer_info = (download_buffer_info*)(out_buffer);
+
+ assert(buffer_info->size >= buffer_info->offset + size * nmemb);
+
+ memcpy(buffer_info->buf + buffer_info->offset, buffer, size * nmemb);
+ buffer_info->offset += size * nmemb;
+
+ return size * nmemb;
+}
+
+
+/*********************************************************************//**
+Downloads contents of URL into buffer. Caller is responsible for
+deallocating the buffer.
+@return pointer to a buffer or NULL */
+static
+char *
+swift_fetch_into_buffer(swift_auth_info *auth, const char *url,
+ char **buf, size_t *buf_size, size_t *result_len,
+ curl_read_callback header_callback,
+ void *header_callback_data)
+{
+ char auth_token[SWIFT_MAX_HDR_SIZE];
+ download_buffer_info buffer_info;
+ struct curl_slist *slist = NULL;
+ long http_code;
+ CURL *curl;
+ CURLcode res;
+
+ memset(&buffer_info, 0, sizeof(buffer_info));
+ buffer_info.buf = *buf;
+ buffer_info.size = *buf_size;
+ buffer_info.custom_header_callback = header_callback;
+ buffer_info.custom_header_callback_data = header_callback_data;
+
+ snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s",
+ auth->token);
+
+ curl = curl_easy_init();
+
+ if (curl != NULL) {
+ slist = curl_slist_append(slist, auth_token);
+
+ curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
+ curl_easy_setopt(curl, CURLOPT_URL, url);
+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer_info);
+ curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION,
+ fetch_buffer_header_cb);
+ curl_easy_setopt(curl, CURLOPT_HEADERDATA,
+ &buffer_info);
+ if (opt_cacert != NULL)
+ curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
+ if (opt_insecure)
+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
+
+ res = curl_easy_perform(curl);
+
+ if (res != CURLE_OK) {
+ fprintf(stderr,
+ "error: curl_easy_perform() failed: %s\n",
+ curl_easy_strerror(res));
+ goto cleanup;
+ }
+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+ if (http_code < 200 || http_code >= 300) {
+ fprintf(stderr, "error: request failed "
+ "with response code: %ld\n", http_code);
+ res = CURLE_LOGIN_DENIED;
+ goto cleanup;
+ }
+ } else {
+ res = CURLE_FAILED_INIT;
+ fprintf(stderr, "error: curl_easy_init() failed\n");
+ goto cleanup;
+ }
+
+cleanup:
+ if (slist) {
+ curl_slist_free_all(slist);
+ }
+ if (curl) {
+ curl_easy_cleanup(curl);
+ }
+
+ if (res == CURLE_OK) {
+ *buf = buffer_info.buf;
+ *buf_size = buffer_info.size;
+ *result_len = buffer_info.result_len;
+ return(buffer_info.buf);
+ }
+
+ free(buffer_info.buf);
+ *buf = NULL;
+ *buf_size = 0;
+ *result_len = 0;
+
+ return(NULL);
+}
+
+static
+container_list *
+container_list_new()
+{
+ container_list *list =
+ (container_list *)(calloc(1, sizeof(container_list)));
+
+ list->object_count = 1000;
+ list->objects = (object_info*)
+ (calloc(list->object_count, sizeof(object_info)));
+
+ if (list->objects == NULL) {
+ fprintf(stderr, "error: out of memory\n");
+ free(list);
+ return(NULL);
+ }
+
+ return(list);
+}
+
+static
+void
+container_list_free(container_list *list)
+{
+ free(list->content_json);
+ free(list->objects);
+ free(list);
+}
+
+static
+void
+container_list_add_object(container_list *list, const char *name,
+ const char *hash, size_t bytes)
+{
+ const size_t object_count_step = 1000;
+
+ if (list->idx >= list->object_count) {
+ list->objects = (object_info*)
+ realloc(list->objects,
+ (list->object_count + object_count_step) *
+ sizeof(object_info));
+ memset(list->objects + list->object_count, 0,
+ object_count_step * sizeof(object_info));
+ list->object_count += object_count_step;
+ }
+ assert(list->idx <= list->object_count);
+ safe_strcpy(list->objects[list->idx].name,
+ sizeof(list->objects[list->idx].name), name);
+ safe_strcpy(list->objects[list->idx].hash,
+ sizeof(list->objects[list->idx].hash), hash);
+
+ list->objects[list->idx].bytes = bytes;
+ ++list->idx;
+}
+
+
+/*********************************************************************//**
+Tokenize json string. Return array of tokens. Caller is responsoble for
+deallocating the array. */
+jsmntok_t *
+json_tokenise(char *json, size_t len, int initial_tokens)
+{
+ jsmn_parser parser;
+ jsmn_init(&parser);
+
+ unsigned int n = initial_tokens;
+ jsmntok_t *tokens = (jsmntok_t *)(malloc(sizeof(jsmntok_t) * n));
+
+ int ret = jsmn_parse(&parser, json, len, tokens, n);
+
+ while (ret == JSMN_ERROR_NOMEM)
+ {
+ n = n * 2 + 1;
+ tokens = (jsmntok_t*)(realloc(tokens, sizeof(jsmntok_t) * n));
+ ret = jsmn_parse(&parser, json, len, tokens, n);
+ }
+
+ if (ret == JSMN_ERROR_INVAL) {
+ fprintf(stderr, "error: invalid JSON string\n");
+
+ }
+ if (ret == JSMN_ERROR_PART) {
+ fprintf(stderr, "error: truncated JSON string\n");
+ }
+
+ return tokens;
+}
+
+/*********************************************************************//**
+Return true if token representation equal to given string. */
+static
+bool
+json_token_eq(const char *buf, jsmntok_t *t, const char *s)
+{
+ size_t len = strlen(s);
+
+ assert(t->end > t->start);
+
+ return((size_t)(t->end - t->start) == len &&
+ (strncmp(buf + t->start, s, len) == 0));
+}
+
+/*********************************************************************//**
+Copy given token as string. */
+static
+bool
+json_token_str(const char *buf, jsmntok_t *t, char *out, int out_size)
+{
+ size_t len = min(t->end - t->start, out_size - 1);
+
+ memcpy(out, buf + t->start, len);
+ out[len] = 0;
+
+ return(true);
+}
+
+/*********************************************************************//**
+Parse SWIFT container list response and fill output array with values
+sorted by object name. */
+static
+bool
+swift_parse_container_list(container_list *list)
+{
+ enum {MAX_DEPTH=20};
+ enum label_t {NONE, OBJECT};
+
+ char name[SWIFT_MAX_URL_SIZE];
+ char hash[33];
+ char bytes[30];
+ char *response = list->content_json;
+
+ struct stack_t {
+ jsmntok_t *t;
+ int n_items;
+ label_t label;
+ };
+
+ stack_t stack[MAX_DEPTH];
+ jsmntok_t *tokens;
+ int level;
+ size_t count = 0;
+
+ tokens = json_tokenise(list->content_json, list->content_length, 200);
+
+ stack[0].t = &tokens[0];
+ stack[0].label = NONE;
+ stack[0].n_items = 1;
+ level = 0;
+
+ for (size_t i = 0, j = 1; j > 0; i++, j--) {
+ jsmntok_t *t = &tokens[i];
+
+ assert(t->start != -1 && t->end != -1);
+ assert(level >= 0);
+
+ --stack[level].n_items;
+
+ switch (t->type) {
+ case JSMN_ARRAY:
+ case JSMN_OBJECT:
+ if (level < MAX_DEPTH - 1) {
+ level++;
+ }
+ stack[level].t = t;
+ stack[level].label = NONE;
+ if (t->type == JSMN_ARRAY) {
+ stack[level].n_items = t->size;
+ j += t->size;
+ } else {
+ stack[level].n_items = t->size * 2;
+ j += t->size * 2;
+ }
+ break;
+ case JSMN_PRIMITIVE:
+ case JSMN_STRING:
+ if (stack[level].t->type == JSMN_OBJECT &&
+ stack[level].n_items % 2 == 1) {
+ /* key */
+ if (json_token_eq(response, t, "name")) {
+ json_token_str(response, &tokens[i + 1],
+ name, sizeof(name));
+ }
+ if (json_token_eq(response, t, "hash")) {
+ json_token_str(response, &tokens[i + 1],
+ hash, sizeof(hash));
+ }
+ if (json_token_eq(response, t, "bytes")) {
+ json_token_str(response, &tokens[i + 1],
+ bytes, sizeof(bytes));
+ }
+ }
+ break;
+ }
+
+ while (stack[level].n_items == 0 && level > 0) {
+ if (stack[level].t->type == JSMN_OBJECT
+ && level == 2) {
+ char *endptr;
+ container_list_add_object(list, name, hash,
+ strtoull(bytes, &endptr, 10));
+ ++count;
+ }
+ --level;
+ }
+ }
+
+ if (count == 0) {
+ list->final = true;
+ }
+
+ free(tokens);
+
+ return(true);
+}
+
+/*********************************************************************//**
+List swift container with given name. Return list of objects sorted by
+object name. */
+static
+container_list *
+swift_list(swift_auth_info *auth, const char *container, const char *path)
+{
+ container_list *list;
+ char url[SWIFT_MAX_URL_SIZE];
+
+ list = container_list_new();
+
+ while (!list->final) {
+
+ /* download the list in json format */
+ snprintf(url, array_elements(url),
+ "%s/%s?format=json&limit=1000%s%s%s%s",
+ auth->url, container, path ? "&prefix=" : "",
+ path ? path : "", list->idx > 0 ? "&marker=" : "",
+ list->idx > 0 ?
+ list->objects[list->idx - 1].name : "");
+
+ list->content_json = swift_fetch_into_buffer(auth, url,
+ &list->content_json, &list->content_bufsize,
+ &list->content_length, NULL, NULL);
+
+ if (list->content_json == NULL) {
+ container_list_free(list);
+ return(NULL);
+ }
+
+ /* parse downloaded list */
+ if (!swift_parse_container_list(list)) {
+ fprintf(stderr, "error: unable to parse "
+ "container list\n");
+ container_list_free(list);
+ return(NULL);
+ }
+ }
+
+ return(list);
+}
+
+
+/*********************************************************************//**
+Return true if chunk is a part of backup with given name. */
+static
+bool
+chunk_belongs_to(const char *chunk_name, const char *backup_name)
+{
+ size_t backup_name_len = strlen(backup_name);
+
+ return((strlen(chunk_name) > backup_name_len)
+ && (chunk_name[backup_name_len] == '/')
+ && strncmp(chunk_name, backup_name, backup_name_len) == 0);
+}
+
+/*********************************************************************//**
+Return true if chunk is in given list. */
+static
+bool
+chunk_in_list(const char *chunk_name, char **list, int list_size)
+{
+ size_t chunk_name_len;
+
+ if (list_size == 0) {
+ return(true);
+ }
+
+ chunk_name_len = strlen(chunk_name);
+ if (chunk_name_len < 20) {
+ return(false);
+ }
+
+ for (int i = 0; i < list_size; i++) {
+ size_t item_len = strlen(list[i]);
+
+ if ((strncmp(chunk_name - item_len + chunk_name_len - 21,
+ list[i], item_len) == 0)
+ && (chunk_name[chunk_name_len - 21] == '.')
+ && (chunk_name[chunk_name_len - item_len - 22] == '/')) {
+ return(true);
+ }
+ }
+
+ return(false);
+}
+
+static
+int swift_download(swift_auth_info *auth, const char *container,
+ const char *name)
+{
+ container_list *list;
+ char *buf = NULL;
+ size_t buf_size = 0;
+ size_t result_len = 0;
+
+ if ((list = swift_list(auth, container, name)) == NULL) {
+ return(CURLE_FAILED_INIT);
+ }
+
+ for (size_t i = 0; i < list->idx; i++) {
+ const char *chunk_name = list->objects[i].name;
+
+ if (chunk_belongs_to(chunk_name, name)
+ && chunk_in_list(chunk_name, file_list, file_list_size)) {
+ char url[SWIFT_MAX_URL_SIZE];
+
+ snprintf(url, sizeof(url), "%s/%s/%s",
+ auth->url, container, chunk_name);
+
+ if ((buf = swift_fetch_into_buffer(
+ auth, url, &buf, &buf_size, &result_len,
+ NULL, NULL)) == NULL) {
+ fprintf(stderr, "error: failed to download "
+ "chunk %s\n", chunk_name);
+ container_list_free(list);
+ return(CURLE_FAILED_INIT);
+ }
+
+ fwrite(buf, 1, result_len, stdout);
+ }
+ }
+
+ free(buf);
+
+ container_list_free(list);
+
+ return(CURLE_OK);
+}
+
+
+/*********************************************************************//**
+Delete backup with given name from given container.
+@return true if backup deleted successfully */
+static
+bool swift_delete(swift_auth_info *auth, const char *container,
+ const char *name)
+{
+ container_list *list;
+
+ if ((list = swift_list(auth, container, name)) == NULL) {
+ return(CURLE_FAILED_INIT);
+ }
+
+ for (size_t i = 0; i < list->object_count; i++) {
+ const char *chunk_name = list->objects[i].name;
+
+ if (chunk_belongs_to(chunk_name, name)) {
+ char url[SWIFT_MAX_URL_SIZE];
+
+ snprintf(url, sizeof(url), "%s/%s/%s",
+ auth->url, container, chunk_name);
+
+ fprintf(stderr, "delete %s\n", chunk_name);
+ if (!swift_delete_object(auth, url)) {
+ fprintf(stderr, "error: failed to delete "
+ "chunk %s\n", chunk_name);
+ container_list_free(list);
+ return(CURLE_FAILED_INIT);
+ }
+ }
+ }
+
+ container_list_free(list);
+
+ return(CURLE_OK);
+}
+
+/*********************************************************************//**
+Check if backup with given name exists.
+@return true if backup exists */
+static
+bool swift_backup_exists(swift_auth_info *auth, const char *container,
+ const char *backup_name)
+{
+ container_list *list;
+
+ if ((list = swift_list(auth, container, backup_name)) == NULL) {
+ fprintf(stderr, "error: unable to list container %s\n",
+ container);
+ exit(EXIT_FAILURE);
+ }
+
+ for (size_t i = 0; i < list->object_count; i++) {
+ if (chunk_belongs_to(list->objects[i].name, backup_name)) {
+ container_list_free(list);
+ return(true);
+ }
+ }
+
+ container_list_free(list);
+
+ return(false);
+}
+
+/*********************************************************************//**
+Fills auth_info with response from keystone response.
+@return true is response parsed successfully */
+static
+bool
+swift_parse_keystone_response_v2(char *response, size_t response_length,
+ swift_auth_info *auth_info)
+{
+ enum {MAX_DEPTH=20};
+ enum label_t {NONE, ACCESS, CATALOG, ENDPOINTS, TOKEN};
+
+ char filtered_url[SWIFT_MAX_URL_SIZE];
+ char public_url[SWIFT_MAX_URL_SIZE];
+ char region[SWIFT_MAX_URL_SIZE];
+ char id[SWIFT_MAX_URL_SIZE];
+ char token_id[SWIFT_MAX_URL_SIZE];
+ char type[SWIFT_MAX_URL_SIZE];
+
+ struct stack_t {
+ jsmntok_t *t;
+ int n_items;
+ label_t label;
+ };
+
+ stack_t stack[MAX_DEPTH];
+ jsmntok_t *tokens;
+ int level;
+
+ tokens = json_tokenise(response, response_length, 200);
+
+ stack[0].t = &tokens[0];
+ stack[0].label = NONE;
+ stack[0].n_items = 1;
+ level = 0;
+
+ for (size_t i = 0, j = 1; j > 0; i++, j--) {
+ jsmntok_t *t = &tokens[i];
+
+ assert(t->start != -1 && t->end != -1);
+ assert(level >= 0);
+
+ --stack[level].n_items;
+
+ switch (t->type) {
+ case JSMN_ARRAY:
+ case JSMN_OBJECT:
+ if (level < MAX_DEPTH - 1) {
+ level++;
+ }
+ stack[level].t = t;
+ stack[level].label = NONE;
+ if (t->type == JSMN_ARRAY) {
+ stack[level].n_items = t->size;
+ j += t->size;
+ } else {
+ stack[level].n_items = t->size * 2;
+ j += t->size * 2;
+ }
+ break;
+ case JSMN_PRIMITIVE:
+ case JSMN_STRING:
+ if (stack[level].t->type == JSMN_OBJECT &&
+ stack[level].n_items % 2 == 1) {
+ /* key */
+ if (json_token_eq(response, t, "access")) {
+ stack[level].label = ACCESS;
+ }
+ if (json_token_eq(response, t,
+ "serviceCatalog")) {
+ stack[level].label = CATALOG;
+ }
+ if (json_token_eq(response, t, "endpoints")) {
+ stack[level].label = ENDPOINTS;
+ }
+ if (json_token_eq(response, t, "token")) {
+ stack[level].label = TOKEN;
+ }
+ if (json_token_eq(response, t, "id")) {
+ json_token_str(response, &tokens[i + 1],
+ id, sizeof(id));
+ }
+ if (json_token_eq(response, t, "id")
+ && stack[level - 1].label == TOKEN) {
+ json_token_str(response, &tokens[i + 1],
+ token_id, sizeof(token_id));
+ }
+ if (json_token_eq(response, t, "region")) {
+ json_token_str(response, &tokens[i + 1],
+ region, sizeof(region));
+ }
+ if (json_token_eq(response, t, "publicURL")) {
+ json_token_str(response, &tokens[i + 1],
+ public_url, sizeof(public_url));
+ }
+ if (json_token_eq(response, t, "type")) {
+ json_token_str(response, &tokens[i + 1],
+ type, sizeof(type));
+ }
+ }
+ break;
+ }
+
+ while (stack[level].n_items == 0 && level > 0) {
+ if (stack[level].t->type == JSMN_OBJECT
+ && level == 6
+ && stack[level - 1].t->type == JSMN_ARRAY
+ && stack[level - 2].label == ENDPOINTS) {
+ if (opt_swift_region == NULL
+ || strcmp(opt_swift_region, region) == 0) {
+ strncpy(filtered_url, public_url,
+ sizeof(filtered_url));
+ }
+ }
+ if (stack[level].t->type == JSMN_OBJECT &&
+ level == 4 &&
+ stack[level - 1].t->type == JSMN_ARRAY &&
+ stack[level - 2].label == CATALOG) {
+ if (strcmp(type, "object-store") == 0) {
+ strncpy(auth_info->url, filtered_url,
+ sizeof(auth_info->url));
+ }
+ }
+ --level;
+ }
+ }
+
+ free(tokens);
+
+ strncpy(auth_info->token, token_id, sizeof(auth_info->token));
+
+ assert(level == 0);
+
+ if (*auth_info->token == 0) {
+ fprintf(stderr, "error: can not receive token from response\n");
+ return(false);
+ }
+
+ if (*auth_info->url == 0) {
+ fprintf(stderr, "error: can not get URL from response\n");
+ return(false);
+ }
+
+ return(true);
+}
+
+/*********************************************************************//**
+Authenticate against Swift TempAuth. Fills swift_auth_info struct.
+Uses creadentials privided as global variables.
+@returns true if access is granted and token received. */
+static
+bool
+swift_keystone_auth_v2(const char *auth_url, swift_auth_info *info)
+{
+ char tenant_arg[SWIFT_MAX_URL_SIZE];
+ char payload[SWIFT_MAX_URL_SIZE];
+ struct curl_slist *slist = NULL;
+ download_buffer_info buf_info;
+ long http_code;
+ CURLcode res;
+ CURL *curl;
+ bool auth_res = false;
+
+ memset(&buf_info, 0, sizeof(buf_info));
+
+ if (opt_swift_user == NULL) {
+ fprintf(stderr, "error: both --swift-user is required "
+ "for keystone authentication.\n");
+ return(false);
+ }
+
+ if (opt_swift_password == NULL) {
+ fprintf(stderr, "error: both --swift-password is required "
+ "for keystone authentication.\n");
+ return(false);
+ }
+
+ if (opt_swift_tenant != NULL && opt_swift_tenant_id != NULL) {
+ fprintf(stderr, "error: both --swift-tenant and "
+ "--swift-tenant-id specified for keystone "
+ "authentication.\n");
+ return(false);
+ }
+
+ if (opt_swift_tenant != NULL) {
+ snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"",
+ "tenantName", opt_swift_tenant);
+ } else if (opt_swift_tenant_id != NULL) {
+ snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"",
+ "tenantId", opt_swift_tenant_id);
+ } else {
+ *tenant_arg = 0;
+ }
+
+ snprintf(payload, sizeof(payload), "{\"auth\": "
+ "{\"passwordCredentials\": {\"username\":\"%s\","
+ "\"password\":\"%s\"}%s}}",
+ opt_swift_user, opt_swift_password, tenant_arg);
+
+ curl = curl_easy_init();
+
+ if (curl != NULL) {
+
+ slist = curl_slist_append(slist,
+ "Content-Type: application/json");
+ slist = curl_slist_append(slist,
+ "Accept: application/json");
+
+ curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
+ curl_easy_setopt(curl, CURLOPT_POST, 1L);
+ curl_easy_setopt(curl, CURLOPT_URL, auth_url);
+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload);
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info);
+ curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION,
+ fetch_buffer_header_cb);
+ curl_easy_setopt(curl, CURLOPT_HEADERDATA,
+ &buf_info);
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+
+ if (opt_cacert != NULL)
+ curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
+ if (opt_insecure)
+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
+
+ res = curl_easy_perform(curl);
+
+ if (res != CURLE_OK) {
+ fprintf(stderr,
+ "error: curl_easy_perform() failed: %s\n",
+ curl_easy_strerror(res));
+ goto cleanup;
+ }
+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+ if (http_code < 200 || http_code >= 300) {
+ fprintf(stderr, "error: request failed "
+ "with response code: %ld\n", http_code);
+ res = CURLE_LOGIN_DENIED;
+ goto cleanup;
+ }
+ } else {
+ res = CURLE_FAILED_INIT;
+ fprintf(stderr, "error: curl_easy_init() failed\n");
+ goto cleanup;
+ }
+
+ if (!swift_parse_keystone_response_v2(buf_info.buf,
+ buf_info.size, info)) {
+ goto cleanup;
+ }
+
+ auth_res = true;
+
+cleanup:
+ if (slist) {
+ curl_slist_free_all(slist);
+ }
+ if (curl) {
+ curl_easy_cleanup(curl);
+ }
+
+ free(buf_info.buf);
+
+ return(auth_res);
+}
+
+
+/*********************************************************************//**
+Fills auth_info with response from keystone response.
+@return true is response parsed successfully */
+static
+bool
+swift_parse_keystone_response_v3(char *response, size_t response_length,
+ swift_auth_info *auth_info)
+{
+ enum {MAX_DEPTH=20};
+ enum label_t {NONE, TOKEN, CATALOG, ENDPOINTS};
+
+ char url[SWIFT_MAX_URL_SIZE];
+ char filtered_url[SWIFT_MAX_URL_SIZE];
+ char region[SWIFT_MAX_URL_SIZE];
+ char interface[SWIFT_MAX_URL_SIZE];
+ char type[SWIFT_MAX_URL_SIZE];
+
+ struct stack_t {
+ jsmntok_t *t;
+ int n_items;
+ label_t label;
+ };
+
+ stack_t stack[MAX_DEPTH];
+ jsmntok_t *tokens;
+ int level;
+
+ tokens = json_tokenise(response, response_length, 200);
+
+ stack[0].t = &tokens[0];
+ stack[0].label = NONE;
+ stack[0].n_items = 1;
+ level = 0;
+
+ for (size_t i = 0, j = 1; j > 0; i++, j--) {
+ jsmntok_t *t = &tokens[i];
+
+ assert(t->start != -1 && t->end != -1);
+ assert(level >= 0);
+
+ --stack[level].n_items;
+
+ switch (t->type) {
+ case JSMN_ARRAY:
+ case JSMN_OBJECT:
+ if (level < MAX_DEPTH - 1) {
+ level++;
+ }
+ stack[level].t = t;
+ stack[level].label = NONE;
+ if (t->type == JSMN_ARRAY) {
+ stack[level].n_items = t->size;
+ j += t->size;
+ } else {
+ stack[level].n_items = t->size * 2;
+ j += t->size * 2;
+ }
+ break;
+ case JSMN_PRIMITIVE:
+ case JSMN_STRING:
+ if (stack[level].t->type == JSMN_OBJECT &&
+ stack[level].n_items % 2 == 1) {
+ /* key */
+ if (json_token_eq(response, t, "token")) {
+ stack[level].label = TOKEN;
+ fprintf(stderr, "token\n");
+ }
+ if (json_token_eq(response, t,
+ "catalog")) {
+ stack[level].label = CATALOG;
+ fprintf(stderr, "catalog\n");
+ }
+ if (json_token_eq(response, t, "endpoints")) {
+ stack[level].label = ENDPOINTS;
+ }
+ if (json_token_eq(response, t, "region")) {
+ json_token_str(response, &tokens[i + 1],
+ region, sizeof(region));
+ }
+ if (json_token_eq(response, t, "url")) {
+ json_token_str(response, &tokens[i + 1],
+ url, sizeof(url));
+ }
+ if (json_token_eq(response, t, "interface")) {
+ json_token_str(response, &tokens[i + 1],
+ interface, sizeof(interface));
+ }
+ if (json_token_eq(response, t, "type")) {
+ json_token_str(response, &tokens[i + 1],
+ type, sizeof(type));
+ }
+ }
+ break;
+ }
+
+ while (stack[level].n_items == 0 && level > 0) {
+ if (stack[level].t->type == JSMN_OBJECT
+ && level == 6
+ && stack[level - 1].t->type == JSMN_ARRAY
+ && stack[level - 2].label == ENDPOINTS) {
+ if ((opt_swift_region == NULL
+ || strcmp(opt_swift_region, region) == 0)
+ && strcmp(interface, "public") == 0) {
+ strncpy(filtered_url, url,
+ sizeof(filtered_url));
+ }
+ }
+ if (stack[level].t->type == JSMN_OBJECT &&
+ level == 4 &&
+ stack[level - 1].t->type == JSMN_ARRAY &&
+ stack[level - 2].label == CATALOG) {
+ if (strcmp(type, "object-store") == 0) {
+ strncpy(auth_info->url, filtered_url,
+ sizeof(auth_info->url));
+ }
+ }
+ --level;
+ }
+ }
+
+ free(tokens);
+
+ assert(level == 0);
+
+ if (*auth_info->url == 0) {
+ fprintf(stderr, "error: can not get URL from response\n");
+ return(false);
+ }
+
+ return(true);
+}
+
+/*********************************************************************//**
+Captures X-Subject-Token header. */
+static
+size_t keystone_v3_header_cb(char *ptr, size_t size, size_t nmemb, void *data)
+{
+ swift_auth_info *info = (swift_auth_info*)(data);
+
+ get_http_header("X-Subject-Token: ", ptr,
+ info->token, array_elements(info->token));
+
+ return nmemb * size;
+}
+
+/*********************************************************************//**
+Authenticate against Swift TempAuth. Fills swift_auth_info struct.
+Uses creadentials privided as global variables.
+@returns true if access is granted and token received. */
+static
+bool
+swift_keystone_auth_v3(const char *auth_url, swift_auth_info *info)
+{
+ char scope[SWIFT_MAX_URL_SIZE];
+ char domain[SWIFT_MAX_URL_SIZE];
+ char payload[SWIFT_MAX_URL_SIZE];
+ struct curl_slist *slist = NULL;
+ download_buffer_info buf_info;
+ long http_code;
+ CURLcode res;
+ CURL *curl;
+ bool auth_res = false;
+
+ memset(&buf_info, 0, sizeof(buf_info));
+ buf_info.custom_header_callback = keystone_v3_header_cb;
+ buf_info.custom_header_callback_data = info;
+
+ if (opt_swift_user == NULL) {
+ fprintf(stderr, "error: both --swift-user is required "
+ "for keystone authentication.\n");
+ return(false);
+ }
+
+ if (opt_swift_password == NULL) {
+ fprintf(stderr, "error: both --swift-password is required "
+ "for keystone authentication.\n");
+ return(false);
+ }
+
+ if (opt_swift_project_id != NULL && opt_swift_project != NULL) {
+ fprintf(stderr, "error: both --swift-project and "
+ "--swift-project-id specified for keystone "
+ "authentication.\n");
+ return(false);
+ }
+
+ if (opt_swift_domain_id != NULL && opt_swift_domain != NULL) {
+ fprintf(stderr, "error: both --swift-domain and "
+ "--swift-domain-id specified for keystone "
+ "authentication.\n");
+ return(false);
+ }
+
+ if (opt_swift_project_id != NULL && opt_swift_domain != NULL) {
+ fprintf(stderr, "error: both --swift-project-id and "
+ "--swift-domain specified for keystone "
+ "authentication.\n");
+ return(false);
+ }
+
+ if (opt_swift_project_id != NULL && opt_swift_domain_id != NULL) {
+ fprintf(stderr, "error: both --swift-project-id and "
+ "--swift-domain-id specified for keystone "
+ "authentication.\n");
+ return(false);
+ }
+
+ scope[0] = 0; domain[0] = 0;
+
+ if (opt_swift_domain != NULL) {
+ snprintf(domain, sizeof(domain),
+ ",{\"domain\":{\"name\":\"%s\"}}",
+ opt_swift_domain);
+ } else if (opt_swift_domain_id != NULL) {
+ snprintf(domain, sizeof(domain),
+ ",{\"domain\":{\"id\":\"%s\"}}",
+ opt_swift_domain_id);
+ }
+
+ if (opt_swift_project_id != NULL) {
+ snprintf(scope, sizeof(scope),
+ ",\"scope\":{\"project\":{\"id\":\"%s\"}}",
+ opt_swift_project_id);
+ } else if (opt_swift_project != NULL) {
+ snprintf(scope, sizeof(scope),
+ ",\"scope\":{\"project\":{\"name\":\"%s\"%s}}",
+ opt_swift_project, domain);
+ }
+
+ snprintf(payload, sizeof(payload), "{\"auth\":{\"identity\":"
+ "{\"methods\":[\"password\"],\"password\":{\"user\":"
+ "{\"name\":\"%s\",\"password\":\"%s\"%s}}}%s}}",
+ opt_swift_user, opt_swift_password,
+ *scope ? "" : ",\"domain\":{\"id\":\"default\"}",
+ scope);
+
+ curl = curl_easy_init();
+
+ if (curl != NULL) {
+
+ slist = curl_slist_append(slist,
+ "Content-Type: application/json");
+ slist = curl_slist_append(slist,
+ "Accept: application/json");
+
+ curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose);
+ curl_easy_setopt(curl, CURLOPT_POST, 1L);
+ curl_easy_setopt(curl, CURLOPT_URL, auth_url);
+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload);
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info);
+ curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION,
+ fetch_buffer_header_cb);
+ curl_easy_setopt(curl, CURLOPT_HEADERDATA,
+ &buf_info);
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+
+ if (opt_cacert != NULL)
+ curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert);
+ if (opt_insecure)
+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
+
+ res = curl_easy_perform(curl);
+
+ if (res != CURLE_OK) {
+ fprintf(stderr,
+ "error: curl_easy_perform() failed: %s\n",
+ curl_easy_strerror(res));
+ goto cleanup;
+ }
+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+ if (http_code < 200 || http_code >= 300) {
+ fprintf(stderr, "error: request failed "
+ "with response code: %ld\n", http_code);
+ res = CURLE_LOGIN_DENIED;
+ goto cleanup;
+ }
+ } else {
+ res = CURLE_FAILED_INIT;
+ fprintf(stderr, "error: curl_easy_init() failed\n");
+ goto cleanup;
+ }
+
+ if (!swift_parse_keystone_response_v3(buf_info.buf,
+ buf_info.size, info)) {
+ goto cleanup;
+ }
+
+ auth_res = true;
+
+cleanup:
+ if (slist) {
+ curl_slist_free_all(slist);
+ }
+ if (curl) {
+ curl_easy_cleanup(curl);
+ }
+
+ free(buf_info.buf);
+
+ return(auth_res);
+}
+
+int main(int argc, char **argv)
+{
+ swift_auth_info info;
+ char auth_url[SWIFT_MAX_URL_SIZE];
+
+ MY_INIT(argv[0]);
+
+ /* handle_options in parse_args is destructive so
+ * we make a copy of our argument pointers so we can
+ * mask the sensitive values afterwards */
+ char **mask_argv = (char **)malloc(sizeof(char *) * (argc - 1));
+ memcpy(mask_argv, argv + 1, sizeof(char *) * (argc - 1));
+
+ if (parse_args(argc, argv)) {
+ return(EXIT_FAILURE);
+ }
+
+ mask_args(argc, mask_argv); /* mask args on cmdline */
+
+ curl_global_init(CURL_GLOBAL_ALL);
+
+ if (opt_swift_auth_version == NULL || *opt_swift_auth_version == '1') {
+ /* TempAuth */
+ snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sauth/v%s/",
+ opt_swift_auth_url, opt_swift_auth_version ?
+ opt_swift_auth_version : "1.0");
+
+ if (!swift_temp_auth(auth_url, &info)) {
+ fprintf(stderr, "error: failed to authenticate\n");
+ return(EXIT_FAILURE);
+ }
+
+ } else if (*opt_swift_auth_version == '2') {
+ /* Keystone v2 */
+ snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/tokens",
+ opt_swift_auth_url, opt_swift_auth_version);
+
+ if (!swift_keystone_auth_v2(auth_url, &info)) {
+ fprintf(stderr, "error: failed to authenticate\n");
+ return(EXIT_FAILURE);
+ }
+
+ } else if (*opt_swift_auth_version == '3') {
+ /* Keystone v3 */
+ snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/auth/tokens",
+ opt_swift_auth_url, opt_swift_auth_version);
+
+ if (!swift_keystone_auth_v3(auth_url, &info)) {
+ fprintf(stderr, "error: failed to authenticate\n");
+ exit(EXIT_FAILURE);
+ }
+
+ }
+
+ if (opt_swift_storage_url != NULL) {
+ snprintf(info.url, sizeof(info.url), "%s",
+ opt_swift_storage_url);
+ }
+
+ fprintf(stderr, "Object store URL: %s\n", info.url);
+
+ if (opt_mode == MODE_PUT) {
+
+ if (swift_create_container(&info, opt_swift_container) != 0) {
+ fprintf(stderr, "error: failed to create "
+ "container %s\n",
+ opt_swift_container);
+ return(EXIT_FAILURE);
+ }
+
+ if (swift_backup_exists(&info, opt_swift_container, opt_name)) {
+ fprintf(stderr, "error: backup named '%s' "
+ "already exists!\n",
+ opt_name);
+ return(EXIT_FAILURE);
+ }
+
+ if (swift_upload_parts(&info, opt_swift_container,
+ opt_name) != 0) {
+ fprintf(stderr, "error: upload failed\n");
+ return(EXIT_FAILURE);
+ }
+
+ } else if (opt_mode == MODE_GET) {
+
+ if (swift_download(&info, opt_swift_container, opt_name)
+ != CURLE_OK) {
+ fprintf(stderr, "error: download failed\n");
+ return(EXIT_FAILURE);
+ }
+
+ } else if (opt_mode == MODE_DELETE) {
+
+ if (swift_delete(&info, opt_swift_container, opt_name)
+ != CURLE_OK) {
+ fprintf(stderr, "error: delete failed\n");
+ return(EXIT_FAILURE);
+ }
+
+ } else {
+ fprintf(stderr, "Unknown command supplied.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ curl_global_cleanup();
+
+ return(EXIT_SUCCESS);
+}