Merging upstream version 1:10.11.7.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

24 files changed, 304 insertions, 58 deletions

diff --git a/storage/maria/aria_s3_copy.cc b/storage/maria/aria_s3_copy.cc
index 77c41ba4..b8a0f5b7 100644
--- a/storage/maria/aria_s3_copy.cc
+++ b/storage/maria/aria_s3_copy.cc
@@ -87,7 +87,9 @@ static struct my_option my_long_options[] =
    &opt_block_size, &opt_block_size, 0, GET_ULONG, REQUIRED_ARG,
    4*1024*1024, 64*1024, 16*1024*1024, MALLOC_OVERHEAD, 1024, 0 },
   {"s3_protocol_version", 'L',
-   "Protocol used to communication with S3. One of \"Auto\", \"Amazon\" or \"Original\".",
+   "Protocol used to communication with S3. One of \"Auto\", \"Legacy\", "
+   "\"Original\", \"Amazon\", \"Path\" or \"Domain\". "
+   "Note: \"Legacy\", \"Original\" and \"Amazon\" are deprecated.",
    &opt_protocol_version, &opt_protocol_version, &s3_protocol_typelib,
    GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"force", 'f', "Force copy even if target exists",
@@ -195,7 +197,7 @@ static void get_options(int *argc, char ***argv)
     my_exit(-1);
   }
   if (opt_s3_debug)
-    ms3_debug();
+    ms3_debug(1);
 
 } /* get_options */
 
@@ -218,9 +220,20 @@ int main(int argc, char** argv)
 
   ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size);
 
-  if (opt_protocol_version)
+  if (opt_protocol_version > 2)
   {
-    uint8_t protocol_version= (uint8_t) opt_protocol_version;
+    uint8_t protocol_version;
+    switch (opt_protocol_version)
+    {
+      case 3: /* Legacy means v1 */
+      case 4: /* Path means v1 */
+        protocol_version= 1;
+        break;
+      case 5: /* Domain means v2 */
+        protocol_version= 2;
+        break;
+    }
+
     ms3_set_option(global_s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION,
                    &protocol_version);
   }
diff --git a/storage/maria/ha_s3.cc b/storage/maria/ha_s3.cc
index 8c105522..0abb3f07 100644
--- a/storage/maria/ha_s3.cc
+++ b/storage/maria/ha_s3.cc
@@ -121,6 +121,29 @@ static void update_secret_key(MYSQL_THD thd,
   }
 }
 
+static void update_s3_debug(MYSQL_THD thd,
+                            struct st_mysql_sys_var *var
+                            __attribute__((unused)),
+                            void *var_ptr __attribute__((unused)),
+                            const void *save)
+{
+  char new_state= *(char *) save;
+  if (s3_debug != new_state)
+  {
+    s3_debug= new_state;
+    if (s3_hton)                                // If library is initalized
+    {
+      ms3_debug(new_state);
+      if (!new_state)
+      {
+        /* Ensure that all logging is written to log */
+        fflush(stderr);
+      }
+    }
+  }
+}
+
+
 /* Define system variables for S3 */
 
 static MYSQL_SYSVAR_ULONG(block_size, s3_block_size,
@@ -129,9 +152,9 @@ static MYSQL_SYSVAR_ULONG(block_size, s3_block_size,
        4*1024*1024, 65536, 16*1024*1024, 8192);
 
 static MYSQL_SYSVAR_BOOL(debug, s3_debug,
-       PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+       PLUGIN_VAR_RQCMDARG,
       "Generates trace file from libmarias3 on stderr for debugging",
-       0, 0, 0);
+       0, update_s3_debug, 0);
 
 static MYSQL_SYSVAR_BOOL(slave_ignore_updates, s3_slave_ignore_updates,
        PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
@@ -147,7 +170,10 @@ static MYSQL_SYSVAR_BOOL(replicate_alter_as_create_select,
 static MYSQL_SYSVAR_ENUM(protocol_version, s3_protocol_version,
                          PLUGIN_VAR_RQCMDARG,
                          "Protocol used to communication with S3. One of "
-                         "\"Auto\", \"Amazon\" or \"Original\".",
+                         "\"Auto\", \"Legacy\", \"Original\", \"Amazon\", "
+                         "\"Path\" or \"Domain\". "
+                         "Note: \"Legacy\", \"Original\" and \"Amazon\" are "
+                         "deprecated.",
                          NULL, NULL, 0, &s3_protocol_typelib);
 
 static MYSQL_SYSVAR_ULONG(pagecache_age_threshold,
@@ -1048,7 +1074,7 @@ static int ha_s3_init(void *p)
   s3_pagecache.big_block_free= s3_free;
   s3_init_library();
   if (s3_debug)
-    ms3_debug();
+    ms3_debug(1);
 
   struct s3_func s3f_real =
   {
diff --git a/storage/maria/libmarias3/README.rst b/storage/maria/libmarias3/README.rst
index 2d9a7f49..b318e4ae 100644
--- a/storage/maria/libmarias3/README.rst
+++ b/storage/maria/libmarias3/README.rst
@@ -45,6 +45,17 @@ You will need the following OS environment variables set to run the tests:
 | S3NOVERIFY | Set to ``1`` if the host should not use SSL verification |
 +------------+----------------------------------------------------------+
 
+If you have minion installed, you should be able to use same settings as used by
+MariaDB mtr s3 tests:
+
+export S3KEY=minio
+export S3SECRET=minioadmin
+export S3REGION=
+export S3BUCKET=storage-engine
+export S3HOST=127.0.0.1
+export S3PORT=9000
+export S3USEHTTP=1
+
 The test suite is automatically built along with the library and can be executed with ``make check`` or ``make distcheck``.
 
 Before pushing, please ALWAYS ensure that ``make check`` and ``make distcheck`` works!
diff --git a/storage/maria/libmarias3/docs/api/functions.rst b/storage/maria/libmarias3/docs/api/functions.rst
index b30fac92..9be9718e 100644
--- a/storage/maria/libmarias3/docs/api/functions.rst
+++ b/storage/maria/libmarias3/docs/api/functions.rst
@@ -147,7 +147,7 @@ Example
    res= ms3_list(ms3, s3bucket, NULL, &list);
    if (res)
    {
-       printf("Error occured: %d\n", res);
+       printf("Error occurred: %d\n", res);
        return;
    }
    list_it= list;
@@ -220,7 +220,7 @@ Example
    res= ms3_put(ms3, s3bucket, "test/ms3.txt", (const uint8_t*)test_string, strlen(test_string));
    if (res)
    {
-       printf("Error occured: %d\n", res);
+       printf("Error occurred: %d\n", res);
        return;
    }
    ms3_deinit(ms3);
@@ -290,7 +290,7 @@ Example
    res= ms3_get(ms3, s3bucket, "test/ms3.txt", &data, &length);
    if (res)
    {
-       printf("Error occured: %d\n", res);
+       printf("Error occurred: %d\n", res);
        return;
    }
    printf("File contents: %s\n", data);
@@ -348,7 +348,7 @@ Example
    res = ms3_delete(ms3, s3bucket, "test/ms3.txt");
    if (res)
    {
-       printf("Error occured: %d\n", res);
+       printf("Error occurred: %d\n", res);
        return;
    }
    ms3_deinit(ms3);
@@ -384,7 +384,7 @@ Example
    res= ms3_status(ms3, s3bucket, "test/ms3.txt", &status);
    if (res)
    {
-       printf("Error occured: %d\n", res);
+       printf("Error occurred: %d\n", res);
        return;
    }
    printf("File length: %ld\n", status.length);
diff --git a/storage/maria/libmarias3/docs/api/types.rst b/storage/maria/libmarias3/docs/api/types.rst
index eba57466..4ef620b9 100644
--- a/storage/maria/libmarias3/docs/api/types.rst
+++ b/storage/maria/libmarias3/docs/api/types.rst
@@ -49,6 +49,8 @@ Constants
    * ``MS3_OPT_BUFFER_CHUNK_SIZE`` - Set the chunk size in bytes for the receive buffer. Default is 1MB. If you are receiving a large file a realloc will have to happen every time the buffer is full. For performance reasons you may want to increase the size of this buffer to reduce the reallocs and associated memory copies. The ``value`` parameter of :c:func:`ms3_set_option` should be a pointer to a :c:type:`size_t` greater than 1.
    * ``MS3_OPT_FORCE_LIST_VERSION`` - An internal option for the regression suite only. The ``value`` parameter of :c:func:`ms3_set_option` should be a pointer to a :c:type:`uint8_t` of value ``1`` or ``2``
    * ``MS3_OPT_FORCE_PROTOCOL_VERSION`` - Set to 1 to force talking to the S3 server using version 1 of the List Bucket API, this is for S3 compatible servers. Set to 2 to force talking to the S3 server version 2 of the List Bucket API. This is for use when the autodetect bsaed on providing a base_domain does the wrong thing. The ``value`` parameter of :c:func:`ms3_set_option` should be a pointer to a :c:type:`uint8_t` of value ``1`` or ``2``
+   * ``MS3_OPT_READ_CB`` - Custom read callback for :c:func:`ms3_get`. The ``value`` parameter of :c:func:`ms3_set_option` should be a :c:type:`ms3_read_callback` function.
+   * ``MS3_OPT_USER_DATA`` - User data for the custom read callback. The ``value`` parameter of :c:func:`ms3_set_option` is the pointer that will be passed as the ``userdata`` argument of the callback.
 
 Built-In Types
 ==============
diff --git a/storage/maria/libmarias3/libmarias3/marias3.h b/storage/maria/libmarias3/libmarias3/marias3.h
index 80b5e77c..80b1ef78 100644
--- a/storage/maria/libmarias3/libmarias3/marias3.h
+++ b/storage/maria/libmarias3/libmarias3/marias3.h
@@ -55,6 +55,12 @@ typedef void *(*ms3_realloc_callback)(void *ptr, size_t size);
 typedef char *(*ms3_strdup_callback)(const char *str);
 typedef void *(*ms3_calloc_callback)(size_t nmemb, size_t size);
 
+/** The callback function for MS3_OPT_READ_CB. The function and the user data
+ * set with MS3_OPT_USER_DATA are passed to Curl. For more information, refer
+ * to CURLOPT_WRITE_FUNCTION. */
+typedef size_t (*ms3_read_callback)(void *buffer, size_t size,
+                                    size_t nitems, void *userdata);
+
 enum ms3_error_code_t
 {
   MS3_ERR_NONE,
@@ -70,6 +76,7 @@ enum ms3_error_code_t
   MS3_ERR_SERVER,
   MS3_ERR_TOO_BIG,
   MS3_ERR_AUTH_ROLE,
+  MS3_ERR_ENDPOINT,
   MS3_ERR_MAX // Always the last error
 };
 
@@ -82,6 +89,8 @@ enum ms3_set_option_t
   MS3_OPT_BUFFER_CHUNK_SIZE,
   MS3_OPT_FORCE_LIST_VERSION,
   MS3_OPT_FORCE_PROTOCOL_VERSION,
+  MS3_OPT_READ_CB,
+  MS3_OPT_USER_DATA,
   MS3_OPT_PORT_NUMBER
 };
 
@@ -124,7 +133,7 @@ MS3_API
 const char *ms3_error(uint8_t errcode);
 
 MS3_API
-void ms3_debug(void);
+void ms3_debug(int debug_state);
 
 MS3_API
 uint8_t ms3_list(ms3_st *ms3, const char *bucket, const char *prefix,
diff --git a/storage/maria/libmarias3/src/assume_role.c b/storage/maria/libmarias3/src/assume_role.c
index 255b1eca..4135504c 100644
--- a/storage/maria/libmarias3/src/assume_role.c
+++ b/storage/maria/libmarias3/src/assume_role.c
@@ -373,7 +373,7 @@ build_assume_role_request_headers(CURL *curl, struct curl_slist **head,
   time_t now;
   struct tm tmp_tm;
   char headerbuf[3072];
-  char secrethead[45];
+  char secrethead[MAX_S3_SECRET_LENGTH + S3_SECRET_EXTRA_LENGTH];
   char date[9];
   char sha256hash[65];
   char post_hash[65];
@@ -445,7 +445,7 @@ build_assume_role_request_headers(CURL *curl, struct curl_slist **head,
 
   // User signing key hash
   // Date hashed using AWS4:secret_key
-  snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", 40, secret);
+  snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", MAX_S3_SECRET_LENGTH, secret);
   strftime(headerbuf, sizeof(headerbuf), "%Y%m%d", &tmp_tm);
   hmac_sha256((uint8_t *)secrethead, strlen(secrethead), (uint8_t *)headerbuf,
               strlen(headerbuf), hmac_hash);
diff --git a/storage/maria/libmarias3/src/error.c b/storage/maria/libmarias3/src/error.c
index edf95d05..2723c846 100644
--- a/storage/maria/libmarias3/src/error.c
+++ b/storage/maria/libmarias3/src/error.c
@@ -35,5 +35,7 @@ const char *errmsgs[] =
   "Authentication error",
   "File not found",
   "S3 server error",
-  "Data too big. Maximum data size is 4GB"
+  "Data too big. Maximum data size is 4GB",
+  "Error in role",
+  "Endpoint permanently moved"
 };
diff --git a/storage/maria/libmarias3/src/include.am b/storage/maria/libmarias3/src/include.am
index b8c07a6f..f786df3b 100644
--- a/storage/maria/libmarias3/src/include.am
+++ b/storage/maria/libmarias3/src/include.am
@@ -18,7 +18,7 @@ lib_LTLIBRARIES+= src/libmarias3.la
 src_libmarias3_la_SOURCES=
 src_libmarias3_la_LIBADD=
 src_libmarias3_la_LDFLAGS=
-src_libmarias3_la_CFLAGS= -DBUILDING_MS3
+src_libmarias3_la_CFLAGS= -DBUILDING_MS3 -fPIC
 
 src_libmarias3_la_SOURCES+= src/marias3.c
 src_libmarias3_la_SOURCES+= src/request.c
diff --git a/storage/maria/libmarias3/src/marias3.c b/storage/maria/libmarias3/src/marias3.c
index 74d7233a..b146da55 100644
--- a/storage/maria/libmarias3/src/marias3.c
+++ b/storage/maria/libmarias3/src/marias3.c
@@ -52,7 +52,7 @@ static void locking_function(int mode, int n, const char *file, int line)
     pthread_mutex_unlock(&(mutex_buf[n]));
 }
 
-static int curl_needs_openssl_locking()
+static int curl_needs_openssl_locking(void)
 {
   curl_version_info_data *data = curl_version_info(CURLVERSION_NOW);
 
@@ -214,6 +214,8 @@ ms3_st *ms3_init(const char *s3key, const char *s3secret,
   ms3->list_container.start = NULL;
   ms3->list_container.pool_list = NULL;
   ms3->list_container.pool_free = 0;
+  ms3->read_cb= 0;
+  ms3->user_data= 0;
 
   ms3->iam_role = NULL;
   ms3->role_key = NULL;
@@ -354,14 +356,16 @@ const char *ms3_server_error(ms3_st *ms3)
   return ms3->last_error;
 }
 
-void ms3_debug(void)
+void ms3_debug(int debug_state)
 {
   bool state = ms3debug_get();
-  ms3debug_set(!state);
-
-  if (state)
+  if (state != (bool) debug_state)
   {
-    ms3debug("enabling debug");
+    ms3debug_set((bool) debug_state);
+    if (debug_state)
+    {
+      ms3debug("enabling debug");
+    }
   }
 }
 
@@ -449,15 +453,23 @@ uint8_t ms3_get(ms3_st *ms3, const char *bucket, const char *key,
   buf.data = NULL;
   buf.length = 0;
 
-  if (!ms3 || !bucket || !key || key[0] == '\0' || !data || !length)
+  if (!ms3 || !bucket || !key || key[0] == '\0')
+  {
+    return MS3_ERR_PARAMETER;
+  }
+  else if (!ms3->read_cb && (!data || !length))
   {
     return MS3_ERR_PARAMETER;
   }
 
   res = execute_request(ms3, MS3_CMD_GET, bucket, key, NULL, NULL, NULL, NULL, 0,
                         NULL, &buf);
-  *data = buf.data;
-  *length = buf.length;
+  if (!ms3->read_cb)
+  {
+    *data = buf.data;
+    *length = buf.length;
+  }
+
   return res;
 }
 
@@ -617,7 +629,7 @@ uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value)
         return MS3_ERR_PARAMETER;
       }
 
-      ms3->list_version = protocol_version;
+      ms3->protocol_version = protocol_version;
       break;
     }
 
@@ -634,6 +646,24 @@ uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value)
       ms3->port = port_number;
       break;
     }
+
+    case MS3_OPT_READ_CB:
+    {
+      if (!value)
+      {
+        return MS3_ERR_PARAMETER;
+      }
+
+      ms3->read_cb = value;
+      break;
+    }
+
+    case MS3_OPT_USER_DATA:
+    {
+      ms3->user_data = value;
+      break;
+    }
+
     default:
       return MS3_ERR_PARAMETER;
   }
diff --git a/storage/maria/libmarias3/src/request.c b/storage/maria/libmarias3/src/request.c
index 26165474..b2924f9d 100644
--- a/storage/maria/libmarias3/src/request.c
+++ b/storage/maria/libmarias3/src/request.c
@@ -413,7 +413,7 @@ static uint8_t build_request_headers(CURL *curl, struct curl_slist **head,
   time_t now;
   struct tm tmp_tm;
   char headerbuf[3072];
-  char secrethead[45];
+  char secrethead[MAX_S3_SECRET_LENGTH + S3_SECRET_EXTRA_LENGTH];
   char date[9];
   char sha256hash[65];
   char post_hash[65];
@@ -520,7 +520,7 @@ static uint8_t build_request_headers(CURL *curl, struct curl_slist **head,
 
   // User signing key hash
   // Date hashed using AWS4:secret_key
-  snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", 40, secret);
+  snprintf(secrethead, sizeof(secrethead), "AWS4%.*s", MAX_S3_SECRET_LENGTH, secret);
   strftime(headerbuf, sizeof(headerbuf), "%Y%m%d", &tmp_tm);
   hmac_sha256((uint8_t *)secrethead, strlen(secrethead), (uint8_t *)headerbuf,
               strlen(headerbuf), hmac_hash);
@@ -829,9 +829,19 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket,
   if (ms3->port)
     curl_easy_setopt(curl, CURLOPT_PORT, (long)ms3->port);
 
+  if (ms3->read_cb && cmd == MS3_CMD_GET)
+  {
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, ms3->read_cb);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, ms3->user_data);
+  }
+  else
+  {
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem);
+  }
+
+  curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, ms3->buffer_chunk_size);
   curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback);
-  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback);
-  curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem);
   curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
   curl_res = curl_easy_perform(curl);
 
@@ -848,6 +858,18 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket,
   curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
   ms3debug("Response code: %ld", response_code);
 
+  if (response_code == 301)
+  {
+    char *message = parse_error_message((char *)mem.data, mem.length);
+
+    if (message)
+    {
+      ms3debug("Response message: %s", message);
+    }
+
+    set_error_nocopy(ms3, message);
+    res = MS3_ERR_ENDPOINT;
+  }
   if (response_code == 404)
   {
     char *message = parse_error_message((char *)mem.data, mem.length);
diff --git a/storage/maria/libmarias3/src/request.h b/storage/maria/libmarias3/src/request.h
index 9ce8bb5c..29ea79b3 100644
--- a/storage/maria/libmarias3/src/request.h
+++ b/storage/maria/libmarias3/src/request.h
@@ -26,6 +26,8 @@
 // Maxmum S3 file size is 1024 bytes so for protection we make the maximum
 // URI length this
 #define MAX_URI_LENGTH 1024
+#define MAX_S3_SECRET_LENGTH 128
+#define S3_SECRET_EXTRA_LENGTH 5
 
 #define READ_BUFFER_DEFAULT_SIZE 1024*1024
 
diff --git a/storage/maria/libmarias3/src/response.c b/storage/maria/libmarias3/src/response.c
index 4e976aba..da5d6b3c 100644
--- a/storage/maria/libmarias3/src/response.c
+++ b/storage/maria/libmarias3/src/response.c
@@ -395,7 +395,7 @@ uint8_t parse_role_list_response(const char *data, size_t length, char *role_nam
             }
           }
           while ((member = xml_node_child(roles, ++roles_it)));
-          if (!strcmp(response_role_name, role_name))
+          if (response_role_name && !strcmp(response_role_name, role_name))
           {
               ms3debug("Role Found ARN = %s",response_role_arn);
               sprintf(arn, "%s", response_role_arn);
diff --git a/storage/maria/libmarias3/src/structs.h b/storage/maria/libmarias3/src/structs.h
index 34cbd817..30a7fff0 100644
--- a/storage/maria/libmarias3/src/structs.h
+++ b/storage/maria/libmarias3/src/structs.h
@@ -64,6 +64,8 @@ struct ms3_st
   bool first_run;
   char *path_buffer;
   char *query_buffer;
+  void *read_cb;
+  void *user_data;
   struct ms3_list_container_st list_container;
 };
 
diff --git a/storage/maria/libmarias3/src/xml.c b/storage/maria/libmarias3/src/xml.c
index 2c48a4ea..25bd125f 100644
--- a/storage/maria/libmarias3/src/xml.c
+++ b/storage/maria/libmarias3/src/xml.c
@@ -800,7 +800,7 @@ node_creation:;
 	return node;
 
 
-	/* A failure occured, so free all allocalted resources
+	/* A failure occurred, so free all allocalted resources
 	 */
 exit_failure:
 	if (tag_open) {
diff --git a/storage/maria/libmarias3/tests/error.c b/storage/maria/libmarias3/tests/error.c
index 9be45d44..1b38a81c 100644
--- a/storage/maria/libmarias3/tests/error.c
+++ b/storage/maria/libmarias3/tests/error.c
@@ -35,7 +35,7 @@ int main(int argc, char *argv[])
   (void) argv;
 
   // Enable here so cppcheck shows coverage
-  ms3_debug();
+  ms3_debug(1);
   ASSERT_NOT_NULL(ms3);
   errmsg = ms3_error(255);
   ASSERT_STREQ(errmsg, "No such error code");
diff --git a/storage/maria/libmarias3/tests/include.am b/storage/maria/libmarias3/tests/include.am
index 1cb25a1c..fc4b267b 100644
--- a/storage/maria/libmarias3/tests/include.am
+++ b/storage/maria/libmarias3/tests/include.am
@@ -67,3 +67,7 @@ t_list_LDADD= src/libmarias3.la
 check_PROGRAMS+= t/list
 noinst_PROGRAMS+= t/list
 
+t_read_cb_SOURCES= tests/read_cb.c
+t_read_cb_LDADD= src/libmarias3.la
+check_PROGRAMS+= t/read_cb
+noinst_PROGRAMS+= t/read_cb
diff --git a/storage/maria/libmarias3/tests/read_cb.c b/storage/maria/libmarias3/tests/read_cb.c
new file mode 100644
index 00000000..57d50d59
--- /dev/null
+++ b/storage/maria/libmarias3/tests/read_cb.c
@@ -0,0 +1,129 @@
+/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
+ * Copyright 2023 MariaDB Corporation Ab. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301  USA
+ */
+
+#include <yatl/lite.h>
+#include <libmarias3/marias3.h>
+
+#define NUM_BYTES 64 * 1024
+
+/* Tests basic GET with a custom read callback */
+int total_reads = 0;
+
+static size_t read_cb(void *buf, size_t size, size_t nitems, void *userdata)
+{
+    char** dat = (char**)userdata;
+    char* ptr = *dat;
+    memcpy(ptr, buf, size * nitems);
+    ptr += size * nitems;
+    *dat = ptr;
+    total_reads++;
+    return nitems * size;
+}
+
+int main(int argc, char *argv[])
+{
+  int res;
+  int initial_reads;
+  uint8_t *data;
+  size_t length;
+  ms3_st *ms3;
+  size_t buffer_size;
+  char *test_string = malloc(NUM_BYTES);
+  char *dest = malloc(NUM_BYTES);
+  char* userdata = dest;
+  char *s3key = getenv("S3KEY");
+  char *s3secret = getenv("S3SECRET");
+  char *s3region = getenv("S3REGION");
+  char *s3bucket = getenv("S3BUCKET");
+  char *s3host = getenv("S3HOST");
+  char *s3noverify = getenv("S3NOVERIFY");
+  char *s3usehttp = getenv("S3USEHTTP");
+  char *s3port = getenv("S3PORT");
+
+  SKIP_IF_(!s3key, "Environment variable S3KEY missing");
+  SKIP_IF_(!s3secret, "Environment variable S3SECRET missing");
+  SKIP_IF_(!s3region, "Environment variable S3REGION missing");
+  SKIP_IF_(!s3bucket, "Environment variable S3BUCKET missing");
+
+  (void) argc;
+  (void) argv;
+
+  memset(test_string, 'a', NUM_BYTES);
+  memset(dest, 'b', NUM_BYTES);
+
+  ms3_library_init();
+  ms3 = ms3_init(s3key, s3secret, s3region, s3host);
+
+  if (s3noverify && !strcmp(s3noverify, "1"))
+  {
+    ms3_set_option(ms3, MS3_OPT_DISABLE_SSL_VERIFY, NULL);
+  }
+
+  if (s3usehttp && !strcmp(s3usehttp, "1"))
+  {
+    ms3_set_option(ms3, MS3_OPT_USE_HTTP, NULL);
+  }
+
+  if (s3port)
+  {
+    int port = atol(s3port);
+    ms3_set_option(ms3, MS3_OPT_PORT_NUMBER, &port);
+  }
+
+  ASSERT_NOT_NULL(ms3);
+
+  res = ms3_put(ms3, s3bucket, "test/read_cb.dat",
+                (const uint8_t *)test_string,
+                NUM_BYTES);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  res = ms3_set_option(ms3, MS3_OPT_READ_CB, read_cb);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  res = ms3_set_option(ms3, MS3_OPT_USER_DATA, &userdata);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  length = 0;
+  data = 0;
+  res = ms3_get(ms3, s3bucket, "test/read_cb.dat", &data, &length);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  ASSERT_EQ(data, 0);
+  ASSERT_EQ(length, 0);
+  ASSERT_EQ(memcmp(test_string, dest, NUM_BYTES), 0);
+
+  /** Test that the callbacks work with a smaller chunk size */
+  memset(dest, 'c', NUM_BYTES);
+  userdata = dest;
+  buffer_size = 1024;
+  res = ms3_set_option(ms3, MS3_OPT_BUFFER_CHUNK_SIZE, &buffer_size);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  initial_reads = total_reads;
+  res = ms3_get(ms3, s3bucket, "test/read_cb.dat", &data, &length);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  ASSERT_EQ(memcmp(test_string, dest, NUM_BYTES), 0);
+  ASSERT_TRUE_((total_reads - initial_reads) > initial_reads * 2,
+               "Expected more than %d reads but got only %d",
+               initial_reads * 2, total_reads - initial_reads);
+
+  res = ms3_delete(ms3, s3bucket, "test/read_cb.dat");
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  free(test_string);
+  free(dest);
+  ms3_free(data);
+  ms3_deinit(ms3);
+  ms3_library_deinit();
+  return 0;
+}
diff --git a/storage/maria/s3_func.c b/storage/maria/s3_func.c
index 3d18ba88..d85dc8a0 100644
--- a/storage/maria/s3_func.c
+++ b/storage/maria/s3_func.c
@@ -39,7 +39,7 @@ static int s3_read_file_from_disk(const char *filename, uchar **to,
 
 /* Used by ha_s3.cc and tools to define different protocol options */
 
-static const char *protocol_types[]= {"Auto", "Original", "Amazon", NullS};
+static const char *protocol_types[]= {"Auto", "Original", "Amazon", "Legacy", "Path", "Domain", NullS};
 TYPELIB s3_protocol_typelib= {array_elements(protocol_types)-1,"",
                               protocol_types, NULL};
 
@@ -154,9 +154,23 @@ ms3_st *s3_open_connection(S3_INFO *s3)
                     errno, ms3_error(errno));
     my_errno= HA_ERR_NO_SUCH_TABLE;
   }
-  if (s3->protocol_version)
+  if (s3->protocol_version > 2)
+  {
+    uint8_t protocol_version;
+    switch (s3->protocol_version)
+    {
+      case 3: /* Legacy means v1 */
+      case 4: /* Path means v1 */
+        protocol_version= 1;
+        break;
+      case 5: /* Domain means v2 */
+        protocol_version= 2;
+        break;
+    }
+
     ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION,
-                   &s3->protocol_version);
+                   &protocol_version);
+  }
   if (s3->port)
     ms3_set_option(s3_client, MS3_OPT_PORT_NUMBER, &s3->port);
 
diff --git a/storage/maria/unittest/ma_pagecache_consist.c b/storage/maria/unittest/ma_pagecache_consist.c
index ff4a2bcb..938e5b10 100644
--- a/storage/maria/unittest/ma_pagecache_consist.c
+++ b/storage/maria/unittest/ma_pagecache_consist.c
@@ -403,10 +403,6 @@ int main(int argc __attribute__((unused)),
     exit(1);
   }
 
-#ifdef HAVE_THR_SETCONCURRENCY
-  thr_setconcurrency(2);
-#endif
-
   if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0,
                              TEST_PAGE_SIZE, 0, 0)) == 0)
   {
diff --git a/storage/maria/unittest/ma_pagecache_rwconsist.c b/storage/maria/unittest/ma_pagecache_rwconsist.c
index 24c30245..e059ac45 100644
--- a/storage/maria/unittest/ma_pagecache_rwconsist.c
+++ b/storage/maria/unittest/ma_pagecache_rwconsist.c
@@ -272,10 +272,6 @@ int main(int argc __attribute__((unused)),
     exit(1);
   }
 
-#ifdef HAVE_THR_SETCONCURRENCY
-  thr_setconcurrency(2);
-#endif
-
   if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0,
                              TEST_PAGE_SIZE, 0, 0)) == 0)
   {
diff --git a/storage/maria/unittest/ma_pagecache_rwconsist2.c b/storage/maria/unittest/ma_pagecache_rwconsist2.c
index c92bec3c..28d3bb8d 100644
--- a/storage/maria/unittest/ma_pagecache_rwconsist2.c
+++ b/storage/maria/unittest/ma_pagecache_rwconsist2.c
@@ -268,10 +268,6 @@ int main(int argc __attribute__((unused)),
     exit(1);
   }
 
-#ifdef HAVE_THR_SETCONCURRENCY
-  thr_setconcurrency(2);
-#endif
-
   if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0,
                              TEST_PAGE_SIZE, 0, 0)) == 0)
   {
diff --git a/storage/maria/unittest/ma_pagecache_single.c b/storage/maria/unittest/ma_pagecache_single.c
index de2ecaec..9009e59c 100644
--- a/storage/maria/unittest/ma_pagecache_single.c
+++ b/storage/maria/unittest/ma_pagecache_single.c
@@ -795,10 +795,6 @@ int main(int argc __attribute__((unused)),
     exit(1);
   }
 
-#ifdef HAVE_THR_SETCONCURRENCY
-  thr_setconcurrency(2);
-#endif
-
   if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0,
                              TEST_PAGE_SIZE, 0, MYF(MY_WME))) == 0)
   {
diff --git a/storage/maria/unittest/ma_test_loghandler_multithread-t.c b/storage/maria/unittest/ma_test_loghandler_multithread-t.c
index ec097ede..be6046ab 100644
--- a/storage/maria/unittest/ma_test_loghandler_multithread-t.c
+++ b/storage/maria/unittest/ma_test_loghandler_multithread-t.c
@@ -331,10 +331,6 @@ int main(int argc __attribute__((unused)),
     exit(1);
   }
 
-#ifdef HAVE_THR_SETCONCURRENCY
-  thr_setconcurrency(2);
-#endif
-
   if (ma_control_file_open(TRUE, TRUE, TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);