diff options
Diffstat (limited to 'extra/mariabackup')
45 files changed, 22950 insertions, 0 deletions
diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt new file mode 100644 index 00000000..66293dac --- /dev/null +++ b/extra/mariabackup/CMakeLists.txt @@ -0,0 +1,113 @@ +# Copyright (c) 2013, 2017 Percona LLC and/or its affiliates. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + + +OPTION(WITH_MARIABACKUP "Include mariabackup" ON) +ADD_FEATURE_INFO(MARIABACKUP WITH_MARIABACKUP "MariaDB Backup Utility") + +IF(NOT WITH_MARIABACKUP) + RETURN() +ENDIF() + +IF(NOT WIN32) + CHECK_SYMBOL_EXISTS(regcomp regex.h HAVE_SYSTEM_REGEX) + IF(HAVE_SYSTEM_REGEX) + ADD_DEFINITIONS(-DHAVE_SYSTEM_REGEX) + ENDIF() +ENDIF() + +INCLUDE_DIRECTORIES( + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/sql + ${CMAKE_CURRENT_SOURCE_DIR}/quicklz + ${CMAKE_CURRENT_SOURCE_DIR} + ) + +IF(NOT HAVE_SYSTEM_REGEX) + INCLUDE_DIRECTORIES(${PCRE_INCLUDES}) + ADD_DEFINITIONS(${PCRE2_DEBIAN_HACK}) +ENDIF() + + +ADD_DEFINITIONS(-UMYSQL_SERVER) +######################################################################## +# xtrabackup binary +######################################################################## + +ADD_DEFINITIONS(-DPCRE_STATIC=1) +ADD_DEFINITIONS(${SSL_DEFINES}) + +IF(PMEM_FOUND) + ADD_COMPILE_FLAGS(xtrabackup.cc COMPILE_FLAGS "-DHAVE_PMEM") +ENDIF() + +MYSQL_ADD_EXECUTABLE(mariadb-backup + xtrabackup.cc + innobackupex.cc + changed_page_bitmap.cc + datasink.cc + ds_buffer.cc + ds_compress.cc + ds_local.cc + ds_stdout.cc + ds_tmpfile.cc + ds_xbstream.cc + fil_cur.cc + quicklz/quicklz.c + read_filt.cc + write_filt.cc + wsrep.cc + xbstream_write.cc + backup_mysql.cc + backup_copy.cc + xb_plugin.cc + ${PROJECT_BINARY_DIR}/sql/sql_builtin.cc + ${PROJECT_SOURCE_DIR}/sql/net_serv.cc + ${PROJECT_SOURCE_DIR}/libmysqld/libmysql.c + COMPONENT backup + ) + +# Export all symbols on Unix, for better crash callstacks +SET_TARGET_PROPERTIES(mariadb-backup PROPERTIES ENABLE_EXPORTS TRUE) + +TARGET_LINK_LIBRARIES(mariadb-backup sql sql_builtins) +IF(NOT HAVE_SYSTEM_REGEX) + TARGET_LINK_LIBRARIES(mariadb-backup pcre2-posix) +ENDIF() + + +######################################################################## +# mbstream binary +######################################################################## +MYSQL_ADD_EXECUTABLE(mbstream + ds_buffer.cc + ds_local.cc + ds_stdout.cc + datasink.cc + xbstream.cc + xbstream_read.cc + xbstream_write.cc + COMPONENT backup + ) + + +TARGET_LINK_LIBRARIES(mbstream + mysys +) +ADD_DEPENDENCIES(mbstream GenError) + +IF(MSVC) + SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj) +ENDIF() diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc new file mode 100644 index 00000000..dbf12ced --- /dev/null +++ b/extra/mariabackup/backup_copy.cc @@ -0,0 +1,2471 @@ +/****************************************************** +hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +(c) 2017 MariaDB +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 51 Franklin +Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <os0file.h> +#include <my_dir.h> +#include <ut0mem.h> +#include <srv0start.h> +#include <fil0fil.h> +#include <trx0sys.h> +#include <set> +#include <string> +#include <mysqld.h> +#include <sstream> +#include "fil_cur.h" +#include "xtrabackup.h" +#include "common.h" +#include "backup_copy.h" +#include "backup_debug.h" +#include "backup_mysql.h" +#include <btr0btr.h> +#ifdef _WIN32 +#include <direct.h> /* rmdir */ +#endif + +#ifdef _WIN32 +#include <aclapi.h> +#endif + + +#define ROCKSDB_BACKUP_DIR "#rocksdb" + +/* list of files to sync for --rsync mode */ +static std::set<std::string> rsync_list; +/* locations of tablespaces read from .isl files */ +static std::map<std::string, std::string> tablespace_locations; + +/* Whether LOCK BINLOG FOR BACKUP has been issued during backup */ +bool binlog_locked; + +static void rocksdb_create_checkpoint(); +static bool has_rocksdb_plugin(); +static void rocksdb_backup_checkpoint(ds_ctxt *ds_data); +static void rocksdb_copy_back(ds_ctxt *ds_data); + +static bool is_abs_path(const char *path) +{ +#ifdef _WIN32 + return path[0] && path[1] == ':' && (path[2] == '/' || path[2] == '\\'); +#else + return path[0] == '/'; +#endif +} + +/************************************************************************ +Struct represents file or directory. */ +struct datadir_node_t { + ulint dbpath_len; + char *filepath; + ulint filepath_len; + char *filepath_rel; + ulint filepath_rel_len; + bool is_empty_dir; + bool is_file; +}; + +/************************************************************************ +Holds the state needed to enumerate files in MySQL data directory. */ +struct datadir_iter_t { + char *datadir_path; + char *dbpath; + ulint dbpath_len; + char *filepath; + ulint filepath_len; + char *filepath_rel; + ulint filepath_rel_len; + pthread_mutex_t mutex; + os_file_dir_t dir; + os_file_dir_t dbdir; + os_file_stat_t dbinfo; + os_file_stat_t fileinfo; + dberr_t err; + bool is_empty_dir; + bool is_file; + bool skip_first_level; +}; + + +/************************************************************************ +Represents the context of the thread processing MySQL data directory. */ +struct datadir_thread_ctxt_t { + datadir_iter_t *it; + uint n_thread; + uint *count; + pthread_mutex_t* count_mutex; + bool ret; +}; + +static bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix); + +/************************************************************************ +Retirn true if character if file separator */ +bool +is_path_separator(char c) +{ + return(c == FN_LIBCHAR || c == FN_LIBCHAR2); +} + + +/************************************************************************ +Fill the node struct. Memory for node need to be allocated and freed by +the caller. It is caller responsibility to initialize node with +datadir_node_init and cleanup the memory with datadir_node_free. +Node can not be shared between threads. */ +static +void +datadir_node_fill(datadir_node_t *node, datadir_iter_t *it) +{ + if (node->filepath_len < it->filepath_len) { + free(node->filepath); + node->filepath = (char*)(malloc(it->filepath_len)); + node->filepath_len = it->filepath_len; + } + if (node->filepath_rel_len < it->filepath_rel_len) { + free(node->filepath_rel); + node->filepath_rel = (char*)(malloc(it->filepath_rel_len)); + node->filepath_rel_len = it->filepath_rel_len; + } + + strcpy(node->filepath, it->filepath); + strcpy(node->filepath_rel, it->filepath_rel); + node->is_empty_dir = it->is_empty_dir; + node->is_file = it->is_file; +} + +static +void +datadir_node_free(datadir_node_t *node) +{ + free(node->filepath); + free(node->filepath_rel); + memset(node, 0, sizeof(datadir_node_t)); +} + +static +void +datadir_node_init(datadir_node_t *node) +{ + memset(node, 0, sizeof(datadir_node_t)); +} + + +/************************************************************************ +Create the MySQL data directory iterator. Memory needs to be released +with datadir_iter_free. Position should be advanced with +datadir_iter_next_file. Iterator can be shared between multiple +threads. It is guaranteed that each thread receives unique file from +data directory into its local node struct. */ +static +datadir_iter_t * +datadir_iter_new(const char *path, bool skip_first_level = true) +{ + datadir_iter_t *it; + + it = static_cast<datadir_iter_t *>(malloc(sizeof(datadir_iter_t))); + if (!it) + goto error; + memset(it, 0, sizeof(datadir_iter_t)); + + pthread_mutex_init(&it->mutex, NULL); + it->datadir_path = strdup(path); + + it->dir = os_file_opendir(it->datadir_path); + + if (it->dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) { + + goto error; + } + + it->err = DB_SUCCESS; + + it->dbpath_len = FN_REFLEN; + it->dbpath = static_cast<char*>(malloc(it->dbpath_len)); + + it->filepath_len = FN_REFLEN; + it->filepath = static_cast<char*>(malloc(it->filepath_len)); + + it->filepath_rel_len = FN_REFLEN; + it->filepath_rel = static_cast<char*>(malloc(it->filepath_rel_len)); + + it->skip_first_level = skip_first_level; + + return(it); + +error: + free(it); + + return(NULL); +} + +static +bool +datadir_iter_next_database(datadir_iter_t *it) +{ + if (it->dbdir != NULL) { + if (os_file_closedir_failed(it->dbdir)) { + msg("Warning: could not" + " close database directory %s", it->dbpath); + it->err = DB_ERROR; + + } + it->dbdir = NULL; + } + + while (os_file_readdir_next_file(it->datadir_path, + it->dir, &it->dbinfo) == 0) { + ulint len; + + if ((it->dbinfo.type == OS_FILE_TYPE_FILE + && it->skip_first_level) + || it->dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + continue; + } + + /* We found a symlink or a directory; try opening it to see + if a symlink is a directory */ + + len = strlen(it->datadir_path) + + strlen (it->dbinfo.name) + 2; + if (len > it->dbpath_len) { + it->dbpath_len = len; + free(it->dbpath); + + it->dbpath = static_cast<char*>( + malloc(it->dbpath_len)); + } + snprintf(it->dbpath, it->dbpath_len, "%s/%s", + it->datadir_path, it->dbinfo.name); + + if (it->dbinfo.type == OS_FILE_TYPE_FILE) { + it->is_file = true; + return(true); + } + + if (check_if_skip_database_by_path(it->dbpath)) { + msg("Skipping db: %s", it->dbpath); + continue; + } + + /* We want wrong directory permissions to be a fatal error for + XtraBackup. */ + it->dbdir = os_file_opendir(it->dbpath); + + if (it->dir != IF_WIN(INVALID_HANDLE_VALUE, nullptr)) { + it->is_file = false; + return(true); + } + + } + + return(false); +} + +/************************************************************************ +Concatenate n parts into single path */ +static +void +make_path_n(int n, char **path, ulint *path_len, ...) +{ + ulint len_needed = n + 1; + char *p; + int i; + va_list vl; + + ut_ad(n > 0); + + va_start(vl, path_len); + for (i = 0; i < n; i++) { + p = va_arg(vl, char*); + len_needed += strlen(p); + } + va_end(vl); + + if (len_needed < *path_len) { + free(*path); + *path = static_cast<char*>(malloc(len_needed)); + } + + va_start(vl, path_len); + p = va_arg(vl, char*); + strcpy(*path, p); + for (i = 1; i < n; i++) { + size_t plen; + p = va_arg(vl, char*); + plen = strlen(*path); + if (!is_path_separator((*path)[plen - 1])) { + (*path)[plen] = FN_LIBCHAR; + (*path)[plen + 1] = 0; + } + strcat(*path + plen, p); + } + va_end(vl); +} + +static +bool +datadir_iter_next_file(datadir_iter_t *it) +{ + if (it->is_file && it->dbpath) { + make_path_n(2, &it->filepath, &it->filepath_len, + it->datadir_path, it->dbinfo.name); + + make_path_n(1, &it->filepath_rel, &it->filepath_rel_len, + it->dbinfo.name); + + it->is_empty_dir = false; + it->is_file = false; + + return(true); + } + + if (!it->dbpath || !it->dbdir) { + + return(false); + } + + while (os_file_readdir_next_file(it->dbpath, it->dbdir, + &it->fileinfo) == 0) { + + if (it->fileinfo.type == OS_FILE_TYPE_DIR) { + + continue; + } + + /* We found a symlink or a file */ + make_path_n(3, &it->filepath, &it->filepath_len, + it->datadir_path, it->dbinfo.name, + it->fileinfo.name); + + make_path_n(2, &it->filepath_rel, &it->filepath_rel_len, + it->dbinfo.name, it->fileinfo.name); + + it->is_empty_dir = false; + + return(true); + } + + return(false); +} + +static +bool +datadir_iter_next(datadir_iter_t *it, datadir_node_t *node) +{ + bool ret = true; + + pthread_mutex_lock(&it->mutex); + + if (datadir_iter_next_file(it)) { + + datadir_node_fill(node, it); + + goto done; + } + + while (datadir_iter_next_database(it)) { + + if (datadir_iter_next_file(it)) { + + datadir_node_fill(node, it); + + goto done; + } + + make_path_n(2, &it->filepath, &it->filepath_len, + it->datadir_path, it->dbinfo.name); + + make_path_n(1, &it->filepath_rel, &it->filepath_rel_len, + it->dbinfo.name); + + it->is_empty_dir = true; + + datadir_node_fill(node, it); + + goto done; + } + + /* nothing found */ + ret = false; + +done: + pthread_mutex_unlock(&it->mutex); + + return(ret); +} + +/************************************************************************ +Interface to read MySQL data file sequentially. One should open file +with datafile_open to get cursor and close the cursor with +datafile_close. Cursor can not be shared between multiple +threads. */ +static +void +datadir_iter_free(datadir_iter_t *it) +{ + pthread_mutex_destroy(&it->mutex); + + if (it->dbdir) { + + os_file_closedir(it->dbdir); + } + + if (it->dir) { + + os_file_closedir(it->dir); + } + + free(it->dbpath); + free(it->filepath); + free(it->filepath_rel); + free(it->datadir_path); + free(it); +} + + +/************************************************************************ +Holds the state needed to copy single data file. */ +struct datafile_cur_t { + pfs_os_file_t file; + char rel_path[FN_REFLEN]; + char abs_path[FN_REFLEN]; + MY_STAT statinfo; + uint thread_n; + byte* buf; + size_t buf_size; + size_t buf_read; + size_t buf_offset; + + explicit datafile_cur_t(const char* filename = NULL) : + file(), thread_n(0), buf(NULL), buf_size(0), + buf_read(0), buf_offset(0) + { + memset(rel_path, 0, sizeof rel_path); + if (filename) { + strncpy(abs_path, filename, sizeof abs_path - 1); + abs_path[(sizeof abs_path) - 1] = 0; + } else { + abs_path[0] = '\0'; + } + rel_path[0] = '\0'; + memset(&statinfo, 0, sizeof statinfo); + } +}; + +static +void +datafile_close(datafile_cur_t *cursor) +{ + if (cursor->file != OS_FILE_CLOSED) { + os_file_close(cursor->file); + } + free(cursor->buf); +} + +static +bool +datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) +{ + bool success; + + new (cursor) datafile_cur_t(file); + + /* Get the relative path for the destination tablespace name, i.e. the + one that can be appended to the backup root directory. Non-system + tablespaces may have absolute paths for remote tablespaces in MySQL + 5.6+. We want to make "local" copies for the backup. */ + strncpy(cursor->rel_path, + xb_get_relative_path(cursor->abs_path, FALSE), + (sizeof cursor->rel_path) - 1); + cursor->rel_path[(sizeof cursor->rel_path) - 1] = '\0'; + + cursor->file = os_file_create_simple_no_error_handling( + 0, cursor->abs_path, + OS_FILE_OPEN, OS_FILE_READ_ALLOW_DELETE, true, &success); + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + + msg(thread_n,"error: cannot open " + "file %s", cursor->abs_path); + + return(false); + } + + if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) { + msg(thread_n, "error: cannot stat %s", cursor->abs_path); + datafile_close(cursor); + return(false); + } + + posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); + + cursor->buf_size = 10 * 1024 * 1024; + cursor->buf = static_cast<byte *>(malloc((ulint)cursor->buf_size)); + + return(true); +} + + +static +xb_fil_cur_result_t +datafile_read(datafile_cur_t *cursor) +{ + ulint to_read; + + xtrabackup_io_throttling(); + + to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset, + cursor->buf_size); + + if (to_read == 0) { + return(XB_FIL_CUR_EOF); + } + + if (os_file_read(IORequestRead, + cursor->file, cursor->buf, cursor->buf_offset, + to_read, nullptr) != DB_SUCCESS) { + return(XB_FIL_CUR_ERROR); + } + + posix_fadvise(cursor->file, cursor->buf_offset, to_read, + POSIX_FADV_DONTNEED); + + cursor->buf_read = to_read; + cursor->buf_offset += to_read; + + return(XB_FIL_CUR_SUCCESS); +} + + + +/************************************************************************ +Check to see if a file exists. +Takes name of the file to check. +@return true if file exists. */ +static +bool +file_exists(const char *filename) +{ + MY_STAT stat_arg; + + if (!my_stat(filename, &stat_arg, MYF(0))) { + + return(false); + } + + return(true); +} + +/************************************************************************ +Trim leading slashes from absolute path so it becomes relative */ +static +const char * +trim_dotslash(const char *path) +{ + while (*path) { + if (is_path_separator(*path)) { + ++path; + continue; + } + if (*path == '.' && is_path_separator(path[1])) { + path += 2; + continue; + } + break; + } + + return(path); +} + + + +/************************************************************************ +Check if string ends with given suffix. +@return true if string ends with given suffix. */ +bool +ends_with(const char *str, const char *suffix) +{ + size_t suffix_len = strlen(suffix); + size_t str_len = strlen(str); + return(str_len >= suffix_len + && strcmp(str + str_len - suffix_len, suffix) == 0); +} + +static bool starts_with(const char *str, const char *prefix) +{ + return strncmp(str, prefix, strlen(prefix)) == 0; +} + +/************************************************************************ +Create directories recursively. +@return 0 if directories created successfully. */ +static +int +mkdirp(const char *pathname, int Flags, myf MyFlags) +{ + char *parent, *p; + + /* make a parent directory path */ + if (!(parent= strdup(pathname))) + return(-1); + + for (p = parent + strlen(parent); + !is_path_separator(*p) && p != parent; p--) ; + + *p = 0; + + /* try to make parent directory */ + if (p != parent && mkdirp(parent, Flags, MyFlags) != 0) { + free(parent); + return(-1); + } + + /* make this one if parent has been made */ + if (my_mkdir(pathname, Flags, MyFlags) == 0) { + free(parent); + return(0); + } + + /* if it already exists that is fine */ + if (errno == EEXIST) { + free(parent); + return(0); + } + + free(parent); + return(-1); +} + +/************************************************************************ +Return true if first and second arguments are the same path. */ +bool +equal_paths(const char *first, const char *second) +{ +#ifdef HAVE_REALPATH + char *real_first, *real_second; + int result; + + real_first = realpath(first, 0); + if (real_first == NULL) { + return false; + } + + real_second = realpath(second, 0); + if (real_second == NULL) { + free(real_first); + return false; + } + + result = strcmp(real_first, real_second); + free(real_first); + free(real_second); + return result == 0; +#else + return strcmp(first, second) == 0; +#endif +} + +/************************************************************************ +Check if directory exists. Optionally create directory if doesn't +exist. +@return true if directory exists and if it was created successfully. */ +bool +directory_exists(const char *dir, bool create) +{ + os_file_dir_t os_dir; + MY_STAT stat_arg; + char errbuf[MYSYS_STRERROR_SIZE]; + + if (my_stat(dir, &stat_arg, MYF(0)) == NULL) { + + if (!create) { + return(false); + } + + if (mkdirp(dir, 0777, MYF(0)) < 0) { + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not create directory %s: %s", dir, errbuf); + return(false); + } + } + + /* could be symlink */ + os_dir = os_file_opendir(dir); + + if (os_dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) { + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not open directory %s: %s", dir, + errbuf); + + return(false); + } + + os_file_closedir(os_dir); + + return(true); +} + +/************************************************************************ +Check that directory exists and it is empty. */ +static +bool +directory_exists_and_empty(const char *dir, const char *comment) +{ + os_file_dir_t os_dir; + dberr_t err; + os_file_stat_t info; + bool empty; + + if (!directory_exists(dir, true)) { + return(false); + } + + os_dir = os_file_opendir(dir); + + if (os_dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) { + msg("%s can not open directory %s", comment, dir); + return(false); + } + + empty = (fil_file_readdir_next_file(&err, dir, os_dir, &info) != 0); + + os_file_closedir(os_dir); + + if (!empty) { + msg("%s directory %s is not empty!", comment, dir); + } + + return(empty); +} + + +/************************************************************************ +Check if file name ends with given set of suffixes. +@return true if it does. */ +static +bool +filename_matches(const char *filename, const char **ext_list) +{ + const char **ext; + + for (ext = ext_list; *ext; ext++) { + if (ends_with(filename, *ext)) { + return(true); + } + } + + return(false); +} + + +/************************************************************************ +Copy data file for backup. Also check if it is allowed to copy by +comparing its name to the list of known data file types and checking +if passes the rules for partial backup. +@return true if file backed up or skipped successfully. */ +static +bool +datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n) +{ + const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", + "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", + NULL}; + + /* Get the name and the path for the tablespace. node->name always + contains the path (which may be absolute for remote tablespaces in + 5.6+). space->name contains the tablespace name in the form + "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a + multi-node shared tablespace, space->name contains the name of the first + node, but that's irrelevant, since we only need node_name to match them + against filters, and the shared tablespace is always copied regardless + of the filters value. */ + + if (check_if_skip_table(filepath)) { + msg(thread_n,"Skipping %s.", filepath); + return(true); + } + + if (filename_matches(filepath, ext_list)) { + return ds_data->copy_file(filepath, filepath, thread_n); + } + + return(true); +} + + +/************************************************************************ +Same as datafile_copy_backup, but put file name into the list for +rsync command. */ +static +bool +datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f) +{ + const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", + "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", + NULL}; + + /* Get the name and the path for the tablespace. node->name always + contains the path (which may be absolute for remote tablespaces in + 5.6+). space->name contains the tablespace name in the form + "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a + multi-node shared tablespace, space->name contains the name of the first + node, but that's irrelevant, since we only need node_name to match them + against filters, and the shared tablespace is always copied regardless + of the filters value. */ + + if (check_if_skip_table(filepath)) { + return(true); + } + + if (filename_matches(filepath, ext_list)) { + fprintf(f, "%s\n", filepath); + if (save_to_list) { + rsync_list.insert(filepath); + } + } + + return(true); +} + +bool ds_ctxt_t::backup_file_print_buf(const char *filename, + const char *buf, int buf_len) +{ + ds_file_t *dstfile = NULL; + MY_STAT stat; /* unused for now */ + const char *action; + + memset(&stat, 0, sizeof(stat)); + + stat.st_size = buf_len; + stat.st_mtime = my_time(0); + + dstfile = ds_open(this, filename, &stat); + if (dstfile == NULL) { + msg("error: Can't open the destination stream for %s", + filename); + goto error; + } + + action = xb_get_copy_action("Writing"); + msg("%s %s", action, filename); + + if (buf_len == -1) { + goto error; + } + + if (ds_write(dstfile, buf, buf_len)) { + goto error; + } + + /* close */ + msg(" ...done"); + + if (ds_close(dstfile)) { + goto error_close; + } + + return(true); + +error: + if (dstfile != NULL) { + ds_close(dstfile); + } + +error_close: + msg("Error: backup file failed."); + return(false); /*ERROR*/ + + return true; +}; + +bool +ds_ctxt_t::backup_file_vprintf(const char *filename, + const char *fmt, va_list ap) +{ + char *buf = 0; + int buf_len; + buf_len = vasprintf(&buf, fmt, ap); + bool result = backup_file_print_buf(filename, buf, buf_len); + free(buf); + return result; +} + +bool +ds_ctxt_t::backup_file_printf(const char *filename, const char *fmt, ...) +{ + bool result; + va_list ap; + + va_start(ap, fmt); + + result = backup_file_vprintf(filename, fmt, ap); + + va_end(ap); + + return(result); +} + +static +bool +run_data_threads(datadir_iter_t *it, void (*func)(datadir_thread_ctxt_t *ctxt), uint n) +{ + datadir_thread_ctxt_t *data_threads; + uint i, count; + pthread_mutex_t count_mutex; + bool ret; + + data_threads = (datadir_thread_ctxt_t*) + malloc(sizeof(datadir_thread_ctxt_t) * n); + + pthread_mutex_init(&count_mutex, NULL); + count = n; + + for (i = 0; i < n; i++) { + data_threads[i].it = it; + data_threads[i].n_thread = i + 1; + data_threads[i].count = &count; + data_threads[i].count_mutex = &count_mutex; + std::thread(func, data_threads + i).detach(); + } + + /* Wait for threads to exit */ + while (1) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + pthread_mutex_lock(&count_mutex); + if (count == 0) { + pthread_mutex_unlock(&count_mutex); + break; + } + pthread_mutex_unlock(&count_mutex); + } + + pthread_mutex_destroy(&count_mutex); + + ret = true; + for (i = 0; i < n; i++) { + ret = data_threads[i].ret && ret; + if (!data_threads[i].ret) { + msg("Error: thread %u failed.", i); + } + } + + free(data_threads); + + return(ret); +} + + +/************************************************************************ +Copy file for backup/restore. +@return true in case of success. */ +bool +ds_ctxt_t::copy_file(const char *src_file_path, + const char *dst_file_path, + uint thread_n) +{ + char dst_name[FN_REFLEN]; + ds_file_t *dstfile = NULL; + datafile_cur_t cursor; + xb_fil_cur_result_t res; + DBUG_ASSERT(datasink->remove); + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back)? + dst_file_path : trim_dotslash(dst_file_path); + + if (!datafile_open(src_file_path, &cursor, thread_n)) { + goto error_close; + } + + strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); + + dstfile = ds_open(this, dst_path, &cursor.statinfo); + if (dstfile == NULL) { + msg(thread_n,"error: " + "cannot open the destination stream for %s", dst_name); + goto error; + } + + msg(thread_n, "%s %s to %s", xb_get_copy_action(), src_file_path, dstfile->path); + + /* The main copy loop */ + while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) { + + if (ds_write(dstfile, cursor.buf, cursor.buf_read)) { + goto error; + } + DBUG_EXECUTE_IF("copy_file_error", errno=ENOSPC;goto error;); + } + + if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + /* close */ + msg(thread_n," ...done"); + datafile_close(&cursor); + if (ds_close(dstfile)) { + goto error_close; + } + return(true); + +error: + datafile_close(&cursor); + if (dstfile != NULL) { + datasink->remove(dstfile->path); + ds_close(dstfile); + } + +error_close: + msg(thread_n,"Error: copy_file() failed."); + return(false); /*ERROR*/ +} + + +/************************************************************************ +Try to move file by renaming it. If source and destination are on +different devices fall back to copy and unlink. +@return true in case of success. */ +bool +ds_ctxt_t::move_file(const char *src_file_path, + const char *dst_file_path, + const char *dst_dir, uint thread_n) +{ + char errbuf[MYSYS_STRERROR_SIZE]; + char dst_file_path_abs[FN_REFLEN]; + char dst_dir_abs[FN_REFLEN]; + size_t dirname_length; + + snprintf(dst_file_path_abs, sizeof(dst_file_path_abs), + "%s/%s", dst_dir, dst_file_path); + + dirname_part(dst_dir_abs, dst_file_path_abs, &dirname_length); + + if (!directory_exists(dst_dir_abs, true)) { + return(false); + } + + if (file_exists(dst_file_path_abs)) { + msg("Error: Move file %s to %s failed: Destination " + "file exists", src_file_path, dst_file_path_abs); + return(false); + } + + msg(thread_n,"Moving %s to %s", src_file_path, dst_file_path_abs); + + if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) { + if (my_errno == EXDEV) { + /* Fallback to copy/unlink */ + if(!copy_file(src_file_path, + dst_file_path, thread_n)) + return false; + msg(thread_n,"Removing %s", src_file_path); + if (unlink(src_file_path) != 0) { + my_strerror(errbuf, sizeof(errbuf), errno); + msg("Warning: unlink %s failed: %s", + src_file_path, + errbuf); + } + return true; + } + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not move file %s to %s: %s", + src_file_path, dst_file_path_abs, + errbuf); + return(false); + } + msg(thread_n," ...done"); + + return(true); +} + + +/************************************************************************ +Read link from .isl file if any and store it in the global map associated +with given tablespace. */ +static +void +read_link_file(const char *ibd_filepath, const char *link_filepath) +{ + char *filepath= NULL; + + FILE *file = fopen(link_filepath, "r+b"); + if (file) { + filepath = static_cast<char*>(malloc(OS_FILE_MAX_PATH)); + + os_file_read_string(file, filepath, OS_FILE_MAX_PATH); + fclose(file); + + if (size_t len = strlen(filepath)) { + /* Trim whitespace from end of filepath */ + ulint lastch = len - 1; + while (lastch > 4 && filepath[lastch] <= 0x20) { + filepath[lastch--] = 0x00; + } + } + + tablespace_locations[ibd_filepath] = filepath; + } + free(filepath); +} + + +/************************************************************************ +Return the location of given .ibd if it was previously read +from .isl file. +@return NULL or destination .ibd file path. */ +static +const char * +tablespace_filepath(const char *ibd_filepath) +{ + std::map<std::string, std::string>::iterator it; + + it = tablespace_locations.find(ibd_filepath); + + if (it != tablespace_locations.end()) { + return it->second.c_str(); + } + + return NULL; +} + + +/************************************************************************ +Copy or move file depending on current mode. +@return true in case of success. */ +static +bool +copy_or_move_file(ds_ctxt *datasink0, const char *src_file_path, + const char *dst_file_path, + const char *dst_dir, + uint thread_n, + bool copy = xtrabackup_copy_back) +{ + ds_ctxt_t *datasink = datasink0; /* copy to datadir by default */ + char filedir[FN_REFLEN]; + size_t filedir_len; + bool ret; + + /* read the link from .isl file */ + if (ends_with(src_file_path, ".isl")) { + char *ibd_filepath; + + ibd_filepath = strdup(src_file_path); + strcpy(ibd_filepath + strlen(ibd_filepath) - 3, "ibd"); + + read_link_file(ibd_filepath, src_file_path); + + free(ibd_filepath); + } + + /* check if there is .isl file */ + if (ends_with(src_file_path, ".ibd")) { + char *link_filepath; + const char *filepath; + + link_filepath = strdup(src_file_path); + strcpy(link_filepath + strlen(link_filepath) - 3, "isl"); + + read_link_file(src_file_path, link_filepath); + + filepath = tablespace_filepath(src_file_path); + + if (filepath != NULL) { + dirname_part(filedir, filepath, &filedir_len); + + dst_file_path = filepath + filedir_len; + dst_dir = filedir; + + if (!directory_exists(dst_dir, true)) { + ret = false; + free(link_filepath); + goto cleanup; + } + + datasink = ds_create(dst_dir, DS_TYPE_LOCAL); + } + + free(link_filepath); + } + + ret = (copy ? + datasink->copy_file(src_file_path, dst_file_path, thread_n) : + datasink->move_file(src_file_path, dst_file_path, + dst_dir, thread_n)); + +cleanup: + + if (datasink != datasink0) { + ds_destroy(datasink); + } + + return(ret); +} + + + + +static +bool +backup_files(ds_ctxt *ds_data, const char *from, bool prep_mode) +{ + char rsync_tmpfile_name[FN_REFLEN]; + FILE *rsync_tmpfile = NULL; + datadir_iter_t *it; + datadir_node_t node; + bool ret = true; + + if (prep_mode && !opt_rsync) { + return(true); + } + + if (opt_rsync) { + snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), + "%s/%s%d", opt_mysql_tmpdir, + "xtrabackup_rsyncfiles_pass", + prep_mode ? 1 : 2); + rsync_tmpfile = fopen(rsync_tmpfile_name, "w"); + if (rsync_tmpfile == NULL) { + msg("Error: can't create file %s", + rsync_tmpfile_name); + return(false); + } + } + + msg("Starting %s non-InnoDB tables and files", + prep_mode ? "prep copy of" : "to backup"); + + datadir_node_init(&node); + it = datadir_iter_new(from); + + while (datadir_iter_next(it, &node)) { + + if (!node.is_empty_dir) { + if (opt_rsync) { + ret = datafile_rsync_backup(node.filepath, + !prep_mode, rsync_tmpfile); + } else { + ret = datafile_copy_backup(ds_data, node.filepath, 1); + } + if (!ret) { + msg("Failed to copy file %s", node.filepath); + goto out; + } + } else if (!prep_mode) { + /* backup fake file into empty directory */ + char path[FN_REFLEN]; + snprintf(path, sizeof(path), + "%s/db.opt", node.filepath); + if (!(ret = ds_data->backup_file_printf( + trim_dotslash(path), "%s", ""))) { + msg("Failed to create file %s", path); + goto out; + } + } + } + + if (opt_rsync) { + std::stringstream cmd; + int err; + + if (buffer_pool_filename && file_exists(buffer_pool_filename)) { + fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename); + rsync_list.insert(buffer_pool_filename); + } + if (file_exists("ib_lru_dump")) { + fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump"); + rsync_list.insert("ib_lru_dump"); + } + + fclose(rsync_tmpfile); + rsync_tmpfile = NULL; + + cmd << "rsync -t . --files-from=" << rsync_tmpfile_name + << " " << xtrabackup_target_dir; + + msg("Starting rsync as: %s", cmd.str().c_str()); + if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) { + msg("Error: rsync failed with error code %d", err); + ret = false; + goto out; + } + msg("rsync finished successfully."); + + if (!prep_mode && !opt_no_lock) { + char path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + char *newline; + + /* Remove files that have been removed between first and + second passes. Cannot use "rsync --delete" because it + does not work with --files-from. */ + snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), + "%s/%s", opt_mysql_tmpdir, + "xtrabackup_rsyncfiles_pass1"); + + rsync_tmpfile = fopen(rsync_tmpfile_name, "r"); + if (rsync_tmpfile == NULL) { + msg("Error: can't open file %s", + rsync_tmpfile_name); + ret = false; + goto out; + } + + while (fgets(path, sizeof(path), rsync_tmpfile)) { + + newline = strchr(path, '\n'); + if (newline) { + *newline = 0; + } + if (rsync_list.count(path) < 1) { + snprintf(dst_path, sizeof(dst_path), + "%s/%s", xtrabackup_target_dir, + path); + msg("Removing %s", dst_path); + unlink(dst_path); + } + } + + fclose(rsync_tmpfile); + rsync_tmpfile = NULL; + } + } + + msg("Finished %s non-InnoDB tables and files", + prep_mode ? "a prep copy of" : "backing up"); + +out: + datadir_iter_free(it); + datadir_node_free(&node); + + if (rsync_tmpfile != NULL) { + fclose(rsync_tmpfile); + } + + return(ret); +} + + +lsn_t get_current_lsn(MYSQL *connection) +{ + static const char lsn_prefix[] = "\nLog sequence number "; + lsn_t lsn = 0; + if (MYSQL_RES *res = xb_mysql_query(connection, + "SHOW ENGINE INNODB STATUS", + true, false)) { + if (MYSQL_ROW row = mysql_fetch_row(res)) { + const char *p= strstr(row[2], lsn_prefix); + DBUG_ASSERT(p); + if (p) { + p += sizeof lsn_prefix - 1; + lsn = lsn_t(strtoll(p, NULL, 10)); + } + } + mysql_free_result(res); + } + return lsn; +} + +lsn_t server_lsn_after_lock; +extern void backup_wait_for_lsn(lsn_t lsn); +/** Start --backup */ +bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, + CorruptedPages &corrupted_pages) +{ + if (!opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + + if (!backup_files(ds_data, fil_path_to_mysql_datadir, true)) { + return(false); + } + + history_lock_time = time(NULL); + + if (!lock_tables(mysql_connection)) { + return(false); + } + server_lsn_after_lock = get_current_lsn(mysql_connection); + } + + if (!backup_files(ds_data, fil_path_to_mysql_datadir, false)) { + return(false); + } + + if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir, + "aws-kms-key") || + !backup_files_from_datadir(ds_data, + aria_log_dir_path, + "aria_log")) { + return false; + } + + if (has_rocksdb_plugin()) { + rocksdb_create_checkpoint(); + } + + msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock); + backup_wait_for_lsn(server_lsn_after_lock); + DBUG_EXECUTE_FOR_KEY("sleep_after_waiting_for_lsn", {}, + { + ulong milliseconds = strtoul(dbug_val, NULL, 10); + msg("sleep_after_waiting_for_lsn"); + my_sleep(milliseconds*1000UL); + }); + + corrupted_pages.backup_fix_ddl(ds_data, ds_meta); + + // There is no need to stop slave thread before coping non-Innodb data when + // --no-lock option is used because --no-lock option requires that no DDL or + // DML to non-transaction tables can occur. + if (opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + } + + if (opt_slave_info) { + lock_binlog_maybe(mysql_connection); + + if (!write_slave_info(ds_data, mysql_connection)) { + return(false); + } + } + + /* The only reason why Galera/binlog info is written before + wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup + binary will start streamig a temporary copy of REDO log to stdout and + thus, any streaming from innobackupex would interfere. The only way to + avoid that is to have a single process, i.e. merge innobackupex and + xtrabackup. */ + if (opt_galera_info) { + if (!write_galera_info(ds_data, mysql_connection)) { + return(false); + } + } + + if (opt_binlog_info == BINLOG_INFO_ON) { + + lock_binlog_maybe(mysql_connection); + write_binlog_info(ds_data, mysql_connection); + } + + if (!opt_no_lock) { + msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS..."); + xb_mysql_query(mysql_connection, + "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false); + } + + return(true); +} + +/** Release resources after backup_start() */ +void backup_release() +{ + /* release all locks */ + if (!opt_no_lock) { + unlock_all(mysql_connection); + history_lock_time = 0; + } else { + history_lock_time = time(NULL) - history_lock_time; + } + + if (opt_lock_ddl_per_table) { + mdl_unlock_all(); + } + + if (opt_safe_slave_backup && sql_thread_started) { + msg("Starting slave SQL thread"); + xb_mysql_query(mysql_connection, + "START SLAVE SQL_THREAD", false); + } +} + +static const char *default_buffer_pool_file = "ib_buffer_pool"; + +/** Finish after backup_start() and backup_release() */ +bool backup_finish(ds_ctxt *ds_data) +{ + /* Copy buffer pool dump or LRU dump */ + if (!opt_rsync && opt_galera_info) { + if (buffer_pool_filename && file_exists(buffer_pool_filename)) { + ds_data->copy_file(buffer_pool_filename, default_buffer_pool_file, 0); + } + if (file_exists("ib_lru_dump")) { + ds_data->copy_file("ib_lru_dump", "ib_lru_dump", 0); + } + } + + if (has_rocksdb_plugin()) { + rocksdb_backup_checkpoint(ds_data); + } + + msg("Backup created in directory '%s'", xtrabackup_target_dir); + if (mysql_binlog_position != NULL) { + msg("MySQL binlog position: %s", mysql_binlog_position); + } + if (mysql_slave_position && opt_slave_info) { + msg("MySQL slave binlog position: %s", + mysql_slave_position); + } + + if (!write_backup_config_file(ds_data)) { + return(false); + } + + if (!write_xtrabackup_info(ds_data, mysql_connection, XTRABACKUP_INFO, + opt_history != 0, true)) { + return(false); + } + + return(true); +} + + +/* + Drop all empty database directories in the base backup + that do not exists in the icremental backup. + + This effectively re-plays all DROP DATABASE statements happened + in between base backup and incremental backup creation time. + + Note, only checking if base_dir/db/ is empty is not enough, + because inc_dir/db/db.opt might have been dropped for some reasons, + which may also result into empty base_dir/db/. + + Only the fact that at the same time: + - base_dir/db/ exists + - inc_dir/db/ does not exist + means that DROP DATABASE happened. +*/ +static void +ibx_incremental_drop_databases(const char *base_dir, + const char *inc_dir) +{ + datadir_node_t node; + datadir_node_init(&node); + datadir_iter_t *it = datadir_iter_new(base_dir); + + while (datadir_iter_next(it, &node)) { + if (node.is_empty_dir) { + char path[FN_REFLEN]; + snprintf(path, sizeof(path), "%s/%s", + inc_dir, node.filepath_rel); + if (!directory_exists(path, false)) { + msg("Removing %s", node.filepath); + rmdir(node.filepath); + } + } + + } + datadir_iter_free(it); + datadir_node_free(&node); +} + + +static bool +ibx_copy_incremental_over_full() +{ + const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", + "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", + NULL}; + const char *sup_files[] = {"xtrabackup_binlog_info", + "xtrabackup_galera_info", + "xtrabackup_slave_info", + "xtrabackup_info", + "ib_lru_dump", + NULL}; + datadir_iter_t *it = NULL; + datadir_node_t node; + bool ret = true; + char path[FN_REFLEN]; + int i; + ds_ctxt *ds_data= NULL; + + DBUG_ASSERT(!opt_galera_info); + datadir_node_init(&node); + + /* If we were applying an incremental change set, we need to make + sure non-InnoDB files and xtrabackup_* metainfo files are copied + to the full backup directory. */ + + if (xtrabackup_incremental) { + + ds_data = ds_create(xtrabackup_target_dir, DS_TYPE_LOCAL); + + it = datadir_iter_new(xtrabackup_incremental_dir); + + while (datadir_iter_next(it, &node)) { + + /* copy only non-innodb files */ + + if (node.is_empty_dir + || !filename_matches(node.filepath, ext_list)) { + continue; + } + + if (file_exists(node.filepath_rel)) { + unlink(node.filepath_rel); + } + + if (!(ret = ds_data->copy_file(node.filepath, + node.filepath_rel, 1))) { + msg("Failed to copy file %s", + node.filepath); + goto cleanup; + } + } + + if (!(ret = backup_files_from_datadir(ds_data, + xtrabackup_incremental_dir, + "aws-kms-key")) || + !(ret = backup_files_from_datadir(ds_data, + xtrabackup_incremental_dir, + "aria_log"))) + goto cleanup; + + /* copy supplementary files */ + + for (i = 0; sup_files[i]; i++) { + snprintf(path, sizeof(path), "%s/%s", + xtrabackup_incremental_dir, + sup_files[i]); + + if (file_exists(path)) + { + if (file_exists(sup_files[i])) { + unlink(sup_files[i]); + } + ds_data->copy_file(path, sup_files[i], 0); + } + } + + if (directory_exists(ROCKSDB_BACKUP_DIR, false)) { + if (my_rmtree(ROCKSDB_BACKUP_DIR, MYF(0))) { + die("Can't remove " ROCKSDB_BACKUP_DIR); + } + } + snprintf(path, sizeof(path), "%s/" ROCKSDB_BACKUP_DIR, xtrabackup_incremental_dir); + if (directory_exists(path, false)) { + if (my_mkdir(ROCKSDB_BACKUP_DIR, 0777, MYF(0))) { + die("my_mkdir failed for " ROCKSDB_BACKUP_DIR); + } + ds_data->copy_or_move_dir(path, ROCKSDB_BACKUP_DIR, true, true); + } + ibx_incremental_drop_databases(xtrabackup_target_dir, + xtrabackup_incremental_dir); + } + + +cleanup: + if (it != NULL) { + datadir_iter_free(it); + } + + if (ds_data != NULL) { + ds_destroy(ds_data); + } + + datadir_node_free(&node); + + return(ret); +} + +bool +ibx_cleanup_full_backup() +{ + const char *ext_list[] = {"delta", "meta", "ibd", NULL}; + datadir_iter_t *it = NULL; + datadir_node_t node; + bool ret = true; + + datadir_node_init(&node); + + /* If we are applying an incremental change set, we need to make + sure non-InnoDB files are cleaned up from full backup dir before + we copy files from incremental dir. */ + + it = datadir_iter_new(xtrabackup_target_dir); + + while (datadir_iter_next(it, &node)) { + + if (node.is_empty_dir) { +#ifdef _WIN32 + DeleteFile(node.filepath); +#else + rmdir(node.filepath); +#endif + } + + if (xtrabackup_incremental && !node.is_empty_dir + && !filename_matches(node.filepath, ext_list)) { + unlink(node.filepath); + } + } + + datadir_iter_free(it); + + datadir_node_free(&node); + + return(ret); +} + +bool +apply_log_finish() +{ + if (!ibx_cleanup_full_backup() + || !ibx_copy_incremental_over_full()) { + return(false); + } + + return(true); +} + +class Copy_back_dst_dir +{ + std::string buf; + +public: + const char *make(const char *path) + { + if (!path || !path[0]) + return mysql_data_home; + if (is_absolute_path(path)) + return path; + return buf.assign(mysql_data_home).append(path).c_str(); + } +}; + + +static inline bool +is_aria_log_dir_file(const datadir_node_t &node) +{ + return starts_with(node.filepath_rel, "aria_log"); +} + + +bool +copy_back_aria_logs(const char *dstdir) +{ + std::unique_ptr<ds_ctxt_t, void (&)(ds_ctxt_t*)> + ds_ctxt_aria_log_dir_path(ds_create(dstdir, DS_TYPE_LOCAL), ds_destroy); + + datadir_node_t node; + datadir_node_init(&node); + datadir_iter_t *it = datadir_iter_new(".", false); + + while (datadir_iter_next(it, &node)) + { + if (!is_aria_log_dir_file(node)) + continue; + if (!copy_or_move_file(ds_ctxt_aria_log_dir_path.get(), + node.filepath, node.filepath_rel, + dstdir, 1)) + return false; + } + datadir_node_free(&node); + datadir_iter_free(it); + return true; +} + + +bool +copy_back() +{ + bool ret = false; + datadir_iter_t *it = NULL; + datadir_node_t node; + const char *dst_dir; + + memset(&node, 0, sizeof(node)); + + if (!opt_force_non_empty_dirs) { + if (!directory_exists_and_empty(mysql_data_home, + "Original data")) { + return(false); + } + } else { + if (!directory_exists(mysql_data_home, true)) { + return(false); + } + } + +#ifdef _WIN32 + /* Initialize security descriptor for the new directories + to be the same as for datadir */ + DWORD res = GetNamedSecurityInfoA(mysql_data_home, + SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, + NULL, NULL, NULL, NULL, + &my_dir_security_attributes.lpSecurityDescriptor); + if (res != ERROR_SUCCESS) { + msg("Unable to read security descriptor of %s",mysql_data_home); + } +#endif + + if (srv_undo_dir && *srv_undo_dir + && !directory_exists(srv_undo_dir, true)) { + return(false); + } + if (innobase_data_home_dir && *innobase_data_home_dir + && !directory_exists(innobase_data_home_dir, true)) { + return(false); + } + if (srv_log_group_home_dir && *srv_log_group_home_dir + && !directory_exists(srv_log_group_home_dir, true)) { + return(false); + } + + Copy_back_dst_dir aria_log_dir_path_dst; + const char *aria_log_dir_path_abs= aria_log_dir_path_dst.make(aria_log_dir_path); + if (aria_log_dir_path && *aria_log_dir_path + && !directory_exists(aria_log_dir_path_abs, true)) { + return false; + } + + /* cd to backup directory */ + if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) + { + msg("Can't my_setwd %s", xtrabackup_target_dir); + return(false); + } + + if (!copy_back_aria_logs(aria_log_dir_path_abs)) + return false; + + /* parse data file path */ + + if (!innobase_data_file_path) { + innobase_data_file_path = (char*) "ibdata1:10M:autoextend"; + } + + srv_sys_space.set_path("."); + + if (!srv_sys_space.parse_params(innobase_data_file_path, true)) { + msg("syntax error in innodb_data_file_path"); + return(false); + } + + srv_max_n_threads = 1000; + + /* copy undo tablespaces */ + + Copy_back_dst_dir dst_dir_buf; + + dst_dir = dst_dir_buf.make(srv_undo_dir); + + ds_ctxt *ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); + + for (uint i = 1; i <= TRX_SYS_MAX_UNDO_SPACES; i++) { + char filename[20]; + sprintf(filename, "undo%03u", i); + if (!file_exists(filename)) { + break; + } + if (!(ret = copy_or_move_file(ds_tmp, filename, filename, + dst_dir, 1))) { + goto cleanup; + } + } + + ds_destroy(ds_tmp); + ds_tmp = NULL; + + /* copy redo logs */ + + dst_dir = dst_dir_buf.make(srv_log_group_home_dir); + + /* --backup generates a single ib_logfile0, which we must copy. */ + + ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); + if (!(ret = copy_or_move_file(ds_tmp, LOG_FILE_NAME, LOG_FILE_NAME, + dst_dir, 1))) { + goto cleanup; + } + ds_destroy(ds_tmp); + + /* copy innodb system tablespace(s) */ + + dst_dir = dst_dir_buf.make(innobase_data_home_dir); + + ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); + + for (Tablespace::const_iterator iter(srv_sys_space.begin()), + end(srv_sys_space.end()); + iter != end; + ++iter) { + const char *filepath = iter->filepath(); + if (!(ret = copy_or_move_file(ds_tmp, base_name(filepath), + filepath, dst_dir, 1))) { + goto cleanup; + } + } + + ds_destroy(ds_tmp); + + /* copy the rest of tablespaces */ + ds_tmp = ds_create(mysql_data_home, DS_TYPE_LOCAL); + + it = datadir_iter_new(".", false); + + datadir_node_init(&node); + + while (datadir_iter_next(it, &node)) { + const char *ext_list[] = {"backup-my.cnf", + "xtrabackup_binary", "xtrabackup_binlog_info", + "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp", + NULL}; + const char *filename; + char c_tmp; + int i_tmp; + + /* Skip aria log files */ + if (is_aria_log_dir_file(node)) + continue; + + if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/") +#ifdef _WIN32 + || strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\") +#endif + ) + { + // copied at later step + continue; + } + + /* create empty directories */ + if (node.is_empty_dir) { + char path[FN_REFLEN]; + + snprintf(path, sizeof(path), "%s/%s", + mysql_data_home, node.filepath_rel); + + msg("Creating directory %s", path); + + if (mkdirp(path, 0777, MYF(0)) < 0) { + char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not create directory %s: %s", + path, errbuf); + ret = false; + + goto cleanup; + + } + + msg(" ...done."); + + continue; + } + + filename = base_name(node.filepath); + + /* skip .qp files */ + if (filename_matches(filename, ext_list)) { + continue; + } + + /* skip undo tablespaces */ + if (sscanf(filename, "undo%d%c", &i_tmp, &c_tmp) == 1) { + continue; + } + + /* skip the redo log (it was already copied) */ + if (!strcmp(filename, LOG_FILE_NAME)) { + continue; + } + + /* skip buffer pool dump */ + if (!strcmp(filename, default_buffer_pool_file)) { + continue; + } + + /* skip innodb data files */ + for (Tablespace::const_iterator iter(srv_sys_space.begin()), + end(srv_sys_space.end()); iter != end; ++iter) { + if (!strcmp(base_name(iter->filepath()), filename)) { + goto next_file; + } + } + + if (!(ret = copy_or_move_file(ds_tmp, node.filepath, node.filepath_rel, + mysql_data_home, 1))) { + goto cleanup; + } + next_file: + continue; + } + + /* copy buffer pool dump */ + + if (file_exists(default_buffer_pool_file) && + innobase_buffer_pool_filename) { + copy_or_move_file(ds_tmp, default_buffer_pool_file, + innobase_buffer_pool_filename, + mysql_data_home, 0); + } + + rocksdb_copy_back(ds_tmp); + +cleanup: + if (it != NULL) { + datadir_iter_free(it); + } + + datadir_node_free(&node); + + if (ds_tmp != NULL) { + ds_destroy(ds_tmp); + } + + ds_tmp = NULL; + + return(ret); +} + +bool +decrypt_decompress_file(const char *filepath, uint thread_n) +{ + std::stringstream cmd, message; + char *dest_filepath = strdup(filepath); + bool needs_action = false; + + cmd << IF_WIN("type ","cat ") << filepath; + + if (opt_decompress + && ends_with(filepath, ".qp")) { + cmd << " | qpress -dio "; + dest_filepath[strlen(dest_filepath) - 3] = 0; + if (needs_action) { + message << " and "; + } + message << "decompressing"; + needs_action = true; + } + + cmd << " > " << dest_filepath; + message << " " << filepath; + + free(dest_filepath); + + if (needs_action) { + + msg(thread_n,"%s\n", message.str().c_str()); + + if (system(cmd.str().c_str()) != 0) { + return(false); + } + + if (opt_remove_original) { + msg(thread_n, "Removing %s", filepath); + if (my_delete(filepath, MYF(MY_WME)) != 0) { + return(false); + } + } + } + + return(true); +} + +static void decrypt_decompress_thread_func(datadir_thread_ctxt_t *ctxt) +{ + bool ret = true; + datadir_node_t node; + + datadir_node_init(&node); + + while (datadir_iter_next(ctxt->it, &node)) { + + /* skip empty directories in backup */ + if (node.is_empty_dir) { + continue; + } + + if (!ends_with(node.filepath, ".qp")) { + continue; + } + + if (!(ret = decrypt_decompress_file(node.filepath, + ctxt->n_thread))) { + goto cleanup; + } + } + +cleanup: + + datadir_node_free(&node); + + pthread_mutex_lock(ctxt->count_mutex); + --(*ctxt->count); + pthread_mutex_unlock(ctxt->count_mutex); + + ctxt->ret = ret; +} + +bool +decrypt_decompress() +{ + bool ret; + datadir_iter_t *it = NULL; + + srv_max_n_threads = 1000; + + /* cd to backup directory */ + if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) + { + msg("Can't my_setwd %s", xtrabackup_target_dir); + return(false); + } + + /* copy the rest of tablespaces */ + ds_ctxt *ds_tmp = ds_create(".", DS_TYPE_LOCAL); + + it = datadir_iter_new(".", false); + + ut_a(xtrabackup_parallel >= 0); + + ret = run_data_threads(it, decrypt_decompress_thread_func, + xtrabackup_parallel ? xtrabackup_parallel : 1); + + if (it != NULL) { + datadir_iter_free(it); + } + + if (ds_tmp != NULL) { + ds_destroy(ds_tmp); + } + + ds_tmp = NULL; + + return(ret); +} + +/* + Copy some files from top level datadir. + Do not copy the Innodb files (ibdata1, redo log files), + as this is done in a separate step. +*/ +static bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix) +{ + os_file_dir_t dir = os_file_opendir(dir_path); + if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false; + + os_file_stat_t info; + bool ret = true; + while (os_file_readdir_next_file(dir_path, dir, &info) == 0) { + + if (info.type != OS_FILE_TYPE_FILE) + continue; + + const char *pname = strrchr(info.name, '/'); +#ifdef _WIN32 + if (const char *last = strrchr(info.name, '\\')) { + if (!pname || last >pname) { + pname = last; + } + } +#endif + if (!pname) + pname = info.name; + + if (!starts_with(pname, prefix)) + /* For ES exchange the above line with the following code: + (!xtrabackup_prepare || !xtrabackup_incremental_dir || + !starts_with(pname, "aria_log"))) + */ + continue; + + if (xtrabackup_prepare && xtrabackup_incremental_dir && + file_exists(info.name)) + unlink(info.name); + + std::string full_path(dir_path); + full_path.append(1, '/').append(info.name); + if (!(ret = ds_data->copy_file(full_path.c_str() , info.name, 1))) + break; + } + os_file_closedir(dir); + return ret; +} + + +static int rocksdb_remove_checkpoint_directory() +{ + xb_mysql_query(mysql_connection, "set global rocksdb_remove_mariabackup_checkpoint=ON", false); + return 0; +} + +static bool has_rocksdb_plugin() +{ + static bool first_time = true; + static bool has_plugin= false; + if (!first_time || !xb_backup_rocksdb) + return has_plugin; + + const char *query = "SELECT COUNT(*) FROM information_schema.plugins WHERE plugin_name='rocksdb'"; + MYSQL_RES* result = xb_mysql_query(mysql_connection, query, true); + MYSQL_ROW row = mysql_fetch_row(result); + if (row) + has_plugin = !strcmp(row[0], "1"); + mysql_free_result(result); + first_time = false; + return has_plugin; +} + +static char *trim_trailing_dir_sep(char *path) +{ + size_t path_len = strlen(path); + while (path_len) + { + char c = path[path_len - 1]; + if (c == '/' IF_WIN(|| c == '\\', )) + path_len--; + else + break; + } + path[path_len] = 0; + return path; +} + +/* +Create a file hardlink. +@return true on success, false on error. +*/ +bool +ds_ctxt_t::make_hardlink(const char *from_path, const char *to_path) +{ + DBUG_EXECUTE_IF("no_hardlinks", return false;); + char to_path_full[FN_REFLEN]; + if (!is_abs_path(to_path)) + { + fn_format(to_path_full, to_path, root, "", MYF(MY_RELATIVE_PATH)); + } + else + { + strncpy(to_path_full, to_path, sizeof(to_path_full)); + } +#ifdef _WIN32 + return CreateHardLink(to_path_full, from_path, NULL); +#else + return !link(from_path, to_path_full); +#endif +} + +/* + Copies or moves a directory (non-recursively so far). + Helper function used to backup rocksdb checkpoint, or copy-back the + rocksdb files. + + Has optimization that allows to use hardlinks when possible + (source and destination are directories on the same device) +*/ +void +ds_ctxt_t::copy_or_move_dir(const char *from, const char *to, + bool do_copy, bool allow_hardlinks) +{ + datadir_node_t node; + datadir_node_init(&node); + datadir_iter_t *it = datadir_iter_new(from, false); + + while (datadir_iter_next(it, &node)) + { + char to_path[FN_REFLEN]; + const char *from_path = node.filepath; + snprintf(to_path, sizeof(to_path), "%s/%s", to, base_name(from_path)); + bool rc = false; + if (do_copy && allow_hardlinks) + { + rc = make_hardlink(from_path, to_path); + if (rc) + { + msg("Creating hardlink from %s to %s",from_path, to_path); + } + else + { + allow_hardlinks = false; + } + } + + if (!rc) + { + rc = (do_copy ? + copy_file(from_path, to_path, 1) : + move_file(from_path, node.filepath_rel, + to, 1)); + } + if (!rc) + die("copy or move file failed"); + } + datadir_iter_free(it); + datadir_node_free(&node); + +} + +/* + Obtain user level lock , to protect the checkpoint directory of the server + from being user/overwritten by different backup processes, if backups are + running in parallel. + + This lock will be acquired before rocksdb checkpoint is created, held + while all files from it are being copied to their final backup destination, + and finally released after the checkpoint is removed. +*/ +static void rocksdb_lock_checkpoint() +{ + msg("Obtaining rocksdb checkpoint lock."); + MYSQL_RES *res = + xb_mysql_query(mysql_connection, "SELECT GET_LOCK('mariabackup_rocksdb_checkpoint',3600)", true, true); + + MYSQL_ROW r = mysql_fetch_row(res); + if (r && r[0] && strcmp(r[0], "1")) + { + msg("Could not obtain rocksdb checkpont lock."); + exit(EXIT_FAILURE); + } + mysql_free_result(res); +} + +static void rocksdb_unlock_checkpoint() +{ + xb_mysql_query(mysql_connection, + "SELECT RELEASE_LOCK('mariabackup_rocksdb_checkpoint')", false, true); +} + + +/* + Create temporary checkpoint in $rocksdb_datadir/mariabackup-checkpoint + directory. + A (user-level) lock named 'mariabackup_rocksdb_checkpoint' will also be + acquired be this function. +*/ +#define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint" +static char rocksdb_checkpoint_dir[FN_REFLEN]; + +static void rocksdb_create_checkpoint() +{ + MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true); + MYSQL_ROW row = mysql_fetch_row(result); + + DBUG_ASSERT(row && row[0] && row[1]); + + char *rocksdbdir = row[0]; + char *datadir = row[1]; + + if (is_abs_path(rocksdbdir)) + { + snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir), + "%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(rocksdbdir)); + } + else + { + snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir), + "%s/%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(datadir), + trim_dotslash(rocksdbdir)); + } + mysql_free_result(result); + +#ifdef _WIN32 + for (char *p = rocksdb_checkpoint_dir; *p; p++) + if (*p == '\\') *p = '/'; +#endif + + rocksdb_lock_checkpoint(); + + if (!access(rocksdb_checkpoint_dir, 0)) + { + msg("Removing rocksdb checkpoint from previous backup attempt."); + rocksdb_remove_checkpoint_directory(); + } + + char query[FN_REFLEN + 32]; + snprintf(query, sizeof(query), "SET GLOBAL rocksdb_create_checkpoint='%s'", rocksdb_checkpoint_dir); + xb_mysql_query(mysql_connection, query, false, true); +} + +/* + Copy files from rocksdb temporary checkpoint to final destination. + remove temp.checkpoint directory (in server's datadir) + and release user level lock acquired inside rocksdb_create_checkpoint(). +*/ +static void rocksdb_backup_checkpoint(ds_ctxt *ds_data) +{ + msg("Backing up rocksdb files."); + char rocksdb_backup_dir[FN_REFLEN]; + snprintf(rocksdb_backup_dir, sizeof(rocksdb_backup_dir), "%s/" ROCKSDB_BACKUP_DIR , xtrabackup_target_dir); + bool backup_to_directory = xtrabackup_backup && xtrabackup_stream_fmt == XB_STREAM_FMT_NONE; + if (backup_to_directory) + { + if (my_mkdir(rocksdb_backup_dir, 0777, MYF(0))){ + die("Can't create rocksdb backup directory %s", rocksdb_backup_dir); + } + } + ds_data->copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory); + rocksdb_remove_checkpoint_directory(); + rocksdb_unlock_checkpoint(); +} + +/* + Copies #rocksdb directory to the $rockdb_data_dir, on copy-back +*/ +static void rocksdb_copy_back(ds_ctxt *ds_data) { + if (access(ROCKSDB_BACKUP_DIR, 0)) + return; + char rocksdb_home_dir[FN_REFLEN]; + if (xb_rocksdb_datadir && is_abs_path(xb_rocksdb_datadir)) { + strncpy(rocksdb_home_dir, xb_rocksdb_datadir, sizeof rocksdb_home_dir - 1); + rocksdb_home_dir[sizeof rocksdb_home_dir - 1] = '\0'; + } else { + snprintf(rocksdb_home_dir, sizeof(rocksdb_home_dir), "%s/%s", mysql_data_home, + xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR); + } + mkdirp(rocksdb_home_dir, 0777, MYF(0)); + ds_data->copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back); +} diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h new file mode 100644 index 00000000..b4a323f2 --- /dev/null +++ b/extra/mariabackup/backup_copy.h @@ -0,0 +1,43 @@ + +#ifndef XTRABACKUP_BACKUP_COPY_H +#define XTRABACKUP_BACKUP_COPY_H + +#include <my_global.h> +#include <mysql.h> +#include "datasink.h" + +/* special files */ +#define XTRABACKUP_SLAVE_INFO "xtrabackup_slave_info" +#define XTRABACKUP_GALERA_INFO "xtrabackup_galera_info" +#define XTRABACKUP_BINLOG_INFO "xtrabackup_binlog_info" +#define XTRABACKUP_INFO "xtrabackup_info" + +extern bool binlog_locked; + +/************************************************************************ +Return true if first and second arguments are the same path. */ +bool +equal_paths(const char *first, const char *second); + +/** Start --backup */ +bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, + CorruptedPages &corrupted_pages); +/** Release resources after backup_start() */ +void backup_release(); +/** Finish after backup_start() and backup_release() */ +bool backup_finish(ds_ctxt *ds_data); +bool +apply_log_finish(); +bool +copy_back(); +bool +decrypt_decompress(); +bool +is_path_separator(char); +bool +directory_exists(const char *dir, bool create); + +lsn_t +get_current_lsn(MYSQL *connection); + +#endif diff --git a/extra/mariabackup/backup_debug.h b/extra/mariabackup/backup_debug.h new file mode 100644 index 00000000..777b4f4a --- /dev/null +++ b/extra/mariabackup/backup_debug.h @@ -0,0 +1,24 @@ +#pragma once +#include "my_dbug.h" +#ifndef DBUG_OFF +char *dbug_mariabackup_get_val(const char *event, fil_space_t::name_type key); +/* +In debug mode, execute SQL statement that was passed via environment. +To use this facility, you need to + +1. Add code DBUG_EXECUTE_MARIABACKUP_EVENT("my_event_name", key);); + to the code. key is usually a table name +2. Set environment variable my_event_name_$key SQL statement you want to execute + when event occurs, in DBUG_EXECUTE_IF from above. + In mtr , you can set environment via 'let' statement (do not use $ as the first char + for the variable) +3. start mariabackup with --dbug=+d,debug_mariabackup_events +*/ +#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) \ + DBUG_EXECUTE_IF("mariabackup_inject_code", \ + { char *dbug_val= dbug_mariabackup_get_val(EVENT, KEY); \ + if (dbug_val) CODE }) +#else +#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) +#endif + diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc new file mode 100644 index 00000000..cf8a5051 --- /dev/null +++ b/extra/mariabackup/backup_mysql.cc @@ -0,0 +1,1947 @@ +/****************************************************** +hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 51 Franklin +Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ +#define MYSQL_CLIENT + +#include <my_global.h> +#include <mysql.h> +#include <mysqld.h> +#include <my_sys.h> +#include <stdlib.h> +#include <string.h> +#include <limits> +#include "common.h" +#include "xtrabackup.h" +#include "srv0srv.h" +#include "mysql_version.h" +#include "backup_copy.h" +#include "backup_mysql.h" +#include "mysqld.h" +#include "xb_plugin.h" +#include <sstream> +#include <sql_error.h> +#include "page0zip.h" + +char *tool_name; +char tool_args[2048]; + +ulong mysql_server_version; + +/* server capabilities */ +bool have_changed_page_bitmaps = false; +bool have_backup_locks = false; +bool have_lock_wait_timeout = false; +bool have_galera_enabled = false; +bool have_multi_threaded_slave = false; +bool have_gtid_slave = false; + +/* Kill long selects */ +static mysql_mutex_t kill_query_thread_mutex; +static bool kill_query_thread_running, kill_query_thread_stopping; +static mysql_cond_t kill_query_thread_stopped; +static mysql_cond_t kill_query_thread_stop; + +bool sql_thread_started = false; +char *mysql_slave_position = NULL; +char *mysql_binlog_position = NULL; +char *buffer_pool_filename = NULL; + +/* History on server */ +time_t history_start_time; +time_t history_end_time; +time_t history_lock_time; + +MYSQL *mysql_connection; + +extern my_bool opt_ssl_verify_server_cert, opt_use_ssl; + +MYSQL * +xb_mysql_connect() +{ + MYSQL *connection = mysql_init(NULL); + char mysql_port_str[std::numeric_limits<int>::digits10 + 3]; + + sprintf(mysql_port_str, "%d", opt_port); + + if (connection == NULL) { + msg("Failed to init MariaDB struct: %s.", + mysql_error(connection)); + return(NULL); + } + +#if !defined(DONT_USE_MYSQL_PWD) + if (!opt_password) + { + opt_password=getenv("MYSQL_PWD"); + } +#endif + + if (!opt_secure_auth) { + mysql_options(connection, MYSQL_SECURE_AUTH, + (char *) &opt_secure_auth); + } + + if (xb_plugin_dir && *xb_plugin_dir){ + mysql_options(connection, MYSQL_PLUGIN_DIR, xb_plugin_dir); + } + mysql_options(connection, MYSQL_OPT_PROTOCOL, &opt_protocol); + mysql_options(connection,MYSQL_SET_CHARSET_NAME, "utf8"); + + msg("Connecting to MariaDB server host: %s, user: %s, password: %s, " + "port: %s, socket: %s", opt_host ? opt_host : "localhost", + opt_user ? opt_user : "not set", + opt_password ? "set" : "not set", + opt_port != 0 ? mysql_port_str : "not set", + opt_socket ? opt_socket : "not set"); + +#ifdef HAVE_OPENSSL + if (opt_use_ssl && opt_protocol <= MYSQL_PROTOCOL_SOCKET) + { + mysql_ssl_set(connection, opt_ssl_key, opt_ssl_cert, + opt_ssl_ca, opt_ssl_capath, + opt_ssl_cipher); + mysql_options(connection, MYSQL_OPT_SSL_CRL, opt_ssl_crl); + mysql_options(connection, MYSQL_OPT_SSL_CRLPATH, + opt_ssl_crlpath); + } + mysql_options(connection,MYSQL_OPT_SSL_VERIFY_SERVER_CERT, + (char*)&opt_ssl_verify_server_cert); +#endif + + if (!mysql_real_connect(connection, + opt_host ? opt_host : "localhost", + opt_user, + opt_password, + "" /*database*/, opt_port, + opt_socket, 0)) { + msg("Failed to connect to MariaDB server: %s.", mysql_error(connection)); + mysql_close(connection); + return(NULL); + } + + xb_mysql_query(connection, "SET SESSION wait_timeout=2147483, max_statement_time=0", + false, true); + + return(connection); +} + +/*********************************************************************//** +Execute mysql query. */ +MYSQL_RES * +xb_mysql_query(MYSQL *connection, const char *query, bool use_result, + bool die_on_error) +{ + MYSQL_RES *mysql_result = NULL; + + if (mysql_query(connection, query)) { + if (die_on_error) { + die("failed to execute query %s: %s", query, mysql_error(connection)); + } else { + msg("Error: failed to execute query %s: %s", query, mysql_error(connection)); + } + return(NULL); + } + + /* store result set on client if there is a result */ + if (mysql_field_count(connection) > 0) { + if ((mysql_result = mysql_store_result(connection)) == NULL) { + die("failed to fetch query result %s: %s", + query, mysql_error(connection)); + } + + if (!use_result) { + mysql_free_result(mysql_result); + mysql_result = NULL; + } + } + + return mysql_result; +} + + +struct mysql_variable { + const char *name; + char **value; +}; + + +static +void +read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, + bool vertical_result) +{ + MYSQL_RES *mysql_result; + MYSQL_ROW row; + mysql_variable *var; + + mysql_result = xb_mysql_query(connection, query, true); + + ut_ad(!vertical_result || mysql_num_fields(mysql_result) == 2); + + if (vertical_result) { + while ((row = mysql_fetch_row(mysql_result))) { + char *name = row[0]; + char *value = row[1]; + for (var = vars; var->name; var++) { + if (strcmp(var->name, name) == 0 + && value != NULL) { + *(var->value) = strdup(value); + } + } + } + } else { + MYSQL_FIELD *field; + + if ((row = mysql_fetch_row(mysql_result)) != NULL) { + int i = 0; + while ((field = mysql_fetch_field(mysql_result)) + != NULL) { + char *name = field->name; + char *value = row[i]; + for (var = vars; var->name; var++) { + if (strcmp(var->name, name) == 0 + && value != NULL) { + *(var->value) = strdup(value); + } + } + ++i; + } + } + } + + mysql_free_result(mysql_result); +} + + +static +void +free_mysql_variables(mysql_variable *vars) +{ + mysql_variable *var; + + for (var = vars; var->name; var++) { + free(*(var->value)); + } +} + + +static +char * +read_mysql_one_value(MYSQL *connection, const char *query, + uint column, uint expect_columns) +{ + MYSQL_RES *mysql_result; + MYSQL_ROW row; + char *result = NULL; + + mysql_result = xb_mysql_query(connection, query, true); + + ut_ad(mysql_num_fields(mysql_result) == expect_columns); + + if ((row = mysql_fetch_row(mysql_result))) { + result = strdup(row[column]); + } + + mysql_free_result(mysql_result); + + return(result); +} + + +static +char * +read_mysql_one_value(MYSQL *mysql, const char *query) +{ + return read_mysql_one_value(mysql, query, 0/*offset*/, 1/*total columns*/); +} + + +static +bool +check_server_version(ulong version_number, const char *version_string) +{ + if (strstr(version_string, "MariaDB") && version_number >= 100800) + return true; + + msg("Error: Unsupported server version: '%s'.", version_string); + return false; +} + +/*********************************************************************//** +Receive options important for XtraBackup from server. +@return true on success. */ +bool get_mysql_vars(MYSQL *connection) +{ + char *gtid_mode_var= NULL; + char *version_var= NULL; + char *have_backup_locks_var= NULL; + char *log_bin_var= NULL; + char *lock_wait_timeout_var= NULL; + char *wsrep_on_var= NULL; + char *slave_parallel_workers_var= NULL; + char *gtid_slave_pos_var= NULL; + char *innodb_buffer_pool_filename_var= NULL; + char *datadir_var= NULL; + char *innodb_log_group_home_dir_var= NULL; + char *innodb_log_file_size_var= NULL; + char *innodb_log_files_in_group_var= NULL; + char *innodb_data_file_path_var= NULL; + char *innodb_data_home_dir_var= NULL; + char *innodb_undo_directory_var= NULL; + char *innodb_page_size_var= NULL; + char *innodb_undo_tablespaces_var= NULL; + char *aria_log_dir_path_var= NULL; + char *page_zip_level_var= NULL; + char *ignore_db_dirs= NULL; + char *endptr; + ulong server_version= mysql_get_server_version(connection); + + bool ret= true; + + mysql_variable mysql_vars[]= { + {"have_backup_locks", &have_backup_locks_var}, + {"log_bin", &log_bin_var}, + {"lock_wait_timeout", &lock_wait_timeout_var}, + {"gtid_mode", >id_mode_var}, + {"version", &version_var}, + {"wsrep_on", &wsrep_on_var}, + {"slave_parallel_workers", &slave_parallel_workers_var}, + {"gtid_slave_pos", >id_slave_pos_var}, + {"innodb_buffer_pool_filename", &innodb_buffer_pool_filename_var}, + {"datadir", &datadir_var}, + {"innodb_log_group_home_dir", &innodb_log_group_home_dir_var}, + {"innodb_log_file_size", &innodb_log_file_size_var}, + {"innodb_log_files_in_group", &innodb_log_files_in_group_var}, + {"innodb_data_file_path", &innodb_data_file_path_var}, + {"innodb_data_home_dir", &innodb_data_home_dir_var}, + {"innodb_undo_directory", &innodb_undo_directory_var}, + {"innodb_page_size", &innodb_page_size_var}, + {"innodb_undo_tablespaces", &innodb_undo_tablespaces_var}, + {"innodb_compression_level", &page_zip_level_var}, + {"ignore_db_dirs", &ignore_db_dirs}, + {"aria_log_dir_path", &aria_log_dir_path_var}, + {NULL, NULL}}; + + read_mysql_variables(connection, "SHOW VARIABLES", mysql_vars, true); + + if (have_backup_locks_var != NULL && !opt_no_backup_locks) + { + have_backup_locks= true; + } + + if (opt_binlog_info == BINLOG_INFO_AUTO) + { + if (log_bin_var != NULL && !strcmp(log_bin_var, "ON")) + opt_binlog_info= BINLOG_INFO_ON; + else + opt_binlog_info= BINLOG_INFO_OFF; + } + + if (lock_wait_timeout_var != NULL) + { + have_lock_wait_timeout= true; + } + + if (wsrep_on_var != NULL) + { + have_galera_enabled= true; + } + + /* Check server version compatibility */ + if (!(ret= check_server_version(server_version, version_var))) + { + goto out; + } + + mysql_server_version= server_version; + + if (slave_parallel_workers_var != NULL && + atoi(slave_parallel_workers_var) > 0) + { + have_multi_threaded_slave= true; + } + + if (innodb_buffer_pool_filename_var != NULL) + { + buffer_pool_filename= strdup(innodb_buffer_pool_filename_var); + } + + if ((gtid_mode_var && strcmp(gtid_mode_var, "ON") == 0) || + (gtid_slave_pos_var && *gtid_slave_pos_var)) + { + have_gtid_slave= true; + } + + msg("Using server version %s", version_var); + + if (!(ret= detect_mysql_capabilities_for_backup())) + { + goto out; + } + + /* make sure datadir value is the same in configuration file */ + if (check_if_param_set("datadir")) + { + if (!directory_exists(mysql_data_home, false)) + { + msg("Warning: option 'datadir' points to " + "nonexistent directory '%s'", + mysql_data_home); + } + if (!directory_exists(datadir_var, false)) + { + msg("Warning: MariaDB variable 'datadir' points to " + "nonexistent directory '%s'", + datadir_var); + } + if (!equal_paths(mysql_data_home, datadir_var)) + { + msg("Warning: option 'datadir' has different " + "values:\n" + " '%s' in defaults file\n" + " '%s' in SHOW VARIABLES", + mysql_data_home, datadir_var); + } + } + + /* get some default values is they are missing from my.cnf */ + if (datadir_var && *datadir_var) + { + strmake(mysql_real_data_home, datadir_var, FN_REFLEN - 1); + mysql_data_home= mysql_real_data_home; + } + + if (innodb_data_file_path_var && *innodb_data_file_path_var) + innobase_data_file_path= my_strdup(PSI_NOT_INSTRUMENTED, + innodb_data_file_path_var, MYF(MY_FAE)); + + if (innodb_data_home_dir_var) + innobase_data_home_dir= my_strdup(PSI_NOT_INSTRUMENTED, + innodb_data_home_dir_var, MYF(MY_FAE)); + + if (innodb_log_group_home_dir_var && *innodb_log_group_home_dir_var) + srv_log_group_home_dir= my_strdup(PSI_NOT_INSTRUMENTED, + innodb_log_group_home_dir_var, + MYF(MY_FAE)); + + if (innodb_undo_directory_var && *innodb_undo_directory_var) + srv_undo_dir= my_strdup(PSI_NOT_INSTRUMENTED, innodb_undo_directory_var, + MYF(MY_FAE)); + + if (innodb_log_file_size_var) + { + srv_log_file_size= strtoll(innodb_log_file_size_var, &endptr, 10); + ut_ad(*endptr == 0); + } + + if (innodb_page_size_var) + { + innobase_page_size= strtoll(innodb_page_size_var, &endptr, 10); + ut_ad(*endptr == 0); + } + + if (innodb_undo_tablespaces_var) + { + srv_undo_tablespaces= static_cast<uint32_t> + (strtoul(innodb_undo_tablespaces_var, &endptr, 10)); + ut_ad(*endptr == 0); + } + + if (aria_log_dir_path_var) + { + aria_log_dir_path= my_strdup(PSI_NOT_INSTRUMENTED, + aria_log_dir_path_var, MYF(MY_FAE)); + } + + if (page_zip_level_var != NULL) + { + page_zip_level= static_cast<uint>(strtoul(page_zip_level_var, &endptr, + 10)); + ut_ad(*endptr == 0); + } + + if (ignore_db_dirs) + xb_load_list_string(ignore_db_dirs, ",", register_ignore_db_dirs_filter); + +out: + free_mysql_variables(mysql_vars); + + return (ret); +} + +/*********************************************************************//** +Query the server to find out what backup capabilities it supports. +@return true on success. */ +bool +detect_mysql_capabilities_for_backup() +{ + const char *query = "SELECT 'INNODB_CHANGED_PAGES', COUNT(*) FROM " + "INFORMATION_SCHEMA.PLUGINS " + "WHERE PLUGIN_NAME LIKE 'INNODB_CHANGED_PAGES'"; + char *innodb_changed_pages = NULL; + mysql_variable vars[] = { + {"INNODB_CHANGED_PAGES", &innodb_changed_pages}, {NULL, NULL}}; + + if (xtrabackup_incremental) { + + read_mysql_variables(mysql_connection, query, vars, true); + + ut_ad(innodb_changed_pages != NULL); + + have_changed_page_bitmaps = (atoi(innodb_changed_pages) == 1); + + free_mysql_variables(vars); + } + + /* do some sanity checks */ + if (opt_galera_info && !have_galera_enabled) { + msg("--galera-info is specified on the command " + "line, but the server does not support Galera " + "replication. Ignoring the option."); + opt_galera_info = false; + } + + if (opt_slave_info && have_multi_threaded_slave && + !have_gtid_slave) { + msg("The --slave-info option requires GTID enabled for a " + "multi-threaded slave."); + return(false); + } + + return(true); +} + +static +bool +select_incremental_lsn_from_history(lsn_t *incremental_lsn) +{ + MYSQL_RES *mysql_result; + char query[1000]; + char buf[100]; + + if (opt_incremental_history_name) { + mysql_real_escape_string(mysql_connection, buf, + opt_incremental_history_name, + (unsigned long)strlen(opt_incremental_history_name)); + snprintf(query, sizeof(query), + "SELECT innodb_to_lsn " + "FROM " XB_HISTORY_TABLE " " + "WHERE name = '%s' " + "AND innodb_to_lsn IS NOT NULL " + "ORDER BY innodb_to_lsn DESC LIMIT 1", + buf); + } + + if (opt_incremental_history_uuid) { + mysql_real_escape_string(mysql_connection, buf, + opt_incremental_history_uuid, + (unsigned long)strlen(opt_incremental_history_uuid)); + snprintf(query, sizeof(query), + "SELECT innodb_to_lsn " + "FROM " XB_HISTORY_TABLE " " + "WHERE uuid = '%s' " + "AND innodb_to_lsn IS NOT NULL " + "ORDER BY innodb_to_lsn DESC LIMIT 1", + buf); + } + + mysql_result = xb_mysql_query(mysql_connection, query, true); + + ut_ad(mysql_num_fields(mysql_result) == 1); + const MYSQL_ROW row = mysql_fetch_row(mysql_result); + if (row) { + *incremental_lsn = strtoull(row[0], NULL, 10); + msg("Found and using lsn: " LSN_PF " for %s %s", + *incremental_lsn, + opt_incremental_history_uuid ? "uuid" : "name", + opt_incremental_history_uuid ? + opt_incremental_history_uuid : + opt_incremental_history_name); + } else { + msg("Error while attempting to find history record " + "for %s %s", + opt_incremental_history_uuid ? "uuid" : "name", + opt_incremental_history_uuid ? + opt_incremental_history_uuid : + opt_incremental_history_name); + } + + mysql_free_result(mysql_result); + + return(row != NULL); +} + +static +const char * +eat_sql_whitespace(const char *query) +{ + bool comment = false; + + while (*query) { + if (comment) { + if (query[0] == '*' && query[1] == '/') { + query += 2; + comment = false; + continue; + } + ++query; + continue; + } + if (query[0] == '/' && query[1] == '*') { + query += 2; + comment = true; + continue; + } + if (strchr("\t\n\r (", query[0])) { + ++query; + continue; + } + break; + } + + return(query); +} + +static +bool +is_query_from_list(const char *query, const char **list) +{ + const char **item; + + query = eat_sql_whitespace(query); + + item = list; + while (*item) { + if (strncasecmp(query, *item, strlen(*item)) == 0) { + return(true); + } + ++item; + } + + return(false); +} + +static +bool +is_query(const char *query) +{ + const char *query_list[] = {"insert", "update", "delete", "replace", + "alter", "load", "select", "do", "handler", "call", "execute", + "begin", NULL}; + + return is_query_from_list(query, query_list); +} + +static +bool +is_select_query(const char *query) +{ + const char *query_list[] = {"select", NULL}; + + return is_query_from_list(query, query_list); +} + +static +bool +is_update_query(const char *query) +{ + const char *query_list[] = {"insert", "update", "delete", "replace", + "alter", "load", NULL}; + + return is_query_from_list(query, query_list); +} + +static +bool +have_queries_to_wait_for(MYSQL *connection, uint threshold) +{ + MYSQL_RES *result = xb_mysql_query(connection, "SHOW FULL PROCESSLIST", + true); + const bool all_queries = (opt_lock_wait_query_type == QUERY_TYPE_ALL); + bool have_to_wait = false; + + while (MYSQL_ROW row = mysql_fetch_row(result)) { + const char *info = row[7]; + int duration = row[5] ? atoi(row[5]) : 0; + char *id = row[0]; + + if (info != NULL + && duration >= (int)threshold + && ((all_queries && is_query(info)) + || is_update_query(info))) { + msg("Waiting for query %s (duration %d sec): %s", + id, duration, info); + have_to_wait = true; + break; + } + } + + mysql_free_result(result); + return(have_to_wait); +} + +static +void +kill_long_queries(MYSQL *connection, time_t timeout) +{ + char kill_stmt[100]; + + MYSQL_RES *result = xb_mysql_query(connection, "SHOW FULL PROCESSLIST", + true); + const bool all_queries = (opt_kill_long_query_type == QUERY_TYPE_ALL); + while (MYSQL_ROW row = mysql_fetch_row(result)) { + const char *info = row[7]; + long long duration = row[5]? atoll(row[5]) : 0; + char *id = row[0]; + + if (info != NULL && + (time_t)duration >= timeout && + ((all_queries && is_query(info)) || + is_select_query(info))) { + msg("Killing query %s (duration %d sec): %s", + id, (int)duration, info); + snprintf(kill_stmt, sizeof(kill_stmt), + "KILL %s", id); + xb_mysql_query(connection, kill_stmt, false, false); + } + } + + mysql_free_result(result); +} + +static +bool +wait_for_no_updates(MYSQL *connection, uint timeout, uint threshold) +{ + time_t start_time; + + start_time = time(NULL); + + msg("Waiting %u seconds for queries running longer than %u seconds " + "to finish", timeout, threshold); + + while (time(NULL) <= (time_t)(start_time + timeout)) { + if (!have_queries_to_wait_for(connection, threshold)) { + return(true); + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + msg("Unable to obtain lock. Please try again later."); + + return(false); +} + +static void kill_query_thread() +{ + mysql_mutex_lock(&kill_query_thread_mutex); + + msg("Kill query timeout %d seconds.", opt_kill_long_queries_timeout); + + time_t start_time= time(nullptr); + timespec abstime; + set_timespec(abstime, opt_kill_long_queries_timeout); + + while (!kill_query_thread_stopping) + if (!mysql_cond_timedwait(&kill_query_thread_stop, + &kill_query_thread_mutex, &abstime)) + goto func_exit; + + if (MYSQL *mysql= xb_mysql_connect()) + { + do + { + kill_long_queries(mysql, time(nullptr) - start_time); + set_timespec(abstime, 1); + } + while (mysql_cond_timedwait(&kill_query_thread_stop, + &kill_query_thread_mutex, &abstime) && + !kill_query_thread_stopping); + mysql_close(mysql); + } + else + msg("Error: kill query thread failed"); + +func_exit: + msg("Kill query thread stopped"); + + kill_query_thread_running= false; + mysql_cond_signal(&kill_query_thread_stopped); + mysql_mutex_unlock(&kill_query_thread_mutex); +} + + +static void start_query_killer() +{ + ut_ad(!kill_query_thread_running); + kill_query_thread_running= true; + kill_query_thread_stopping= false; + mysql_mutex_init(0, &kill_query_thread_mutex, nullptr); + mysql_cond_init(0, &kill_query_thread_stop, nullptr); + mysql_cond_init(0, &kill_query_thread_stopped, nullptr); + std::thread(kill_query_thread).detach(); +} + +static void stop_query_killer() +{ + mysql_mutex_lock(&kill_query_thread_mutex); + kill_query_thread_stopping= true; + mysql_cond_signal(&kill_query_thread_stop); + + do + mysql_cond_wait(&kill_query_thread_stopped, &kill_query_thread_mutex); + while (kill_query_thread_running); + + mysql_cond_destroy(&kill_query_thread_stop); + mysql_cond_destroy(&kill_query_thread_stopped); + mysql_mutex_unlock(&kill_query_thread_mutex); + mysql_mutex_destroy(&kill_query_thread_mutex); +} + + +/*********************************************************************//** +Function acquires either a backup tables lock, if supported +by the server, or a global read lock (FLUSH TABLES WITH READ LOCK) +otherwise. +@returns true if lock acquired */ +bool lock_tables(MYSQL *connection) +{ + if (have_lock_wait_timeout || opt_lock_wait_timeout) + { + char buf[FN_REFLEN]; + /* Set the maximum supported session value for + lock_wait_timeout if opt_lock_wait_timeout is not set to prevent + unnecessary timeouts when the global value is changed from the default */ + snprintf(buf, sizeof(buf), "SET SESSION lock_wait_timeout=%u", + opt_lock_wait_timeout ? opt_lock_wait_timeout : 31536000); + xb_mysql_query(connection, buf, false); + } + + if (have_backup_locks) + { + msg("Executing LOCK TABLES FOR BACKUP..."); + xb_mysql_query(connection, "LOCK TABLES FOR BACKUP", false); + return (true); + } + + if (opt_lock_wait_timeout) + { + if (!wait_for_no_updates(connection, opt_lock_wait_timeout, + opt_lock_wait_threshold)) + { + return (false); + } + } + + msg("Acquiring BACKUP LOCKS..."); + + if (opt_kill_long_queries_timeout) + { + start_query_killer(); + } + + if (have_galera_enabled) + { + xb_mysql_query(connection, "SET SESSION wsrep_sync_wait=0", false); + } + + xb_mysql_query(connection, "BACKUP STAGE START", true); + DBUG_MARIABACKUP_EVENT("after_backup_stage_start", {}); + xb_mysql_query(connection, "BACKUP STAGE BLOCK_COMMIT", true); + DBUG_MARIABACKUP_EVENT("after_backup_stage_block_commit", {}); + /* Set the maximum supported session value for + lock_wait_timeout to prevent unnecessary timeouts when the + global value is changed from the default */ + if (opt_lock_wait_timeout) + xb_mysql_query(connection, "SET SESSION lock_wait_timeout=31536000", + false); + + if (opt_kill_long_queries_timeout) + { + stop_query_killer(); + } + + return (true); +} + +/*********************************************************************//** +If backup locks are used, execute LOCK BINLOG FOR BACKUP provided that we are +not in the --no-lock mode and the lock has not been acquired already. +@returns true if lock acquired */ +bool +lock_binlog_maybe(MYSQL *connection) +{ + if (have_backup_locks && !opt_no_lock && !binlog_locked) { + msg("Executing LOCK BINLOG FOR BACKUP..."); + xb_mysql_query(connection, "LOCK BINLOG FOR BACKUP", false); + binlog_locked = true; + + return(true); + } + + return(false); +} + + +/*********************************************************************//** +Releases either global read lock acquired with FTWRL and the binlog +lock acquired with LOCK BINLOG FOR BACKUP, depending on +the locking strategy being used */ +void +unlock_all(MYSQL *connection) +{ + if (opt_debug_sleep_before_unlock) { + msg("Debug sleep for %u seconds", + opt_debug_sleep_before_unlock); + std::this_thread::sleep_for( + std::chrono::milliseconds(opt_debug_sleep_before_unlock)); + } + + msg("Executing BACKUP STAGE END"); + xb_mysql_query(connection, "BACKUP STAGE END", false); + + msg("All tables unlocked"); +} + + +static +int +get_open_temp_tables(MYSQL *connection) +{ + char *slave_open_temp_tables = NULL; + mysql_variable status[] = { + {"Slave_open_temp_tables", &slave_open_temp_tables}, + {NULL, NULL} + }; + int result = false; + + read_mysql_variables(connection, + "SHOW STATUS LIKE 'slave_open_temp_tables'", status, true); + + result = slave_open_temp_tables ? atoi(slave_open_temp_tables) : 0; + + free_mysql_variables(status); + + return(result); +} + +/*********************************************************************//** +Wait until it's safe to backup a slave. Returns immediately if +the host isn't a slave. Currently there's only one check: +Slave_open_temp_tables has to be zero. Dies on timeout. */ +bool +wait_for_safe_slave(MYSQL *connection) +{ + char *read_master_log_pos = NULL; + char *slave_sql_running = NULL; + int n_attempts = 1; + const int sleep_time = 3; + int open_temp_tables = 0; + bool result = true; + + mysql_variable status[] = { + {"Read_Master_Log_Pos", &read_master_log_pos}, + {"Slave_SQL_Running", &slave_sql_running}, + {NULL, NULL} + }; + + sql_thread_started = false; + + read_mysql_variables(connection, "SHOW SLAVE STATUS", status, false); + + if (!(read_master_log_pos && slave_sql_running)) { + msg("Not checking slave open temp tables for " + "--safe-slave-backup because host is not a slave"); + goto cleanup; + } + + if (strcmp(slave_sql_running, "Yes") == 0) { + sql_thread_started = true; + xb_mysql_query(connection, "STOP SLAVE SQL_THREAD", false); + } + + if (opt_safe_slave_backup_timeout > 0) { + n_attempts = opt_safe_slave_backup_timeout / sleep_time; + } + + open_temp_tables = get_open_temp_tables(connection); + msg("Slave open temp tables: %d", open_temp_tables); + + while (open_temp_tables && n_attempts--) { + msg("Starting slave SQL thread, waiting %d seconds, then " + "checking Slave_open_temp_tables again (%d attempts " + "remaining)...", sleep_time, n_attempts); + + xb_mysql_query(connection, "START SLAVE SQL_THREAD", false); + std::this_thread::sleep_for(std::chrono::seconds(sleep_time)); + xb_mysql_query(connection, "STOP SLAVE SQL_THREAD", false); + + open_temp_tables = get_open_temp_tables(connection); + msg("Slave open temp tables: %d", open_temp_tables); + } + + /* Restart the slave if it was running at start */ + if (open_temp_tables == 0) { + msg("Slave is safe to backup"); + goto cleanup; + } + + result = false; + + if (sql_thread_started) { + msg("Restarting slave SQL thread."); + xb_mysql_query(connection, "START SLAVE SQL_THREAD", false); + } + + msg("Slave_open_temp_tables did not become zero after " + "%d seconds", opt_safe_slave_backup_timeout); + +cleanup: + free_mysql_variables(status); + + return(result); +} + + +class Var +{ + const char *m_name; + char *m_value; + /* + Disable copying constructors for safety, as the default binary copying + which would be wrong. If we ever want them, the m_value + member should be copied using an strdup()-alike function. + */ + Var(const Var &); // Disabled + Var(Var &); // Disabled +public: + ~Var() + { + free(m_value); + } + Var(const char *name) + :m_name(name), + m_value(NULL) + { } + // Init using a SHOW VARIABLES LIKE 'name' query + Var(const char *name, MYSQL *mysql) + :m_name(name) + { + char buf[128]; + my_snprintf(buf, sizeof(buf), "SHOW VARIABLES LIKE '%s'", m_name); + m_value= read_mysql_one_value(mysql, buf, 1/*offset*/, 2/*total columns*/); + } + /* + Init by name from a result set. + If the variable name is not found in the result set metadata field names, + it's value stays untouched. + */ + bool init(MYSQL_RES *mysql_result, MYSQL_ROW row) + { + MYSQL_FIELD *field= mysql_fetch_fields(mysql_result); + for (uint i= 0; i < mysql_num_fields(mysql_result); i++) + { + if (!strcmp(field[i].name, m_name)) + { + free(m_value); // In case it was initialized earlier + m_value= row[i] ? strdup(row[i]) : NULL; + return false; + } + } + return true; + } + void replace(char from, char to) + { + ut_ad(m_value); + for (char *ptr= strchr(m_value, from); ptr; ptr= strchr(ptr, from)) + *ptr= to; + } + + const char *value() const { return m_value; } + bool eq_value(const char *str, size_t length) const + { + return m_value && !strncmp(m_value, str, length) && m_value[length] == '\0'; + } + bool is_null_or_empty() const { return !m_value || !m_value[0]; } + bool print(String *to) const + { + ut_ad(m_value); + return to->append(m_value, strlen(m_value)); + } + bool print_quoted(String *to) const + { + ut_ad(m_value); + return to->append('\'') || to->append(m_value, strlen(m_value)) || + to->append('\''); + } + bool print_set_global(String *to) const + { + ut_ad(m_value); + return + to->append(STRING_WITH_LEN("SET GLOBAL ")) || + to->append(m_name, strlen(m_name)) || + to->append(STRING_WITH_LEN(" = '")) || + to->append(m_value, strlen(m_value)) || + to->append(STRING_WITH_LEN("';\n")); + } +}; + + +class Show_slave_status +{ + Var m_mariadb_connection_name; // MariaDB: e.g. 'master1' + Var m_master; // e.g. 'localhost' + Var m_filename; // e.g. 'source-bin.000002' + Var m_position; // a number + Var m_mysql_gtid_executed; // MySQL56: e.g. single '<UUID>:1-5" or multiline + // '<UUID1>:1-10,\n<UUID2>:1-20\n<UUID3>:1-30' + Var m_mariadb_using_gtid; // MariaDB: 'No','Slave_Pos','Current_Pos' + +public: + + Show_slave_status() + :m_mariadb_connection_name("Connection_name"), + m_master("Master_Host"), + m_filename("Relay_Master_Log_File"), + m_position("Exec_Master_Log_Pos"), + m_mysql_gtid_executed("Executed_Gtid_Set"), + m_mariadb_using_gtid("Using_Gtid") + { } + + void init(MYSQL_RES *res, MYSQL_ROW row) + { + m_mariadb_connection_name.init(res, row); + m_master.init(res, row); + m_filename.init(res, row); + m_position.init(res, row); + m_mysql_gtid_executed.init(res, row); + m_mariadb_using_gtid.init(res, row); + // Normalize + if (m_mysql_gtid_executed.value()) + m_mysql_gtid_executed.replace('\n', ' '); + } + + static void msg_is_not_slave() + { + msg("Failed to get master binlog coordinates " + "from SHOW SLAVE STATUS.This means that the server is not a " + "replication slave. Ignoring the --slave-info option"); + } + + bool is_mariadb_using_gtid() const + { + return !m_mariadb_using_gtid.eq_value("No", 2); + } + + static bool start_comment_chunk(String *to) + { + return to->length() ? to->append(STRING_WITH_LEN("; ")) : false; + } + + bool print_connection_name_if_set(String *to) const + { + if (!m_mariadb_connection_name.is_null_or_empty()) + return m_mariadb_connection_name.print_quoted(to) || to->append(' '); + return false; + } + + bool print_comment_master_identity(String *comment) const + { + if (comment->append(STRING_WITH_LEN("master "))) + return true; + if (!m_mariadb_connection_name.is_null_or_empty()) + return m_mariadb_connection_name.print_quoted(comment); + return comment->append(STRING_WITH_LEN("''")); // Default not named master + } + + bool print_using_master_log_pos(String *sql, String *comment) const + { + return + sql->append(STRING_WITH_LEN("CHANGE MASTER ")) || + print_connection_name_if_set(sql) || + sql->append(STRING_WITH_LEN("TO MASTER_LOG_FILE=")) || + m_filename.print_quoted(sql) || + sql->append(STRING_WITH_LEN(", MASTER_LOG_POS=")) || + m_position.print(sql) || + sql->append(STRING_WITH_LEN(";\n")) || + print_comment_master_identity(comment) || + comment->append(STRING_WITH_LEN(" filename ")) || + m_filename.print_quoted(comment) || + comment->append(STRING_WITH_LEN(" position ")) || + m_position.print_quoted(comment); + } + + bool print_mysql56(String *sql, String *comment) const + { + /* + SET @@GLOBAL.gtid_purged = '2174B383-5441-11E8-B90A-C80AA9429562:1-1029, ' + '224DA167-0C0C-11E8-8442-00059A3C7B00:1-2695'; + CHANGE MASTER TO MASTER_AUTO_POSITION=1; + */ + return + sql->append(STRING_WITH_LEN("SET GLOBAL gtid_purged=")) || + m_mysql_gtid_executed.print_quoted(sql) || + sql->append(STRING_WITH_LEN(";\n")) || + sql->append(STRING_WITH_LEN("CHANGE MASTER TO MASTER_AUTO_POSITION=1;\n")) || + print_comment_master_identity(comment) || + comment->append(STRING_WITH_LEN(" purge list ")) || + m_mysql_gtid_executed.print_quoted(comment); + } + + bool print_mariadb10_using_gtid(String *sql, String *comment) const + { + return + sql->append(STRING_WITH_LEN("CHANGE MASTER ")) || + print_connection_name_if_set(sql) || + sql->append(STRING_WITH_LEN("TO master_use_gtid = slave_pos;\n")) || + print_comment_master_identity(comment) || + comment->append(STRING_WITH_LEN(" master_use_gtid = slave_pos")); + } + + bool print(String *sql, String *comment, const Var >id_slave_pos) const + { + if (!m_mysql_gtid_executed.is_null_or_empty()) + { + /* MySQL >= 5.6 with GTID enabled */ + return print_mysql56(sql, comment); + } + + if (!gtid_slave_pos.is_null_or_empty() && is_mariadb_using_gtid()) + { + /* MariaDB >= 10.0 with GTID enabled */ + return print_mariadb10_using_gtid(sql, comment); + } + + return print_using_master_log_pos(sql, comment); + } + + /* + Get master info into strings "sql" and "comment" from a MYSQL_RES. + @return false on success + @return true on error + */ + static bool get_slave_info(MYSQL_RES *show_slave_info_result, + const Var >id_slave_pos, + String *sql, String *comment) + { + if (!gtid_slave_pos.is_null_or_empty()) + { + // Print gtid_slave_pos if any of the masters really needs it. + while (MYSQL_ROW row= mysql_fetch_row(show_slave_info_result)) + { + Show_slave_status status; + status.init(show_slave_info_result, row); + if (status.is_mariadb_using_gtid()) + { + if (gtid_slave_pos.print_set_global(sql) || + comment->append(STRING_WITH_LEN("gtid_slave_pos ")) || + gtid_slave_pos.print_quoted(comment)) + return true; // Error + break; + } + } + } + + // Print the list of masters + mysql_data_seek(show_slave_info_result, 0); + while (MYSQL_ROW row= mysql_fetch_row(show_slave_info_result)) + { + Show_slave_status status; + status.init(show_slave_info_result, row); + if (start_comment_chunk(comment) || + status.print(sql, comment, gtid_slave_pos)) + return true; // Error + } + return false; // Success + } + + /* + Get master info into strings "sql" and "comment". + @return false on success + @return true on error + */ + static bool get_slave_info(MYSQL *mysql, bool show_all_slave_status, + String *sql, String *comment) + { + bool rc= false; // Success + // gtid_slave_pos - MariaDB variable : e.g. "0-1-1" or "1-10-100,2-20-500" + Var gtid_slave_pos("gtid_slave_pos", mysql); + const char *query= show_all_slave_status ? "SHOW ALL SLAVES STATUS" : + "SHOW SLAVE STATUS"; + MYSQL_RES *mysql_result= xb_mysql_query(mysql, query, true); + if (!mysql_num_rows(mysql_result)) + { + msg_is_not_slave(); + // Don't change rc, we still want to continue the backup + } + else + { + rc= get_slave_info(mysql_result, gtid_slave_pos, sql, comment); + } + mysql_free_result(mysql_result); + return rc; + } +}; + + + +/*********************************************************************//** +Retrieves MySQL binlog position of the master server in a replication +setup and saves it in a file. It also saves it in mysql_slave_position +variable. +@returns false on error +@returns true on success +*/ +bool +write_slave_info(ds_ctxt *datasink, MYSQL *connection) +{ + String sql, comment; + + if (Show_slave_status::get_slave_info(connection, true, &sql, &comment)) + return false; // Error + + if (!sql.length()) + { + /* + SHOW [ALL] SLAVE STATUS returned no rows. + Don't create the file, but return success to continue the backup. + */ + return true; // Success + } + + mysql_slave_position= strdup(comment.c_ptr()); + return datasink->backup_file_print_buf(XTRABACKUP_SLAVE_INFO, + sql.ptr(), sql.length()); +} + + +/*********************************************************************//** +Retrieves MySQL Galera and +saves it in a file. It also prints it to stdout. */ +bool +write_galera_info(ds_ctxt *datasink, MYSQL *connection) +{ + char *state_uuid = NULL, *state_uuid55 = NULL; + char *last_committed = NULL, *last_committed55 = NULL; + bool result; + + mysql_variable status[] = { + {"Wsrep_local_state_uuid", &state_uuid}, + {"wsrep_local_state_uuid", &state_uuid55}, + {"Wsrep_last_committed", &last_committed}, + {"wsrep_last_committed", &last_committed55}, + {NULL, NULL} + }; + + /* When backup locks are supported by the server, we should skip + creating xtrabackup_galera_info file on the backup stage, because + wsrep_local_state_uuid and wsrep_last_committed will be inconsistent + without blocking commits. The state file will be created on the prepare + stage using the WSREP recovery procedure. */ + if (have_backup_locks) { + return(true); + } + + read_mysql_variables(connection, "SHOW STATUS", status, true); + + if ((state_uuid == NULL && state_uuid55 == NULL) + || (last_committed == NULL && last_committed55 == NULL)) { + msg("Warning: failed to get master wsrep state from SHOW STATUS."); + result = true; + goto cleanup; + } + + result = datasink->backup_file_printf(XTRABACKUP_GALERA_INFO, + "%s:%s\n", state_uuid ? state_uuid : state_uuid55, + last_committed ? last_committed : last_committed55); + if (result) + { + write_current_binlog_file(datasink, connection); + } + +cleanup: + free_mysql_variables(status); + + return(result); +} + + +/*********************************************************************//** +Flush and copy the current binary log file into the backup, +if GTID is enabled */ +bool +write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection) +{ + char *executed_gtid_set = NULL; + char *gtid_binlog_state = NULL; + char *log_bin_file = NULL; + char *log_bin_dir = NULL; + bool gtid_exists; + bool result = true; + char filepath[FN_REFLEN]; + + mysql_variable status[] = { + {"Executed_Gtid_Set", &executed_gtid_set}, + {NULL, NULL} + }; + + mysql_variable status_after_flush[] = { + {"File", &log_bin_file}, + {NULL, NULL} + }; + + mysql_variable vars[] = { + {"gtid_binlog_state", >id_binlog_state}, + {"log_bin_basename", &log_bin_dir}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + read_mysql_variables(connection, "SHOW VARIABLES", vars, true); + + gtid_exists = (executed_gtid_set && *executed_gtid_set) + || (gtid_binlog_state && *gtid_binlog_state); + + if (gtid_exists) { + size_t log_bin_dir_length; + + lock_binlog_maybe(connection); + + xb_mysql_query(connection, "FLUSH BINARY LOGS", false); + + read_mysql_variables(connection, "SHOW MASTER STATUS", + status_after_flush, false); + + if (opt_log_bin != NULL && strchr(opt_log_bin, FN_LIBCHAR)) { + /* If log_bin is set, it has priority */ + if (log_bin_dir) { + free(log_bin_dir); + } + log_bin_dir = strdup(opt_log_bin); + } else if (log_bin_dir == NULL) { + /* Default location is MySQL datadir */ + log_bin_dir = strdup("./"); + } + + dirname_part(log_bin_dir, log_bin_dir, &log_bin_dir_length); + + /* strip final slash if it is not the only path component */ + if (log_bin_dir_length > 1 && + log_bin_dir[log_bin_dir_length - 1] == FN_LIBCHAR) { + log_bin_dir[log_bin_dir_length - 1] = 0; + } + + if (log_bin_dir == NULL || log_bin_file == NULL) { + msg("Failed to get master binlog coordinates from " + "SHOW MASTER STATUS"); + result = false; + goto cleanup; + } + + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, log_bin_file); + result = datasink->copy_file(filepath, log_bin_file, 0); + } + +cleanup: + free_mysql_variables(status_after_flush); + free_mysql_variables(status); + free_mysql_variables(vars); + + return(result); +} + + +/*********************************************************************//** +Retrieves MySQL binlog position and +saves it in a file. It also prints it to stdout. */ +bool +write_binlog_info(ds_ctxt *datasink, MYSQL *connection) +{ + char *filename = NULL; + char *position = NULL; + char *gtid_mode = NULL; + char *gtid_current_pos = NULL; + char *gtid_executed = NULL; + char *gtid = NULL; + bool result; + bool mysql_gtid; + bool mariadb_gtid; + + mysql_variable status[] = { + {"File", &filename}, + {"Position", &position}, + {"Executed_Gtid_Set", >id_executed}, + {NULL, NULL} + }; + + mysql_variable vars[] = { + {"gtid_mode", >id_mode}, + {"gtid_current_pos", >id_current_pos}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + read_mysql_variables(connection, "SHOW VARIABLES", vars, true); + + if (filename == NULL || position == NULL) { + /* Do not create xtrabackup_binlog_info if binary + log is disabled */ + result = true; + goto cleanup; + } + + mysql_gtid = ((gtid_mode != NULL) && (strcmp(gtid_mode, "ON") == 0)); + mariadb_gtid = (gtid_current_pos != NULL); + + gtid = (gtid_executed != NULL ? gtid_executed : gtid_current_pos); + + if (mariadb_gtid || mysql_gtid) { + ut_a(asprintf(&mysql_binlog_position, + "filename '%s', position '%s', " + "GTID of the last change '%s'", + filename, position, gtid) != -1); + result = datasink->backup_file_printf(XTRABACKUP_BINLOG_INFO, + "%s\t%s\t%s\n", filename, position, + gtid); + } else { + ut_a(asprintf(&mysql_binlog_position, + "filename '%s', position '%s'", + filename, position) != -1); + result = datasink->backup_file_printf(XTRABACKUP_BINLOG_INFO, + "%s\t%s\n", filename, position); + } + +cleanup: + free_mysql_variables(status); + free_mysql_variables(vars); + + return(result); +} + +struct escape_and_quote +{ + escape_and_quote(MYSQL *mysql, const char *str) + : mysql(mysql), str(str) {} + MYSQL * const mysql; + const char * const str; +}; + +static +std::ostream& +operator<<(std::ostream& s, const escape_and_quote& eq) +{ + if (!eq.str) + return s << "NULL"; + s << '\''; + size_t len = strlen(eq.str); + char* escaped = (char *)alloca(2 * len + 1); + len = mysql_real_escape_string(eq.mysql, escaped, eq.str, (ulong)len); + s << std::string(escaped, len); + s << '\''; + return s; +} + +/*********************************************************************//** +Writes xtrabackup_info file and if backup_history is enable creates +mysql.mariabackup_history and writes a new history record to the +table containing all the history info particular to the just completed +backup. */ +bool +write_xtrabackup_info(ds_ctxt *datasink, + MYSQL *connection, const char * filename, bool history, + bool stream) +{ + + bool result = true; + FILE *fp = NULL; + char *uuid = NULL; + char *server_version = NULL; + char buf_start_time[100]; + char buf_end_time[100]; + tm tm; + std::ostringstream oss; + const char *xb_stream_name[] = {"file", "tar", "xbstream"}; + + uuid = read_mysql_one_value(connection, "SELECT UUID()"); + server_version = read_mysql_one_value(connection, "SELECT VERSION()"); + localtime_r(&history_start_time, &tm); + strftime(buf_start_time, sizeof(buf_start_time), + "%Y-%m-%d %H:%M:%S", &tm); + history_end_time = time(NULL); + localtime_r(&history_end_time, &tm); + strftime(buf_end_time, sizeof(buf_end_time), + "%Y-%m-%d %H:%M:%S", &tm); + bool is_partial = (xtrabackup_tables + || xtrabackup_tables_file + || xtrabackup_databases + || xtrabackup_databases_file + || xtrabackup_tables_exclude + || xtrabackup_databases_exclude + ); + + char *buf = NULL; + int buf_len = asprintf(&buf, + "uuid = %s\n" + "name = %s\n" + "tool_name = %s\n" + "tool_command = %s\n" + "tool_version = %s\n" + "ibbackup_version = %s\n" + "server_version = %s\n" + "start_time = %s\n" + "end_time = %s\n" + "lock_time = %d\n" + "binlog_pos = %s\n" + "innodb_from_lsn = " LSN_PF "\n" + "innodb_to_lsn = " LSN_PF "\n" + "partial = %s\n" + "incremental = %s\n" + "format = %s\n" + "compressed = %s\n", + uuid, /* uuid */ + opt_history ? opt_history : "", /* name */ + tool_name, /* tool_name */ + tool_args, /* tool_command */ + MYSQL_SERVER_VERSION, /* tool_version */ + MYSQL_SERVER_VERSION, /* ibbackup_version */ + server_version, /* server_version */ + buf_start_time, /* start_time */ + buf_end_time, /* end_time */ + (int)history_lock_time, /* lock_time */ + mysql_binlog_position ? + mysql_binlog_position : "", /* binlog_pos */ + incremental_lsn, + /* innodb_from_lsn */ + metadata_to_lsn, + /* innodb_to_lsn */ + is_partial? "Y" : "N", + xtrabackup_incremental ? "Y" : "N", /* incremental */ + xb_stream_name[xtrabackup_stream_fmt], /* format */ + xtrabackup_compress ? "compressed" : "N"); /* compressed */ + if (buf_len < 0) { + msg("Error: cannot generate xtrabackup_info"); + result = false; + goto cleanup; + } + + if (stream) { + datasink->backup_file_printf(filename, "%s", buf); + } else { + fp = fopen(filename, "w"); + if (!fp) { + msg("Error: cannot open %s", filename); + result = false; + goto cleanup; + } + if (fwrite(buf, buf_len, 1, fp) < 1) { + result = false; + goto cleanup; + } + } + + if (!history) { + goto cleanup; + } + + xb_mysql_query(connection, + "CREATE TABLE IF NOT EXISTS " XB_HISTORY_TABLE "(" + "uuid VARCHAR(40) NOT NULL PRIMARY KEY," + "name VARCHAR(255) DEFAULT NULL," + "tool_name VARCHAR(255) DEFAULT NULL," + "tool_command TEXT DEFAULT NULL," + "tool_version VARCHAR(255) DEFAULT NULL," + "ibbackup_version VARCHAR(255) DEFAULT NULL," + "server_version VARCHAR(255) DEFAULT NULL," + "start_time TIMESTAMP NULL DEFAULT NULL," + "end_time TIMESTAMP NULL DEFAULT NULL," + "lock_time BIGINT UNSIGNED DEFAULT NULL," + "binlog_pos VARCHAR(128) DEFAULT NULL," + "innodb_from_lsn BIGINT UNSIGNED DEFAULT NULL," + "innodb_to_lsn BIGINT UNSIGNED DEFAULT NULL," + "partial ENUM('Y', 'N') DEFAULT NULL," + "incremental ENUM('Y', 'N') DEFAULT NULL," + "format ENUM('file', 'tar', 'xbstream') DEFAULT NULL," + "compressed ENUM('Y', 'N') DEFAULT NULL" + ") CHARACTER SET utf8 ENGINE=innodb", false); + + +#define ESCAPE_BOOL(expr) ((expr)?"'Y'":"'N'") + + oss << "insert into " XB_HISTORY_TABLE "(" + << "uuid, name, tool_name, tool_command, tool_version," + << "ibbackup_version, server_version, start_time, end_time," + << "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn," + << "partial, incremental, format, compressed) " + << "values(" + << escape_and_quote(connection, uuid) << "," + << escape_and_quote(connection, opt_history) << "," + << escape_and_quote(connection, tool_name) << "," + << escape_and_quote(connection, tool_args) << "," + << escape_and_quote(connection, MYSQL_SERVER_VERSION) << "," + << escape_and_quote(connection, MYSQL_SERVER_VERSION) << "," + << escape_and_quote(connection, server_version) << "," + << "from_unixtime(" << history_start_time << ")," + << "from_unixtime(" << history_end_time << ")," + << history_lock_time << "," + << escape_and_quote(connection, mysql_binlog_position) << "," + << incremental_lsn << "," + << metadata_to_lsn << "," + << ESCAPE_BOOL(is_partial) << "," + << ESCAPE_BOOL(xtrabackup_incremental)<< "," + << escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<"," + << ESCAPE_BOOL(xtrabackup_compress) << ")"; + + xb_mysql_query(mysql_connection, oss.str().c_str(), false); + +cleanup: + + free(uuid); + free(server_version); + free(buf); + if (fp) + fclose(fp); + + return(result); +} + +extern const char *innodb_checksum_algorithm_names[]; + +#ifdef _WIN32 +#include <algorithm> +#endif + +static std::string make_local_paths(const char *data_file_path) +{ + if (strchr(data_file_path, '/') == 0 +#ifdef _WIN32 + && strchr(data_file_path, '\\') == 0 +#endif + ){ + return std::string(data_file_path); + } + + std::ostringstream buf; + + char *dup = strdup(innobase_data_file_path); + ut_a(dup); + char *p; + char * token = strtok_r(dup, ";", &p); + while (token) { + if (buf.tellp()) + buf << ";"; + + char *fname = strrchr(token, '/'); +#ifdef _WIN32 + fname = std::max(fname,strrchr(token, '\\')); +#endif + if (fname) + buf << fname + 1; + else + buf << token; + token = strtok_r(NULL, ";", &p); + } + free(dup); + return buf.str(); +} + +bool write_backup_config_file(ds_ctxt *datasink) +{ + int rc= datasink->backup_file_printf("backup-my.cnf", + "# This options file was generated by innobackupex.\n\n" + "# The server\n" + "[mysqld]\n" + "innodb_checksum_algorithm=%s\n" + "innodb_data_file_path=%s\n" + "innodb_log_file_size=%llu\n" + "innodb_page_size=%lu\n" + "innodb_undo_directory=%s\n" + "innodb_undo_tablespaces=%u\n" + "innodb_compression_level=%u\n" + "%s%s\n" + "%s\n", + innodb_checksum_algorithm_names[srv_checksum_algorithm], + make_local_paths(innobase_data_file_path).c_str(), + srv_log_file_size, + srv_page_size, + srv_undo_dir, + srv_undo_tablespaces, + page_zip_level, + innobase_buffer_pool_filename ? + "innodb_buffer_pool_filename=" : "", + innobase_buffer_pool_filename ? + innobase_buffer_pool_filename : "", + xb_plugin_get_config()); + return rc; +} + + +static +char *make_argv(char *buf, size_t len, int argc, char **argv) +{ + size_t left= len; + const char *arg; + + buf[0]= 0; + ++argv; --argc; + while (argc > 0 && left > 0) + { + arg = *argv; + if (strncmp(*argv, "--password", strlen("--password")) == 0) { + arg = "--password=..."; + } + left-= snprintf(buf + len - left, left, + "%s%c", arg, argc > 1 ? ' ' : 0); + ++argv; --argc; + } + + return buf; +} + +void +capture_tool_command(int argc, char **argv) +{ + /* capture tool name tool args */ + tool_name = strrchr(argv[0], '/'); + tool_name = tool_name ? tool_name + 1 : argv[0]; + + make_argv(tool_args, sizeof(tool_args), argc, argv); +} + + +bool +select_history() +{ + if (opt_incremental_history_name || opt_incremental_history_uuid) { + if (!select_incremental_lsn_from_history( + &incremental_lsn)) { + return(false); + } + } + return(true); +} + +bool +flush_changed_page_bitmaps() +{ + if (xtrabackup_incremental && have_changed_page_bitmaps && + !xtrabackup_incremental_force_scan) { + xb_mysql_query(mysql_connection, + "FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS", false); + } + return(true); +} + + +/*********************************************************************//** +Deallocate memory, disconnect from server, etc. +@return true on success. */ +void +backup_cleanup() +{ + free(mysql_slave_position); + free(mysql_binlog_position); + free(buffer_pool_filename); + + if (mysql_connection) { + mysql_close(mysql_connection); + } +} + + +static MYSQL *mdl_con = NULL; + +std::map<ulint, std::string> spaceid_to_tablename; + +void +mdl_lock_init() +{ + mdl_con = xb_mysql_connect(); + if (!mdl_con) + { + msg("FATAL: cannot create connection for MDL locks"); + exit(1); + } + const char *query = + "SELECT NAME, SPACE FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE '%%/%%'"; + + MYSQL_RES *mysql_result = xb_mysql_query(mdl_con, query, true, true); + while (MYSQL_ROW row = mysql_fetch_row(mysql_result)) { + int err; + ulint id = (ulint)my_strtoll10(row[1], 0, &err); + spaceid_to_tablename[id] = ut_get_name(0, row[0]); + } + mysql_free_result(mysql_result); + + xb_mysql_query(mdl_con, "BEGIN", false, true); +} + +void +mdl_lock_table(ulint space_id) +{ + if (space_id == 0) + return; + + std::string full_table_name = spaceid_to_tablename[space_id]; + + DBUG_EXECUTE_IF("rename_during_mdl_lock_table", + if (full_table_name == "`test`.`t1`") + xb_mysql_query(mysql_connection, "RENAME TABLE test.t1 to test.t2", false, true); + ); + + std::ostringstream lock_query; + lock_query << "SELECT 1 FROM " << full_table_name << " LIMIT 0"; + msg("Locking MDL for %s", full_table_name.c_str()); + if (mysql_query(mdl_con, lock_query.str().c_str())) { + msg("Warning : locking MDL failed for space id %zu, name %s", space_id, full_table_name.c_str()); + } else { + MYSQL_RES *r = mysql_store_result(mdl_con); + mysql_free_result(r); + } +} + +void +mdl_unlock_all() +{ + msg("Unlocking MDL for all tables"); + xb_mysql_query(mdl_con, "COMMIT", false, true); + mysql_close(mdl_con); + spaceid_to_tablename.clear(); +} diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h new file mode 100644 index 00000000..4b08da0b --- /dev/null +++ b/extra/mariabackup/backup_mysql.h @@ -0,0 +1,90 @@ +#ifndef XTRABACKUP_BACKUP_MYSQL_H +#define XTRABACKUP_BACKUP_MYSQL_H + +#include <mysql.h> + +/* MariaDB version */ +extern ulong mysql_server_version; + +/* server capabilities */ +extern bool have_changed_page_bitmaps; +extern bool have_backup_locks; +extern bool have_lock_wait_timeout; +extern bool have_galera_enabled; +extern bool have_multi_threaded_slave; +extern bool have_gtid_slave; + + +/* History on server */ +extern time_t history_start_time; +extern time_t history_end_time; +extern time_t history_lock_time; + + +extern bool sql_thread_started; +extern char *mysql_slave_position; +extern char *mysql_binlog_position; +extern char *buffer_pool_filename; + +/** connection to mysql server */ +extern MYSQL *mysql_connection; + +void +capture_tool_command(int argc, char **argv); + +bool +select_history(); + +bool +flush_changed_page_bitmaps(); + +void +backup_cleanup(); + +bool +get_mysql_vars(MYSQL *connection); + +bool +detect_mysql_capabilities_for_backup(); + +MYSQL * +xb_mysql_connect(); + +MYSQL_RES * +xb_mysql_query(MYSQL *connection, const char *query, bool use_result, + bool die_on_error = true); + +void +unlock_all(MYSQL *connection); + +bool +write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection); + +bool +write_binlog_info(ds_ctxt *datasink, MYSQL *connection); + +bool +write_xtrabackup_info(ds_ctxt *datasink, + MYSQL *connection, const char * filename, bool history, + bool stream); + +bool +write_backup_config_file(ds_ctxt *datasink); + +bool +lock_binlog_maybe(MYSQL *connection); + +bool +lock_tables(MYSQL *connection); + +bool +wait_for_safe_slave(MYSQL *connection); + +bool +write_galera_info(ds_ctxt *datasink, MYSQL *connection); + +bool +write_slave_info(ds_ctxt *datasink, MYSQL *connection); + + +#endif diff --git a/extra/mariabackup/backup_wsrep.h b/extra/mariabackup/backup_wsrep.h new file mode 100644 index 00000000..50a8a3a5 --- /dev/null +++ b/extra/mariabackup/backup_wsrep.h @@ -0,0 +1,32 @@ +/****************************************************** +Percona XtraBackup: hot backup tool for InnoDB +(c) 2009-2014 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*******************************************************/ + +#ifndef BACKUP_WSREP_H +#define BACKUP_WSREP_H + +/*********************************************************************** +Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that +information is present in the trx system header. Otherwise, do nothing. */ +void +xb_write_galera_info(bool incremental_prepare); +/*==================*/ + +#endif diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc new file mode 100644 index 00000000..39a07a25 --- /dev/null +++ b/extra/mariabackup/changed_page_bitmap.cc @@ -0,0 +1,1040 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Changed page bitmap implementation */ + +#include "changed_page_bitmap.h" + +#include "common.h" +#include "xtrabackup.h" +#include "srv0srv.h" + +/* TODO: copy-pasted shared definitions from the XtraDB bitmap write code. +Remove these on the first opportunity, i.e. single-binary XtraBackup. */ + +/* log0online.h */ + +/** Single bitmap file information */ +struct log_online_bitmap_file_t { + char name[FN_REFLEN]; /*!< Name with full path */ + pfs_os_file_t file; /*!< Handle to opened file */ + ib_uint64_t size; /*!< Size of the file */ + ib_uint64_t offset; /*!< Offset of the next read, + or count of already-read bytes + */ +}; + +/** A set of bitmap files containing some LSN range */ +struct log_online_bitmap_file_range_t { + size_t count; /*!< Number of files */ + /*!< Dynamically-allocated array of info about individual files */ + struct files_t { + char name[FN_REFLEN];/*!< Name of a file */ + lsn_t start_lsn; /*!< Starting LSN of data in this + file */ + ulong seq_num; /*!< Sequence number of this file */ + } *files; +}; + +/* log0online.c */ + +/** File name stem for bitmap files. */ +static const char* bmp_file_name_stem = "ib_modified_log_"; + +/** The bitmap file block size in bytes. All writes will be multiples of this. + */ +enum { + MODIFIED_PAGE_BLOCK_SIZE = 4096 +}; + +/** Offsets in a file bitmap block */ +enum { + MODIFIED_PAGE_IS_LAST_BLOCK = 0,/* 1 if last block in the current + write, 0 otherwise. */ + MODIFIED_PAGE_START_LSN = 4, /* The starting tracked LSN of this and + other blocks in the same write */ + MODIFIED_PAGE_END_LSN = 12, /* The ending tracked LSN of this and + other blocks in the same write */ + MODIFIED_PAGE_SPACE_ID = 20, /* The space ID of tracked pages in + this block */ + MODIFIED_PAGE_1ST_PAGE_ID = 24, /* The page ID of the first tracked + page in this block */ + MODIFIED_PAGE_BLOCK_UNUSED_1 = 28,/* Unused in order to align the start + of bitmap at 8 byte boundary */ + MODIFIED_PAGE_BLOCK_BITMAP = 32,/* Start of the bitmap itself */ + MODIFIED_PAGE_BLOCK_UNUSED_2 = MODIFIED_PAGE_BLOCK_SIZE - 8, + /* Unused in order to align the end of + bitmap at 8 byte boundary */ + MODIFIED_PAGE_BLOCK_CHECKSUM = MODIFIED_PAGE_BLOCK_SIZE - 4 + /* The checksum of the current block */ +}; + +/** Length of the bitmap data in a block */ +enum { MODIFIED_PAGE_BLOCK_BITMAP_LEN + = MODIFIED_PAGE_BLOCK_UNUSED_2 - MODIFIED_PAGE_BLOCK_BITMAP }; + +/** Length of the bitmap data in a block in page ids */ +enum { MODIFIED_PAGE_BLOCK_ID_COUNT = MODIFIED_PAGE_BLOCK_BITMAP_LEN * 8 }; + +typedef ib_uint64_t bitmap_word_t; + +/****************************************************************//** +Calculate a bitmap block checksum. Algorithm borrowed from +log_block_calc_checksum. +@return checksum */ +UNIV_INLINE +ulint +log_online_calc_checksum( +/*=====================*/ + const byte* block); /*!<in: bitmap block */ + +/****************************************************************//** +Provide a comparisson function for the RB-tree tree (space, +block_start_page) pairs. Actual implementation does not matter as +long as the ordering is full. +@return -1 if p1 < p2, 0 if p1 == p2, 1 if p1 > p2 +*/ +static +int +log_online_compare_bmp_keys( +/*========================*/ + const void* p1, /*!<in: 1st key to compare */ + const void* p2) /*!<in: 2nd key to compare */ +{ + const byte *k1 = (const byte *)p1; + const byte *k2 = (const byte *)p2; + + ulint k1_space = mach_read_from_4(k1 + MODIFIED_PAGE_SPACE_ID); + ulint k2_space = mach_read_from_4(k2 + MODIFIED_PAGE_SPACE_ID); + if (k1_space == k2_space) { + + ulint k1_start_page + = mach_read_from_4(k1 + MODIFIED_PAGE_1ST_PAGE_ID); + ulint k2_start_page + = mach_read_from_4(k2 + MODIFIED_PAGE_1ST_PAGE_ID); + return k1_start_page < k2_start_page + ? -1 : k1_start_page > k2_start_page ? 1 : 0; + } + return k1_space < k2_space ? -1 : 1; +} + +/****************************************************************//** +Calculate a bitmap block checksum. Algorithm borrowed from +log_block_calc_checksum. +@return checksum */ +UNIV_INLINE +ulint +log_online_calc_checksum( +/*=====================*/ + const byte* block) /*!<in: bitmap block */ +{ + ulint sum; + ulint sh; + ulint i; + + sum = 1; + sh = 0; + + for (i = 0; i < MODIFIED_PAGE_BLOCK_CHECKSUM; i++) { + + ulint b = block[i]; + sum &= 0x7FFFFFFFUL; + sum += b; + sum += b << sh; + sh++; + if (sh > 24) { + + sh = 0; + } + } + + return sum; +} + +/****************************************************************//** +Read one bitmap data page and check it for corruption. + +@return TRUE if page read OK, FALSE if I/O error */ +static +ibool +log_online_read_bitmap_page( +/*========================*/ + log_online_bitmap_file_t *bitmap_file, /*!<in/out: bitmap + file */ + byte *page, /*!<out: read page. Must be at + least MODIFIED_PAGE_BLOCK_SIZE + bytes long */ + ibool *checksum_ok) /*!<out: TRUE if page + checksum OK */ +{ + ulint checksum; + ulint actual_checksum; + + ut_a(bitmap_file->size >= MODIFIED_PAGE_BLOCK_SIZE); + ut_a(bitmap_file->offset + <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE); + ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); + if (DB_SUCCESS != + os_file_read(IORequestRead, bitmap_file->file, page, + bitmap_file->offset, MODIFIED_PAGE_BLOCK_SIZE, + nullptr)) { + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + msg("InnoDB: Warning: failed reading changed page bitmap " + "file \'%s\'", bitmap_file->name); + return FALSE; + } + + bitmap_file->offset += MODIFIED_PAGE_BLOCK_SIZE; + ut_ad(bitmap_file->offset <= bitmap_file->size); + + checksum = mach_read_from_4(page + MODIFIED_PAGE_BLOCK_CHECKSUM); + actual_checksum = log_online_calc_checksum(page); + *checksum_ok = (checksum == actual_checksum); + + return TRUE; +} + +/*********************************************************************//** +Check the name of a given file if it's a changed page bitmap file and +return file sequence and start LSN name components if it is. If is not, +the values of output parameters are undefined. + +@return TRUE if a given file is a changed page bitmap file. */ +static +ibool +log_online_is_bitmap_file( +/*======================*/ + const os_file_stat_t* file_info, /*!<in: file to + check */ + ulong* bitmap_file_seq_num, /*!<out: bitmap file + sequence number */ + lsn_t* bitmap_file_start_lsn) /*!<out: bitmap file + start LSN */ +{ + char stem[FN_REFLEN]; + + ut_ad (strlen(file_info->name) < OS_FILE_MAX_PATH); + + return ((file_info->type == OS_FILE_TYPE_FILE + || file_info->type == OS_FILE_TYPE_LINK) + && (sscanf(file_info->name, "%[a-z_]%lu_" LSN_PF ".xdb", stem, + bitmap_file_seq_num, bitmap_file_start_lsn) == 3) + && (!strcmp(stem, bmp_file_name_stem))); +} + +/*********************************************************************//** +List the bitmap files in srv_data_home and setup their range that contains the +specified LSN interval. This range, if non-empty, will start with a file that +has the greatest LSN equal to or less than the start LSN and will include all +the files up to the one with the greatest LSN less than the end LSN. Caller +must free bitmap_files->files when done if bitmap_files set to non-NULL and +this function returned TRUE. Field bitmap_files->count might be set to a +larger value than the actual count of the files, and space for the unused array +slots will be allocated but cleared to zeroes. + +@return TRUE if succeeded +*/ +static +ibool +log_online_setup_bitmap_file_range( +/*===============================*/ + log_online_bitmap_file_range_t *bitmap_files, /*!<in/out: bitmap file + range */ + lsn_t range_start, /*!<in: start LSN */ + lsn_t range_end) /*!<in: end LSN */ +{ + os_file_dir_t bitmap_dir; + os_file_stat_t bitmap_dir_file_info; + ulong first_file_seq_num = ULONG_MAX; + ulong last_file_seq_num = 0; + lsn_t first_file_start_lsn = LSN_MAX; + + xb_ad(range_end >= range_start); + + bitmap_files->count = 0; + bitmap_files->files = NULL; + + /* 1st pass: size the info array */ + + bitmap_dir = os_file_opendir(srv_data_home); + if (UNIV_UNLIKELY(bitmap_dir == IF_WIN(INVALID_HANDLE_VALUE, NULL))) { + msg("InnoDB: Error: failed to open bitmap directory \'%s\'", + srv_data_home); + return FALSE; + } + + while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, + &bitmap_dir_file_info)) { + + ulong file_seq_num; + lsn_t file_start_lsn; + + if (!log_online_is_bitmap_file(&bitmap_dir_file_info, + &file_seq_num, + &file_start_lsn) + || file_start_lsn >= range_end) { + + continue; + } + + if (file_seq_num > last_file_seq_num) { + + last_file_seq_num = file_seq_num; + } + + if (file_start_lsn >= range_start + || file_start_lsn == first_file_start_lsn + || first_file_start_lsn > range_start) { + + /* A file that falls into the range */ + + if (file_start_lsn < first_file_start_lsn) { + + first_file_start_lsn = file_start_lsn; + } + if (file_seq_num < first_file_seq_num) { + + first_file_seq_num = file_seq_num; + } + } else if (file_start_lsn > first_file_start_lsn) { + + /* A file that has LSN closer to the range start + but smaller than it, replacing another such file */ + first_file_start_lsn = file_start_lsn; + first_file_seq_num = file_seq_num; + } + } + + if (UNIV_UNLIKELY(os_file_closedir_failed(bitmap_dir))) { + os_file_get_last_error(TRUE); + msg("InnoDB: Error: cannot close \'%s\'",srv_data_home); + return FALSE; + } + + if (first_file_seq_num == ULONG_MAX && last_file_seq_num == 0) { + + bitmap_files->count = 0; + return TRUE; + } + + bitmap_files->count = last_file_seq_num - first_file_seq_num + 1; + + /* 2nd pass: get the file names in the file_seq_num order */ + + bitmap_dir = os_file_opendir(srv_data_home); + if (UNIV_UNLIKELY(bitmap_dir == IF_WIN(INVALID_HANDLE_VALUE, NULL))) { + msg("InnoDB: Error: failed to open bitmap directory \'%s\'", + srv_data_home); + return FALSE; + } + + bitmap_files->files = + static_cast<log_online_bitmap_file_range_t::files_t *> + (malloc(bitmap_files->count * sizeof(bitmap_files->files[0]))); + memset(bitmap_files->files, 0, + bitmap_files->count * sizeof(bitmap_files->files[0])); + + while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, + &bitmap_dir_file_info)) { + + ulong file_seq_num; + lsn_t file_start_lsn; + size_t array_pos; + + if (!log_online_is_bitmap_file(&bitmap_dir_file_info, + &file_seq_num, + &file_start_lsn) + || file_start_lsn >= range_end + || file_start_lsn < first_file_start_lsn) { + + continue; + } + + array_pos = file_seq_num - first_file_seq_num; + if (UNIV_UNLIKELY(array_pos >= bitmap_files->count)) { + + msg("InnoDB: Error: inconsistent bitmap file " + "directory"); + os_file_closedir(bitmap_dir); + free(bitmap_files->files); + return FALSE; + } + + if (file_seq_num > bitmap_files->files[array_pos].seq_num) { + + bitmap_files->files[array_pos].seq_num = file_seq_num; + strncpy(bitmap_files->files[array_pos].name, + bitmap_dir_file_info.name, FN_REFLEN - 1); + bitmap_files->files[array_pos].name[FN_REFLEN - 1] + = '\0'; + bitmap_files->files[array_pos].start_lsn + = file_start_lsn; + } + } + + if (UNIV_UNLIKELY(os_file_closedir_failed(bitmap_dir))) { + os_file_get_last_error(TRUE); + msg("InnoDB: Error: cannot close \'%s\'", srv_data_home); + free(bitmap_files->files); + return FALSE; + } + +#ifdef UNIV_DEBUG + ut_ad(bitmap_files->files[0].seq_num == first_file_seq_num); + + for (size_t i = 1; i < bitmap_files->count; i++) { + if (!bitmap_files->files[i].seq_num) { + + break; + } + ut_ad(bitmap_files->files[i].seq_num + > bitmap_files->files[i - 1].seq_num); + ut_ad(bitmap_files->files[i].start_lsn + >= bitmap_files->files[i - 1].start_lsn); + } +#endif + + return TRUE; +} + +/****************************************************************//** +Open a bitmap file for reading. + +@return whether opened successfully */ +static +bool +log_online_open_bitmap_file_read_only( +/*==================================*/ + const char* name, /*!<in: bitmap file + name without directory, + which is assumed to be + srv_data_home */ + log_online_bitmap_file_t* bitmap_file) /*!<out: opened bitmap + file */ +{ + bool success = false; + + xb_ad(name[0] != '\0'); + + snprintf(bitmap_file->name, FN_REFLEN, "%s%s", srv_data_home, name); + bitmap_file->file = os_file_create_simple_no_error_handling( + 0, bitmap_file->name, + OS_FILE_OPEN, OS_FILE_READ_ONLY, true, &success); + if (UNIV_UNLIKELY(!success)) { + + /* Here and below assume that bitmap file names do not + contain apostrophes, thus no need for ut_print_filename(). */ + msg("InnoDB: Warning: error opening the changed page " + "bitmap \'%s\'", bitmap_file->name); + return success; + } + + bitmap_file->size = os_file_get_size(bitmap_file->file); + bitmap_file->offset = 0; + +#ifdef __linux__ + posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_SEQUENTIAL); + posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE); +#endif + + return success; +} + +/****************************************************************//** +Diagnose one or both of the following situations if we read close to +the end of bitmap file: +1) Warn if the remainder of the file is less than one page. +2) Error if we cannot read any more full pages but the last read page +did not have the last-in-run flag set. + +@return FALSE for the error */ +static +ibool +log_online_diagnose_bitmap_eof( +/*===========================*/ + const log_online_bitmap_file_t* bitmap_file, /*!< in: bitmap file */ + ibool last_page_in_run)/*!< in: "last page in + run" flag value in the + last read page */ +{ + /* Check if we are too close to EOF to read a full page */ + if ((bitmap_file->size < MODIFIED_PAGE_BLOCK_SIZE) + || (bitmap_file->offset + > bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { + + if (UNIV_UNLIKELY(bitmap_file->offset != bitmap_file->size)) { + + /* If we are not at EOF and we have less than one page + to read, it's junk. This error is not fatal in + itself. */ + + msg("InnoDB: Warning: junk at the end of changed " + "page bitmap file \'%s\'.", bitmap_file->name); + } + + if (UNIV_UNLIKELY(!last_page_in_run)) { + + /* We are at EOF but the last read page did not finish + a run */ + /* It's a "Warning" here because it's not a fatal error + for the whole server */ + msg("InnoDB: Warning: changed page bitmap " + "file \'%s\' does not contain a complete run " + "at the end.", bitmap_file->name); + return FALSE; + } + } + return TRUE; +} + +/* End of copy-pasted definitions */ + +/** Iterator structure over changed page bitmap */ +struct xb_page_bitmap_range_struct { + const xb_page_bitmap *bitmap; /* Bitmap with data */ + ulint space_id; /* Space id for this + iterator */ + ulint bit_i; /* Bit index of the iterator + position in the current page */ + const ib_rbt_node_t *bitmap_node; /* Current bitmap tree node */ + const byte *bitmap_page; /* Current bitmap page */ + ulint current_page_id;/* Current page id */ +}; + +/****************************************************************//** +Print a diagnostic message on missing bitmap data for an LSN range. */ +static +void +xb_msg_missing_lsn_data( +/*====================*/ + lsn_t missing_interval_start, /*!<in: interval start */ + lsn_t missing_interval_end) /*!<in: interval end */ +{ + msg("mariabackup: warning: changed page data missing for LSNs between " + LSN_PF " and " LSN_PF, missing_interval_start, + missing_interval_end); +} + +/****************************************************************//** +Scan a bitmap file until data for a desired LSN or EOF is found and check that +the page before the starting one is not corrupted to ensure that the found page +indeed contains the very start of the desired LSN data. The caller must check +the page LSN values to determine if the bitmap file was scanned until the data +was found or until EOF. Page must be at least MODIFIED_PAGE_BLOCK_SIZE big. + +@return TRUE if the scan successful without corruption detected +*/ +static +ibool +xb_find_lsn_in_bitmap_file( +/*=======================*/ + log_online_bitmap_file_t *bitmap_file, /*!<in/out: bitmap + file */ + byte *page, /*!<in/out: last read + bitmap page */ + lsn_t *page_end_lsn, /*!<out: end LSN of the + last read page */ + lsn_t lsn) /*!<in: LSN to find */ +{ + ibool last_page_ok = TRUE; + ibool next_to_last_page_ok = TRUE; + + xb_ad (bitmap_file->size >= MODIFIED_PAGE_BLOCK_SIZE); + + *page_end_lsn = 0; + + while ((*page_end_lsn <= lsn) + && (bitmap_file->offset + <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { + + next_to_last_page_ok = last_page_ok; + if (!log_online_read_bitmap_page(bitmap_file, page, + &last_page_ok)) { + + return FALSE; + } + + *page_end_lsn = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); + } + + /* We check two pages here because the last read page already contains + the required LSN data. If the next to the last one page is corrupted, + then we have no way of telling if that page contained the required LSN + range data too */ + return last_page_ok && next_to_last_page_ok; +} + +/****************************************************************//** +Read the disk bitmap and build the changed page bitmap tree for the +LSN interval incremental_lsn to log_sys.next_checkpoint_lsn. + +@return the built bitmap tree or NULL if unable to read the full interval for +any reason. */ +xb_page_bitmap* +xb_page_bitmap_init(void) +/*=====================*/ +{ + log_online_bitmap_file_t bitmap_file; + lsn_t bmp_start_lsn = incremental_lsn; + const lsn_t bmp_end_lsn{log_sys.next_checkpoint_lsn}; + byte page[MODIFIED_PAGE_BLOCK_SIZE]; + lsn_t current_page_end_lsn; + xb_page_bitmap *result; + ibool last_page_in_run= FALSE; + log_online_bitmap_file_range_t bitmap_files; + size_t bmp_i; + ibool last_page_ok = TRUE; + + if (UNIV_UNLIKELY(bmp_start_lsn > bmp_end_lsn)) { + + msg("mariabackup: incremental backup LSN " LSN_PF + " is larger than than the last checkpoint LSN " LSN_PF + , bmp_start_lsn, bmp_end_lsn); + return NULL; + } + + if (!log_online_setup_bitmap_file_range(&bitmap_files, bmp_start_lsn, + bmp_end_lsn)) { + + return NULL; + } + + /* Only accept no bitmap files returned if start LSN == end LSN */ + if (bitmap_files.count == 0 && bmp_end_lsn != bmp_start_lsn) { + + return NULL; + } + + result = rbt_create(MODIFIED_PAGE_BLOCK_SIZE, + log_online_compare_bmp_keys); + + if (bmp_start_lsn == bmp_end_lsn) { + + /* Empty range - empty bitmap */ + return result; + } + + bmp_i = 0; + + if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].start_lsn + > bmp_start_lsn)) { + + /* The 1st file does not have the starting LSN data */ + xb_msg_missing_lsn_data(bmp_start_lsn, + bitmap_files.files[bmp_i].start_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* Skip any zero-sized files at the start */ + while ((bmp_i < bitmap_files.count - 1) + && (bitmap_files.files[bmp_i].start_lsn + == bitmap_files.files[bmp_i + 1].start_lsn)) { + + bmp_i++; + } + + /* Is the 1st bitmap file missing? */ + if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] == '\0')) { + + /* TODO: this is not the exact missing range */ + xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* Open the 1st bitmap file */ + if (UNIV_UNLIKELY(!log_online_open_bitmap_file_read_only( + bitmap_files.files[bmp_i].name, + &bitmap_file))) { + + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* If the 1st file is truncated, no data. Not merged with the case + below because zero-length file indicates not a corruption but missing + subsequent files instead. */ + if (UNIV_UNLIKELY(bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE)) { + + xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + /* Find the start of the required LSN range in the file */ + if (UNIV_UNLIKELY(!xb_find_lsn_in_bitmap_file(&bitmap_file, page, + ¤t_page_end_lsn, + bmp_start_lsn))) { + + msg("mariabackup: Warning: changed page bitmap file " + "\'%s\' corrupted", bitmap_file.name); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + last_page_in_run + = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); + + if (UNIV_UNLIKELY(!log_online_diagnose_bitmap_eof(&bitmap_file, + last_page_in_run))) { + + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + if (UNIV_UNLIKELY(current_page_end_lsn < bmp_start_lsn)) { + + xb_msg_missing_lsn_data(current_page_end_lsn, bmp_start_lsn); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + /* 1st bitmap page found, add it to the tree. */ + rbt_insert(result, page, page); + + /* Read next pages/files until all required data is read */ + while (last_page_ok + && (current_page_end_lsn < bmp_end_lsn + || (current_page_end_lsn == bmp_end_lsn + && !last_page_in_run))) { + + ib_rbt_bound_t tree_search_pos; + + /* If EOF, advance the file skipping over any empty files */ + while (bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE + || (bitmap_file.offset + > bitmap_file.size - MODIFIED_PAGE_BLOCK_SIZE)) { + + os_file_close(bitmap_file.file); + + if (UNIV_UNLIKELY( + !log_online_diagnose_bitmap_eof( + &bitmap_file, last_page_in_run))) { + + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + bmp_i++; + + if (UNIV_UNLIKELY(bmp_i == bitmap_files.count + || (bitmap_files.files[bmp_i].seq_num + == 0))) { + + xb_msg_missing_lsn_data(current_page_end_lsn, + bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* Is the next file missing? */ + if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] + == '\0')) { + + /* TODO: this is not the exact missing range */ + xb_msg_missing_lsn_data(bitmap_files.files + [bmp_i - 1].start_lsn, + bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + if (UNIV_UNLIKELY( + !log_online_open_bitmap_file_read_only( + bitmap_files.files[bmp_i].name, + &bitmap_file))) { + + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + } + + if (UNIV_UNLIKELY( + !log_online_read_bitmap_page(&bitmap_file, page, + &last_page_ok))) { + + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + if (UNIV_UNLIKELY(!last_page_ok)) { + + msg("mariabackup: warning: changed page bitmap file " + "\'%s\' corrupted.", bitmap_file.name); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + /* Merge the current page with an existing page or insert a new + page into the tree */ + + if (!rbt_search(result, &tree_search_pos, page)) { + + /* Merge the bitmap pages */ + byte *existing_page + = rbt_value(byte, tree_search_pos.last); + bitmap_word_t *bmp_word_1 = (bitmap_word_t *) + (existing_page + MODIFIED_PAGE_BLOCK_BITMAP); + bitmap_word_t *bmp_end = (bitmap_word_t *) + (existing_page + MODIFIED_PAGE_BLOCK_UNUSED_2); + bitmap_word_t *bmp_word_2 = (bitmap_word_t *) + (page + MODIFIED_PAGE_BLOCK_BITMAP); + while (bmp_word_1 < bmp_end) { + + *bmp_word_1++ |= *bmp_word_2++; + } + xb_a (bmp_word_1 == bmp_end); + } else { + + /* Add a new page */ + rbt_add_node(result, &tree_search_pos, page); + } + + current_page_end_lsn + = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); + last_page_in_run + = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); + } + + xb_a (current_page_end_lsn >= bmp_end_lsn); + + free(bitmap_files.files); + os_file_close(bitmap_file.file); + + return result; +} + +/****************************************************************//** +Free the bitmap tree. */ +void +xb_page_bitmap_deinit( +/*==================*/ + xb_page_bitmap* bitmap) /*!<in/out: bitmap tree */ +{ + if (bitmap) { + + rbt_free(bitmap); + } +} + +/****************************************************************//** +Advance to the next bitmap page or setup the first bitmap page for the +given bitmap range. Assumes that bitmap_range->bitmap_page has been +already found/bumped by rbt_search()/rbt_next(). + +@return FALSE if no more bitmap data for the range space ID */ +static +ibool +xb_page_bitmap_setup_next_page( +/*===========================*/ + xb_page_bitmap_range* bitmap_range) /*!<in/out: the bitmap range */ +{ + ulint new_space_id; + ulint new_1st_page_id; + + if (bitmap_range->bitmap_node == NULL) { + + bitmap_range->current_page_id = ULINT_UNDEFINED; + return FALSE; + } + + bitmap_range->bitmap_page = rbt_value(byte, bitmap_range->bitmap_node); + + new_space_id = mach_read_from_4(bitmap_range->bitmap_page + + MODIFIED_PAGE_SPACE_ID); + if (new_space_id != bitmap_range->space_id) { + + /* No more data for the current page id. */ + xb_a(new_space_id > bitmap_range->space_id); + bitmap_range->current_page_id = ULINT_UNDEFINED; + return FALSE; + } + + new_1st_page_id = mach_read_from_4(bitmap_range->bitmap_page + + MODIFIED_PAGE_1ST_PAGE_ID); + xb_a (new_1st_page_id >= bitmap_range->current_page_id + || bitmap_range->current_page_id == ULINT_UNDEFINED); + + bitmap_range->current_page_id = new_1st_page_id; + bitmap_range->bit_i = 0; + + return TRUE; +} + +/** Find the node with the smallest key that greater than equal to search key. +@param[in] tree red-black tree +@param[in] key search key +@return node with the smallest greater-than-or-equal key +@retval NULL if none was found */ +static +const ib_rbt_node_t* +rbt_lower_bound(const ib_rbt_t* tree, const void* key) +{ + ut_ad(!tree->cmp_arg); + const ib_rbt_node_t* ge = NULL; + + for (const ib_rbt_node_t *node = tree->root->left; + node != tree->nil; ) { + int result = tree->compare(node->value, key); + + if (result < 0) { + node = node->right; + } else { + ge = node; + if (result == 0) { + break; + } + + node = node->left; + } + } + + return(ge); +} + +/****************************************************************//** +Set up a new bitmap range iterator over a given space id changed +pages in a given bitmap. + +@return bitmap range iterator */ +xb_page_bitmap_range* +xb_page_bitmap_range_init( +/*======================*/ + xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */ + ulint space_id) /*!< in: space id */ +{ + byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; + xb_page_bitmap_range *result + = static_cast<xb_page_bitmap_range *>(malloc(sizeof(*result))); + + memset(result, 0, sizeof(*result)); + result->bitmap = bitmap; + result->space_id = space_id; + result->current_page_id = ULINT_UNDEFINED; + + /* Search for the 1st page for the given space id */ + /* This also sets MODIFIED_PAGE_1ST_PAGE_ID to 0, which is what we + want. */ + memset(search_page, 0, MODIFIED_PAGE_BLOCK_SIZE); + mach_write_to_4(search_page + MODIFIED_PAGE_SPACE_ID, space_id); + + result->bitmap_node = rbt_lower_bound(result->bitmap, search_page); + + xb_page_bitmap_setup_next_page(result); + + return result; +} + +/****************************************************************//** +Get the value of the bitmap->range->bit_i bitmap bit + +@return the current bit value */ +static inline +ibool +is_bit_set( +/*=======*/ + const xb_page_bitmap_range* bitmap_range) /*!< in: bitmap + range */ +{ + return ((*(((bitmap_word_t *)(bitmap_range->bitmap_page + + MODIFIED_PAGE_BLOCK_BITMAP)) + + (bitmap_range->bit_i >> 6))) + & (1ULL << (bitmap_range->bit_i & 0x3F))) ? TRUE : FALSE; +} + +/****************************************************************//** +Get the next page id that has its bit set or cleared, i.e. equal to +bit_value. + +@return page id */ +ulint +xb_page_bitmap_range_get_next_bit( +/*==============================*/ + xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */ + ibool bit_value) /*!< in: bit value */ +{ + if (UNIV_UNLIKELY(bitmap_range->current_page_id + == ULINT_UNDEFINED)) { + + return ULINT_UNDEFINED; + } + + do { + while (bitmap_range->bit_i < MODIFIED_PAGE_BLOCK_ID_COUNT) { + + while (is_bit_set(bitmap_range) != bit_value + && (bitmap_range->bit_i + < MODIFIED_PAGE_BLOCK_ID_COUNT)) { + + bitmap_range->current_page_id++; + bitmap_range->bit_i++; + } + + if (bitmap_range->bit_i + < MODIFIED_PAGE_BLOCK_ID_COUNT) { + + ulint result = bitmap_range->current_page_id; + bitmap_range->current_page_id++; + bitmap_range->bit_i++; + return result; + } + } + + bitmap_range->bitmap_node + = rbt_next(bitmap_range->bitmap, + bitmap_range->bitmap_node); + + } while (xb_page_bitmap_setup_next_page(bitmap_range)); + + return ULINT_UNDEFINED; +} + +/****************************************************************//** +Free the bitmap range iterator. */ +void +xb_page_bitmap_range_deinit( +/*========================*/ + xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */ +{ + free(bitmap_range); +} diff --git a/extra/mariabackup/changed_page_bitmap.h b/extra/mariabackup/changed_page_bitmap.h new file mode 100644 index 00000000..8d504359 --- /dev/null +++ b/extra/mariabackup/changed_page_bitmap.h @@ -0,0 +1,85 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Changed page bitmap interface */ + +#ifndef XB_CHANGED_PAGE_BITMAP_H +#define XB_CHANGED_PAGE_BITMAP_H + +#include <ut0rbt.h> +#include <fil0fil.h> + +/* The changed page bitmap structure */ +typedef ib_rbt_t xb_page_bitmap; + +struct xb_page_bitmap_range_struct; + +/* The bitmap range iterator over one space id */ +typedef struct xb_page_bitmap_range_struct xb_page_bitmap_range; + +/****************************************************************//** +Read the disk bitmap and build the changed page bitmap tree for the +LSN interval incremental_lsn to log_sys.next_checkpoint_lsn. + +@return the built bitmap tree */ +xb_page_bitmap* +xb_page_bitmap_init(void); +/*=====================*/ + +/****************************************************************//** +Free the bitmap tree. */ +void +xb_page_bitmap_deinit( +/*==================*/ + xb_page_bitmap* bitmap); /*!<in/out: bitmap tree */ + + +/****************************************************************//** +Set up a new bitmap range iterator over a given space id changed +pages in a given bitmap. + +@return bitmap range iterator */ +xb_page_bitmap_range* +xb_page_bitmap_range_init( +/*======================*/ + xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */ + ulint space_id); /*!< in: space id */ + +/****************************************************************//** +Get the next page id that has its bit set or cleared, i.e. equal to +bit_value. + +@return page id */ +ulint +xb_page_bitmap_range_get_next_bit( +/*==============================*/ + xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */ + ibool bit_value); /*!< in: bit value */ + +/****************************************************************//** +Free the bitmap range iterator. */ +void +xb_page_bitmap_range_deinit( +/*========================*/ + xb_page_bitmap_range* bitmap_range); /*! in/out: bitmap range */ + +#endif diff --git a/extra/mariabackup/common.h b/extra/mariabackup/common.h new file mode 100644 index 00000000..89b189e3 --- /dev/null +++ b/extra/mariabackup/common.h @@ -0,0 +1,189 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Common declarations for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef XB_COMMON_H +#define XB_COMMON_H + +#include <my_global.h> +#include <mysql_version.h> +#include <fcntl.h> +#include <stdarg.h> +#include <my_sys.h> + + +/** Determine if (i) is a user tablespace id or not. */ +# define fil_is_user_tablespace_id(i) (i != 0 \ + && !srv_is_undo_tablespace(i)) + +#ifdef _MSC_VER +#define stat _stati64 +#define PATH_MAX MAX_PATH +#endif + +#ifndef HAVE_VASPRINTF +static inline int vasprintf(char **strp, const char *fmt, va_list args) +{ + int len; +#ifdef _MSC_VER + len = _vscprintf(fmt, args); +#else + len = vsnprintf(NULL, 0, fmt, args); +#endif + if (len < 0) + { + return -1; + } + *strp = (char *)malloc(len + 1); + if (!*strp) + { + return -1; + } + vsprintf(*strp, fmt, args); + return len; +} + +static inline int asprintf(char **strp, const char *fmt,...) +{ + va_list args; + va_start(args, fmt); + int len = vasprintf(strp, fmt, args); + va_end(args); + return len; +} +#endif + +#define xb_a(expr) \ + do { \ + if (!(expr)) { \ + fprintf(stderr,"Assertion \"%s\" failed at %s:%lu\n", \ + #expr, __FILE__, (ulong) __LINE__); \ + abort(); \ + } \ + } while (0); + +#ifdef XB_DEBUG +#define xb_ad(expr) xb_a(expr) +#else +#define xb_ad(expr) +#endif + +#define XB_DELTA_INFO_SUFFIX ".meta" + +static inline int msg1(uint thread_num, const char *prefix, const char *fmt, va_list args) +{ + int result; + time_t t = time(NULL); + char date[100]; + char *line; + strftime(date, sizeof(date), "%Y-%m-%d %H:%M:%S", localtime(&t)); + result = vasprintf(&line, fmt, args); + if (result != -1) { + if (fmt && fmt[strlen(fmt)] != '\n') + result = fprintf(stderr, "[%02u] %s%s %s\n", thread_num, prefix, date, line); + else + result = fprintf(stderr, "[%02u] %s%s %s", thread_num, prefix, date, line); + free(line); + } + return result; +} + +static inline ATTRIBUTE_FORMAT(printf, 2, 3) int msg(unsigned int thread_num, const char *fmt, ...) +{ + int result; + va_list args; + va_start(args, fmt); + result = msg1(thread_num,"", fmt, args); + va_end(args); + return result; +} + +static inline ATTRIBUTE_FORMAT(printf, 1, 2) int msg(const char *fmt, ...) +{ + int result; + va_list args; + va_start(args, fmt); + result = msg1(0, "", fmt, args); + va_end(args); + return result; +} + +static inline ATTRIBUTE_FORMAT(printf, 1,2) ATTRIBUTE_NORETURN void die(const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + msg1(0, "FATAL ERROR: ", fmt, args); + va_end(args); + fflush(stderr); + _exit(EXIT_FAILURE); +} + + +/* Use POSIX_FADV_NORMAL when available */ + +#ifdef POSIX_FADV_NORMAL +# define USE_POSIX_FADVISE +#else +# define POSIX_FADV_NORMAL +# define POSIX_FADV_SEQUENTIAL +# define POSIX_FADV_DONTNEED +# define posix_fadvise(a,b,c,d) do {} while(0) +#endif + +/*********************************************************************** +Computes bit shift for a given value. If the argument is not a power +of 2, returns 0.*/ +static inline unsigned get_bit_shift(size_t value) +{ + unsigned shift; + + if (value == 0) + return 0; + + for (shift = 0; !(value & 1); shift++) { + value >>= 1; + } + return (value >> 1) ? 0 : shift; +} + +/**************************************************************************** +Read 'len' bytes from 'fd'. It is identical to my_read(..., MYF(MY_FULL_IO)), +i.e. tries to combine partial reads into a single block of size 'len', except +that it bails out on EOF or error, and returns the number of successfully read +bytes instead. */ +static inline size_t +xb_read_full(File fd, uchar *buf, size_t len) +{ + size_t tlen = 0; + size_t tbytes; + + while (tlen < len) { + tbytes = my_read(fd, buf, len - tlen, MYF(MY_WME)); + if (tbytes == 0 || tbytes == MY_FILE_ERROR) { + break; + } + + buf += tbytes; + tlen += tbytes; + } + + return tlen; +} + +#endif diff --git a/extra/mariabackup/datasink.cc b/extra/mariabackup/datasink.cc new file mode 100644 index 00000000..a576526d --- /dev/null +++ b/extra/mariabackup/datasink.cc @@ -0,0 +1,130 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Data sink interface. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <my_base.h> +#include "common.h" +#include "datasink.h" +#include "ds_compress.h" +#include "ds_xbstream.h" +#include "ds_local.h" +#include "ds_stdout.h" +#include "ds_tmpfile.h" +#include "ds_buffer.h" + +/************************************************************************ +Create a datasink of the specified type */ +ds_ctxt_t * +ds_create(const char *root, ds_type_t type) +{ + datasink_t *ds; + ds_ctxt_t *ctxt; + + switch (type) { + case DS_TYPE_STDOUT: + ds = &datasink_stdout; + break; + case DS_TYPE_LOCAL: + ds = &datasink_local; + break; + case DS_TYPE_XBSTREAM: + ds = &datasink_xbstream; + break; + case DS_TYPE_COMPRESS: + ds = &datasink_compress; + break; + case DS_TYPE_ENCRYPT: + case DS_TYPE_DECRYPT: + die("mariabackup does not support encrypted backups."); + break; + + case DS_TYPE_TMPFILE: + ds = &datasink_tmpfile; + break; + case DS_TYPE_BUFFER: + ds = &datasink_buffer; + break; + default: + msg("Unknown datasink type: %d", type); + xb_ad(0); + return NULL; + } + + ctxt = ds->init(root); + if (ctxt != NULL) { + ctxt->datasink = ds; + } else { + die("failed to initialize datasink."); + } + + return ctxt; +} + +/************************************************************************ +Open a datasink file */ +ds_file_t * +ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat) +{ + ds_file_t *file; + + file = ctxt->datasink->open(ctxt, path, stat); + if (file != NULL) { + file->datasink = ctxt->datasink; + } + + return file; +} + +/************************************************************************ +Write to a datasink file. +@return 0 on success, 1 on error. */ +int +ds_write(ds_file_t *file, const void *buf, size_t len) +{ + if (len == 0) { + return 0; + } + return file->datasink->write(file, (const uchar *)buf, len); +} + +/************************************************************************ +Close a datasink file. +@return 0 on success, 1, on error. */ +int +ds_close(ds_file_t *file) +{ + return file->datasink->close(file); +} + +/************************************************************************ +Destroy a datasink handle */ +void +ds_destroy(ds_ctxt_t *ctxt) +{ + ctxt->datasink->deinit(ctxt); +} + +/************************************************************************ +Set the destination pipe for a datasink (only makes sense for compress and +tmpfile). */ +void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt) +{ + ctxt->pipe_ctxt = pipe_ctxt; +} diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h new file mode 100644 index 00000000..57468e0c --- /dev/null +++ b/extra/mariabackup/datasink.h @@ -0,0 +1,134 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Data sink interface. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef XB_DATASINK_H +#define XB_DATASINK_H + +#include <my_global.h> +#include <my_dir.h> + +#ifdef __cplusplus +extern "C" { +#endif + +extern char *xtrabackup_tmpdir; +struct datasink_struct; +typedef struct datasink_struct datasink_t; + +typedef struct ds_ctxt { + datasink_t *datasink; + char *root; + void *ptr; + struct ds_ctxt *pipe_ctxt; + /* + Copy file for backup/restore. + @return true in case of success. + */ + bool copy_file(const char *src_file_path, + const char *dst_file_path, + uint thread_n); + + bool move_file(const char *src_file_path, + const char *dst_file_path, + const char *dst_dir, + uint thread_n); + + bool make_hardlink(const char *from_path, const char *to_path); + + void copy_or_move_dir(const char *from, const char *to, + bool do_copy, bool allow_hardlinks); + + bool backup_file_vprintf(const char *filename, + const char *fmt, va_list ap); + + bool backup_file_print_buf(const char *filename, + const char *buf, + int buf_len); + + bool backup_file_printf(const char *filename, + const char *fmt, ...) + ATTRIBUTE_FORMAT(printf, 2, 0); + +} ds_ctxt_t; + +typedef struct { + void *ptr; + char *path; + datasink_t *datasink; +} ds_file_t; + +struct datasink_struct { + ds_ctxt_t *(*init)(const char *root); + ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); + int (*write)(ds_file_t *file, const unsigned char *buf, size_t len); + int (*close)(ds_file_t *file); + int (*remove)(const char *path); + void (*deinit)(ds_ctxt_t *ctxt); +}; + + +static inline int dummy_remove(const char *) { + return 0; +} + +/* Supported datasink types */ +typedef enum { + DS_TYPE_STDOUT, + DS_TYPE_LOCAL, + DS_TYPE_XBSTREAM, + DS_TYPE_COMPRESS, + DS_TYPE_ENCRYPT, + DS_TYPE_DECRYPT, + DS_TYPE_TMPFILE, + DS_TYPE_BUFFER +} ds_type_t; + +/************************************************************************ +Create a datasink of the specified type */ +ds_ctxt_t *ds_create(const char *root, ds_type_t type); + +/************************************************************************ +Open a datasink file */ +ds_file_t *ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); + +/************************************************************************ +Write to a datasink file. +@return 0 on success, 1 on error. */ +int ds_write(ds_file_t *file, const void *buf, size_t len); + +/************************************************************************ +Close a datasink file. +@return 0 on success, 1, on error. */ +int ds_close(ds_file_t *file); + +/************************************************************************ +Destroy a datasink handle */ +void ds_destroy(ds_ctxt_t *ctxt); + +/************************************************************************ +Set the destination pipe for a datasink (only makes sense for compress and +tmpfile). */ +void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* XB_DATASINK_H */ diff --git a/extra/mariabackup/ds_buffer.cc b/extra/mariabackup/ds_buffer.cc new file mode 100644 index 00000000..d6a42095 --- /dev/null +++ b/extra/mariabackup/ds_buffer.cc @@ -0,0 +1,189 @@ +/****************************************************** +Copyright (c) 2012-2013 Percona LLC and/or its affiliates. + +buffer datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Does buffered output to a destination datasink set with ds_set_pipe(). +Writes to the destination datasink are guaranteed to not be smaller than a +specified buffer size (DS_DEFAULT_BUFFER_SIZE by default), with the only +exception for the last write for a file. */ + +#include <my_global.h> +#include <my_base.h> +#include "ds_buffer.h" +#include "common.h" +#include "datasink.h" + +#define DS_DEFAULT_BUFFER_SIZE (64 * 1024) + +typedef struct { + ds_file_t *dst_file; + char *buf; + size_t pos; + size_t size; +} ds_buffer_file_t; + +typedef struct { + size_t buffer_size; +} ds_buffer_ctxt_t; + +static ds_ctxt_t *buffer_init(const char *root); +static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int buffer_write(ds_file_t *file, const uchar *buf, size_t len); +static int buffer_close(ds_file_t *file); +static void buffer_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_buffer = { + &buffer_init, + &buffer_open, + &buffer_write, + &buffer_close, + &dummy_remove, + &buffer_deinit +}; + +/* Change the default buffer size */ +void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size) +{ + ds_buffer_ctxt_t *buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr; + + buffer_ctxt->buffer_size = size; +} + +static ds_ctxt_t * +buffer_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_buffer_ctxt_t *buffer_ctxt; + + ctxt = (ds_ctxt_t *)my_malloc(PSI_NOT_INSTRUMENTED, + sizeof(ds_ctxt_t) + sizeof(ds_buffer_ctxt_t), MYF(MY_FAE)); + buffer_ctxt = (ds_buffer_ctxt_t *) (ctxt + 1); + buffer_ctxt->buffer_size = DS_DEFAULT_BUFFER_SIZE; + + ctxt->ptr = buffer_ctxt; + ctxt->root = my_strdup(PSI_NOT_INSTRUMENTED, root, MYF(MY_FAE)); + + return ctxt; +} + +static ds_file_t * +buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_buffer_ctxt_t *buffer_ctxt; + ds_ctxt_t *pipe_ctxt; + ds_file_t *dst_file; + ds_file_t *file; + ds_buffer_file_t *buffer_file; + + pipe_ctxt = ctxt->pipe_ctxt; + xb_a(pipe_ctxt != NULL); + + dst_file = ds_open(pipe_ctxt, path, mystat); + if (dst_file == NULL) { + die("ds_open(%s) failed", path); + } + + buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr; + + file = (ds_file_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(ds_file_t) + + sizeof(ds_buffer_file_t) + buffer_ctxt->buffer_size, + MYF(MY_FAE)); + + buffer_file = (ds_buffer_file_t *) (file + 1); + buffer_file->dst_file = dst_file; + buffer_file->buf = (char *) (buffer_file + 1); + buffer_file->size = buffer_ctxt->buffer_size; + buffer_file->pos = 0; + + file->path = dst_file->path; + file->ptr = buffer_file; + + return file; +} + +static int +buffer_write(ds_file_t *file, const uchar *buf, size_t len) +{ + ds_buffer_file_t *buffer_file; + + buffer_file = (ds_buffer_file_t *) file->ptr; + + while (len > 0) { + if (buffer_file->pos + len > buffer_file->size) { + if (buffer_file->pos > 0) { + size_t bytes; + + bytes = buffer_file->size - buffer_file->pos; + memcpy(buffer_file->buf + buffer_file->pos, buf, + bytes); + + if (ds_write(buffer_file->dst_file, + buffer_file->buf, + buffer_file->size)) { + return 1; + } + + buffer_file->pos = 0; + + buf += bytes; + len -= bytes; + } else { + /* We don't have any buffered bytes, just write + the entire source buffer */ + if (ds_write(buffer_file->dst_file, buf, len)) { + return 1; + } + break; + } + } else { + memcpy(buffer_file->buf + buffer_file->pos, buf, len); + buffer_file->pos += len; + break; + } + } + + return 0; +} + +static int +buffer_close(ds_file_t *file) +{ + ds_buffer_file_t *buffer_file; + int ret; + + buffer_file = (ds_buffer_file_t *) file->ptr; + if (buffer_file->pos > 0) { + ds_write(buffer_file->dst_file, buffer_file->buf, + buffer_file->pos); + } + + ret = ds_close(buffer_file->dst_file); + + my_free(file); + + return ret; +} + +static void +buffer_deinit(ds_ctxt_t *ctxt) +{ + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_buffer.h b/extra/mariabackup/ds_buffer.h new file mode 100644 index 00000000..54ffd5c2 --- /dev/null +++ b/extra/mariabackup/ds_buffer.h @@ -0,0 +1,39 @@ +/****************************************************** +Copyright (c) 2012-2013 Percona LLC and/or its affiliates. + +buffer datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef DS_BUFFER_H +#define DS_BUFFER_H + +#include "datasink.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern datasink_t datasink_buffer; + +/* Change the default buffer size */ +void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/extra/mariabackup/ds_compress.cc b/extra/mariabackup/ds_compress.cc new file mode 100644 index 00000000..f7a9b7a1 --- /dev/null +++ b/extra/mariabackup/ds_compress.cc @@ -0,0 +1,475 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. +Copyright (c) 2022, MariaDB Corporation. + +Compressing datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <mysql_version.h> +#include <my_base.h> +#include <quicklz.h> +#include <zlib.h> +#include "common.h" +#include "datasink.h" + +#define COMPRESS_CHUNK_SIZE ((size_t) (xtrabackup_compress_chunk_size)) +#define MY_QLZ_COMPRESS_OVERHEAD 400 + +typedef struct { + pthread_t id; + uint num; + pthread_mutex_t data_mutex; + pthread_cond_t avail_cond; + pthread_cond_t data_cond; + pthread_cond_t done_cond; + pthread_t data_avail; + my_bool cancelled; + const char *from; + size_t from_len; + char *to; + size_t to_len; + qlz_state_compress state; + ulong adler; +} comp_thread_ctxt_t; + +typedef struct { + comp_thread_ctxt_t *threads; + uint nthreads; +} ds_compress_ctxt_t; + +typedef struct { + ds_file_t *dest_file; + ds_compress_ctxt_t *comp_ctxt; + size_t bytes_processed; +} ds_compress_file_t; + +/* Compression options */ +extern char *xtrabackup_compress_alg; +extern uint xtrabackup_compress_threads; +extern ulonglong xtrabackup_compress_chunk_size; + +static ds_ctxt_t *compress_init(const char *root); +static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int compress_write(ds_file_t *file, const uchar *buf, size_t len); +static int compress_close(ds_file_t *file); +static void compress_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_compress = { + &compress_init, + &compress_open, + &compress_write, + &compress_close, + &dummy_remove, + &compress_deinit +}; + +static inline int write_uint32_le(ds_file_t *file, ulong n); +static inline int write_uint64_le(ds_file_t *file, ulonglong n); + +static comp_thread_ctxt_t *create_worker_threads(uint n); +static void destroy_worker_threads(comp_thread_ctxt_t *threads, uint n); +static void *compress_worker_thread_func(void *arg); + +static +ds_ctxt_t * +compress_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_compress_ctxt_t *compress_ctxt; + comp_thread_ctxt_t *threads; + + /* Create and initialize the worker threads */ + threads = create_worker_threads(xtrabackup_compress_threads); + if (threads == NULL) { + msg("compress: failed to create worker threads."); + return NULL; + } + + ctxt = (ds_ctxt_t *) my_malloc(PSI_NOT_INSTRUMENTED, + sizeof(ds_ctxt_t) + sizeof(ds_compress_ctxt_t), MYF(MY_FAE)); + + compress_ctxt = (ds_compress_ctxt_t *) (ctxt + 1); + compress_ctxt->threads = threads; + compress_ctxt->nthreads = xtrabackup_compress_threads; + + ctxt->ptr = compress_ctxt; + ctxt->root = my_strdup(PSI_NOT_INSTRUMENTED, root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_compress_ctxt_t *comp_ctxt; + ds_ctxt_t *dest_ctxt; + ds_file_t *dest_file; + char new_name[FN_REFLEN]; + size_t name_len; + ds_file_t *file; + ds_compress_file_t *comp_file; + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr; + + /* Append the .qp extension to the filename */ + fn_format(new_name, path, "", ".qp", MYF(MY_APPEND_EXT)); + + dest_file = ds_open(dest_ctxt, new_name, mystat); + if (dest_file == NULL) { + return NULL; + } + + /* Write the qpress archive header */ + if (ds_write(dest_file, "qpress10", 8) || + write_uint64_le(dest_file, COMPRESS_CHUNK_SIZE)) { + goto err; + } + + /* We are going to create a one-file "flat" (i.e. with no + subdirectories) archive. So strip the directory part from the path and + remove the '.qp' suffix. */ + fn_format(new_name, path, "", "", MYF(MY_REPLACE_DIR)); + + /* Write the qpress file header */ + name_len = strlen(new_name); + if (ds_write(dest_file, "F", 1) || + write_uint32_le(dest_file, (uint)name_len) || + /* we want to write the terminating \0 as well */ + ds_write(dest_file, new_name, name_len + 1)) { + goto err; + } + + file = (ds_file_t *) my_malloc(PSI_NOT_INSTRUMENTED, + sizeof(ds_file_t) + sizeof(ds_compress_file_t), MYF(MY_FAE)); + comp_file = (ds_compress_file_t *) (file + 1); + comp_file->dest_file = dest_file; + comp_file->comp_ctxt = comp_ctxt; + comp_file->bytes_processed = 0; + + file->ptr = comp_file; + file->path = dest_file->path; + + return file; + +err: + ds_close(dest_file); + return NULL; +} + +static +int +compress_write(ds_file_t *file, const uchar *buf, size_t len) +{ + ds_compress_file_t *comp_file; + ds_compress_ctxt_t *comp_ctxt; + comp_thread_ctxt_t *threads; + comp_thread_ctxt_t *thd; + uint nthreads; + uint i; + const char *ptr; + ds_file_t *dest_file; + + comp_file = (ds_compress_file_t *) file->ptr; + comp_ctxt = comp_file->comp_ctxt; + dest_file = comp_file->dest_file; + + threads = comp_ctxt->threads; + nthreads = comp_ctxt->nthreads; + + const pthread_t self = pthread_self(); + + ptr = (const char *) buf; + while (len > 0) { + bool wait = nthreads == 1; +retry: + bool submitted = false; + + /* Send data to worker threads for compression */ + for (i = 0; i < nthreads; i++) { + size_t chunk_len; + + thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + if (thd->data_avail == pthread_t(~0UL)) { + } else if (!wait) { +skip: + pthread_mutex_unlock(&thd->data_mutex); + continue; + } else { + for (;;) { + pthread_cond_wait(&thd->avail_cond, + &thd->data_mutex); + if (thd->data_avail + == pthread_t(~0UL)) { + break; + } + goto skip; + } + } + + chunk_len = (len > COMPRESS_CHUNK_SIZE) ? + COMPRESS_CHUNK_SIZE : len; + thd->from = ptr; + thd->from_len = chunk_len; + + thd->data_avail = self; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + submitted = true; + len -= chunk_len; + if (len == 0) { + break; + } + ptr += chunk_len; + } + + if (!submitted) { + wait = true; + goto retry; + } + + for (i = 0; i < nthreads; i++) { + thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + if (thd->data_avail != self) { + pthread_mutex_unlock(&thd->data_mutex); + continue; + } + + while (!thd->to_len) { + pthread_cond_wait(&thd->done_cond, + &thd->data_mutex); + } + + bool fail = ds_write(dest_file, "NEWBNEWB", 8) || + write_uint64_le(dest_file, + comp_file->bytes_processed); + comp_file->bytes_processed += thd->from_len; + + if (!fail) { + fail = write_uint32_le(dest_file, thd->adler) || + ds_write(dest_file, thd->to, + thd->to_len); + } + + thd->to_len = 0; + thd->data_avail = pthread_t(~0UL); + pthread_cond_signal(&thd->avail_cond); + pthread_mutex_unlock(&thd->data_mutex); + + if (fail) { + msg("compress: write to the destination stream " + "failed."); + return 1; + } + } + } + + return 0; +} + +static +int +compress_close(ds_file_t *file) +{ + ds_compress_file_t *comp_file; + ds_file_t *dest_file; + int rc; + + comp_file = (ds_compress_file_t *) file->ptr; + dest_file = comp_file->dest_file; + + /* Write the qpress file trailer */ + ds_write(dest_file, "ENDSENDS", 8); + + /* Supposedly the number of written bytes should be written as a + "recovery information" in the file trailer, but in reality qpress + always writes 8 zeros here. Let's do the same */ + + write_uint64_le(dest_file, 0); + + rc = ds_close(dest_file); + + my_free(file); + + return rc; +} + +static +void +compress_deinit(ds_ctxt_t *ctxt) +{ + ds_compress_ctxt_t *comp_ctxt; + + xb_ad(ctxt->pipe_ctxt != NULL); + + comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr;; + + destroy_worker_threads(comp_ctxt->threads, comp_ctxt->nthreads); + + my_free(ctxt->root); + my_free(ctxt); +} + +static inline +int +write_uint32_le(ds_file_t *file, ulong n) +{ + char tmp[4]; + + int4store(tmp, n); + return ds_write(file, tmp, sizeof(tmp)); +} + +static inline +int +write_uint64_le(ds_file_t *file, ulonglong n) +{ + char tmp[8]; + + int8store(tmp, n); + return ds_write(file, tmp, sizeof(tmp)); +} + +static +void +destroy_worker_thread(comp_thread_ctxt_t *thd) +{ + pthread_mutex_lock(&thd->data_mutex); + thd->cancelled = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + pthread_join(thd->id, NULL); + + pthread_cond_destroy(&thd->avail_cond); + pthread_cond_destroy(&thd->data_cond); + pthread_cond_destroy(&thd->done_cond); + pthread_mutex_destroy(&thd->data_mutex); + + my_free(thd->to); +} + +static +comp_thread_ctxt_t * +create_worker_threads(uint n) +{ + comp_thread_ctxt_t *threads; + uint i; + + threads = static_cast<comp_thread_ctxt_t*> + (my_malloc(PSI_NOT_INSTRUMENTED, n * sizeof *threads, + MYF(MY_ZEROFILL|MY_FAE))); + + for (i = 0; i < n; i++) { + comp_thread_ctxt_t *thd = threads + i; + + thd->num = i + 1; + thd->to = static_cast<char*> + (my_malloc(PSI_NOT_INSTRUMENTED, + COMPRESS_CHUNK_SIZE + + MY_QLZ_COMPRESS_OVERHEAD, + MYF(MY_FAE))); + + /* Initialize and data mutex and condition var */ + if (pthread_mutex_init(&thd->data_mutex, NULL) || + pthread_cond_init(&thd->avail_cond, NULL) || + pthread_cond_init(&thd->data_cond, NULL) || + pthread_cond_init(&thd->done_cond, NULL)) { + goto err; + } + + thd->data_avail = pthread_t(~0UL); + + if (pthread_create(&thd->id, NULL, compress_worker_thread_func, + thd)) { + msg("compress: pthread_create() failed: " + "errno = %d", errno); + goto err; + } + } + + return threads; + +err: + for (; i; i--) { + destroy_worker_thread(threads + i); + } + + my_free(threads); + return NULL; +} + +static +void +destroy_worker_threads(comp_thread_ctxt_t *threads, uint n) +{ + uint i; + + for (i = 0; i < n; i++) { + destroy_worker_thread(threads + i); + } + + my_free(threads); +} + +static +void * +compress_worker_thread_func(void *arg) +{ + comp_thread_ctxt_t *thd = (comp_thread_ctxt_t *) arg; + + pthread_mutex_lock(&thd->data_mutex); + + while (1) { + while (!thd->cancelled + && (thd->to_len || thd->data_avail == pthread_t(~0UL))) { + pthread_cond_wait(&thd->data_cond, &thd->data_mutex); + } + + if (thd->cancelled) + break; + thd->to_len = qlz_compress(thd->from, thd->to, thd->from_len, + &thd->state); + + /* qpress uses 0x00010000 as the initial value, but its own + Adler-32 implementation treats the value differently: + 1. higher order bits are the sum of all bytes in the sequence + 2. lower order bits are the sum of resulting values at every + step. + So it's the other way around as compared to zlib's adler32(). + That's why 0x00000001 is being passed here to be compatible + with qpress implementation. */ + + thd->adler = adler32(0x00000001, (uchar *) thd->to, + (uInt)thd->to_len); + pthread_cond_signal(&thd->done_cond); + } + + pthread_mutex_unlock(&thd->data_mutex); + + return NULL; +} diff --git a/extra/mariabackup/ds_compress.h b/extra/mariabackup/ds_compress.h new file mode 100644 index 00000000..f44c1acf --- /dev/null +++ b/extra/mariabackup/ds_compress.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Compression interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef DS_COMPRESS_H +#define DS_COMPRESS_H + +#include "datasink.h" + +extern datasink_t datasink_compress; + +#endif diff --git a/extra/mariabackup/ds_local.cc b/extra/mariabackup/ds_local.cc new file mode 100644 index 00000000..f86612b9 --- /dev/null +++ b/extra/mariabackup/ds_local.cc @@ -0,0 +1,278 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Local datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <my_base.h> +#include <mysys_err.h> +#include "common.h" +#include "datasink.h" +#include "fsp0fsp.h" +#ifdef _WIN32 +#include <winioctl.h> +#endif + +#ifdef HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE +#include <linux/falloc.h> +#endif + +typedef struct { + File fd; + my_bool init_ibd_done; + my_bool is_ibd; + my_bool compressed; + size_t pagesize; +} ds_local_file_t; + +static ds_ctxt_t *local_init(const char *root); +static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int local_write(ds_file_t *file, const uchar *buf, size_t len); +static int local_close(ds_file_t *file); +static void local_deinit(ds_ctxt_t *ctxt); + +static int local_remove(const char *path) +{ + return unlink(path); +} + +extern "C" { +datasink_t datasink_local = { + &local_init, + &local_open, + &local_write, + &local_close, + &local_remove, + &local_deinit +}; +} + +static +ds_ctxt_t * +local_init(const char *root) +{ + ds_ctxt_t *ctxt; + + if (my_mkdir(root, 0777, MYF(0)) < 0 + && my_errno != EEXIST && my_errno != EISDIR) + { + char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf),my_errno); + my_error(EE_CANT_MKDIR, MYF(ME_BELL), + root, my_errno,errbuf, my_errno); + return NULL; + } + + ctxt = (ds_ctxt_t *)my_malloc(PSI_NOT_INSTRUMENTED, sizeof(ds_ctxt_t), MYF(MY_FAE)); + + ctxt->root = my_strdup(PSI_NOT_INSTRUMENTED, root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +local_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat __attribute__((unused))) +{ + char fullpath[FN_REFLEN]; + char dirpath[FN_REFLEN]; + size_t dirpath_len; + size_t path_len; + ds_local_file_t *local_file; + ds_file_t *file; + File fd; + + fn_format(fullpath, path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + + /* Create the directory if needed */ + dirname_part(dirpath, fullpath, &dirpath_len); + if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) { + char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf), my_errno); + my_error(EE_CANT_MKDIR, MYF(ME_BELL), + dirpath, my_errno, errbuf); + return NULL; + } + + fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW, + MYF(MY_WME)); + if (fd < 0) { + return NULL; + } + + path_len = strlen(fullpath) + 1; /* terminating '\0' */ + + file = (ds_file_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(ds_file_t) + + sizeof(ds_local_file_t) + + path_len, + MYF(MY_FAE)); + local_file = (ds_local_file_t *) (file + 1); + + local_file->fd = fd; + local_file->init_ibd_done = 0; + local_file->is_ibd = (path_len > 5) && !strcmp(fullpath + path_len - 5, ".ibd"); + local_file->compressed = 0; + local_file->pagesize = 0; + file->path = (char *) local_file + sizeof(ds_local_file_t); + memcpy(file->path, fullpath, path_len); + + file->ptr = local_file; + + return file; +} + +/* Calculate size of data without trailing zero bytes. */ +static size_t trim_binary_zeros(uchar *buf, size_t pagesize) +{ + size_t i; + for (i = pagesize; (i > 0) && (buf[i - 1] == 0); i--) {}; + return i; +} + + +/* Write data to the output file, and punch "holes" if needed. */ +static int write_compressed(File fd, uchar *data, size_t len, size_t pagesize) +{ + uchar *ptr = data; + for (size_t written= 0; written < len;) + { + size_t n_bytes = MY_MIN(pagesize, len - written); + size_t datasize= trim_binary_zeros(ptr,n_bytes); + if (datasize > 0) { + if (!my_write(fd, ptr, datasize, MYF(MY_WME | MY_NABP))) + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + else + return 1; + } + if (datasize < n_bytes) { + /* This punches a "hole" in the file. */ + size_t hole_bytes = n_bytes - datasize; + my_off_t off = my_seek(fd, hole_bytes, MY_SEEK_CUR, MYF(MY_WME | MY_NABP)); + if (off == MY_FILEPOS_ERROR) + return 1; +#ifdef HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE + /* punch holes harder for filesystems (like XFS) that + heuristically decide whether leave a hole after the + above or not based on the current access pattern + (which is sequential write and not at all typical for + what InnoDB will be doing with the file later */ + fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off - hole_bytes, hole_bytes); +#endif + } + written += n_bytes; + ptr += n_bytes; + } + return 0; +} + + +/* Calculate Innodb tablespace specific data, when first page is written. + We're interested in page compression and page size. +*/ +static void init_ibd_data(ds_local_file_t *local_file, const uchar *buf, size_t len) +{ + if (len < FIL_PAGE_DATA + FSP_SPACE_FLAGS) { + /* Weird, bail out.*/ + return; + } + + auto flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]); + auto ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + local_file->pagesize= ssize == 0 ? UNIV_PAGE_SIZE_ORIG : ((UNIV_ZIP_SIZE_MIN >> 1) << ssize); + local_file->compressed = fil_space_t::full_crc32(flags) + ? fil_space_t::is_compressed(flags) + : bool(FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); + +#if defined(_WIN32) && (MYSQL_VERSION_ID > 100200) + /* Make compressed file sparse, on Windows. + In 10.1, we do not use sparse files. */ + if (local_file->compressed) { + HANDLE handle= my_get_osfhandle(local_file->fd); + if (!DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, NULL, 0)) { + fprintf(stderr, "Warning: cannot make file sparse"); + local_file->compressed = 0; + } + } +#endif +} + + +static +int +local_write(ds_file_t *file, const uchar *buf, size_t len) +{ + uchar *b = (uchar*)buf; + ds_local_file_t *local_file= (ds_local_file_t *)file->ptr; + File fd = local_file->fd; + + if (local_file->is_ibd && !local_file->init_ibd_done) { + init_ibd_data(local_file, b , len); + local_file->init_ibd_done= 1; + } + + if (local_file->compressed) { + return write_compressed(fd, b, len, local_file->pagesize); + } + + if (!my_write(fd, b , len, MYF(MY_WME | MY_NABP))) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + return 0; + } + return 1; +} + +/* Set EOF at file's current position.*/ +static int set_eof(File fd) +{ +#ifdef _WIN32 + return !SetEndOfFile(my_get_osfhandle(fd)); +#elif defined(HAVE_FTRUNCATE) + return ftruncate(fd, my_tell(fd, MYF(MY_WME))); +#else +#error no ftruncate +#endif +} + + +static +int +local_close(ds_file_t *file) +{ + ds_local_file_t *local_file= (ds_local_file_t *)file->ptr; + File fd = local_file->fd; + int ret= 0; + + if (local_file->compressed) { + ret = set_eof(fd); + } + + my_close(fd, MYF(MY_WME)); + my_free(file); + return ret; +} + +static +void +local_deinit(ds_ctxt_t *ctxt) +{ + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_local.h b/extra/mariabackup/ds_local.h new file mode 100644 index 00000000..5555a332 --- /dev/null +++ b/extra/mariabackup/ds_local.h @@ -0,0 +1,34 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Local datasink interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef DS_LOCAL_H +#define DS_LOCAL_H + +#include "datasink.h" + +#ifdef __cplusplus +extern "C" +#else +extern +#endif + +datasink_t datasink_local; + +#endif diff --git a/extra/mariabackup/ds_stdout.cc b/extra/mariabackup/ds_stdout.cc new file mode 100644 index 00000000..a9639ff7 --- /dev/null +++ b/extra/mariabackup/ds_stdout.cc @@ -0,0 +1,121 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Local datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <my_base.h> +#include <mysys_err.h> +#include "common.h" +#include "datasink.h" + +typedef struct { + File fd; +} ds_stdout_file_t; + +static ds_ctxt_t *stdout_init(const char *root); +static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int stdout_write(ds_file_t *file, const uchar *buf, size_t len); +static int stdout_close(ds_file_t *file); +static void stdout_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_stdout = { + &stdout_init, + &stdout_open, + &stdout_write, + &stdout_close, + &dummy_remove, + &stdout_deinit +}; + +static +ds_ctxt_t * +stdout_init(const char *root) +{ + ds_ctxt_t *ctxt; + + ctxt = (ds_ctxt_t *)my_malloc(PSI_NOT_INSTRUMENTED, sizeof(ds_ctxt_t), MYF(MY_FAE)); + + ctxt->root = my_strdup(PSI_NOT_INSTRUMENTED, root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +stdout_open(ds_ctxt_t *ctxt __attribute__((unused)), + const char *path __attribute__((unused)), + MY_STAT *mystat __attribute__((unused))) +{ + ds_stdout_file_t *stdout_file; + ds_file_t *file; + size_t pathlen; + const char *fullpath = "<STDOUT>"; + + pathlen = strlen(fullpath) + 1; + + file = (ds_file_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(ds_file_t) + + sizeof(ds_stdout_file_t) + pathlen, MYF(MY_FAE)); + stdout_file = (ds_stdout_file_t *) (file + 1); + + +#ifdef _WIN32 + setmode(fileno(stdout), _O_BINARY); +#endif + + stdout_file->fd = my_fileno(stdout); + + file->path = (char *) stdout_file + sizeof(ds_stdout_file_t); + memcpy(file->path, fullpath, pathlen); + + file->ptr = stdout_file; + + return file; +} + +static +int +stdout_write(ds_file_t *file, const uchar *buf, size_t len) +{ + File fd = ((ds_stdout_file_t *) file->ptr)->fd; + + if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + return 0; + } + + return 1; +} + +static +int +stdout_close(ds_file_t *file) +{ + my_free(file); + + return 1; +} + +static +void +stdout_deinit(ds_ctxt_t *ctxt) +{ + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_stdout.h b/extra/mariabackup/ds_stdout.h new file mode 100644 index 00000000..6174720d --- /dev/null +++ b/extra/mariabackup/ds_stdout.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Local datasink interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef DS_STDOUT_H +#define DS_STDOUT_H + +#include "datasink.h" + +extern datasink_t datasink_stdout; + +#endif diff --git a/extra/mariabackup/ds_tmpfile.cc b/extra/mariabackup/ds_tmpfile.cc new file mode 100644 index 00000000..80b9d3bb --- /dev/null +++ b/extra/mariabackup/ds_tmpfile.cc @@ -0,0 +1,230 @@ +/****************************************************** +Copyright (c) 2012 Percona LLC and/or its affiliates. + +tmpfile datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Do all writes to temporary files first, then pipe them to the specified +datasink in a serialized way in deinit(). */ + +#include <my_global.h> +#include <my_base.h> +#include "common.h" +#include "datasink.h" + +typedef struct { + pthread_mutex_t mutex; + LIST *file_list; +} ds_tmpfile_ctxt_t; + +typedef struct { + LIST list; + File fd; + char *orig_path; + MY_STAT mystat; + ds_file_t *file; +} ds_tmp_file_t; + +static ds_ctxt_t *tmpfile_init(const char *root); +static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int tmpfile_write(ds_file_t *file, const uchar *buf, size_t len); +static int tmpfile_close(ds_file_t *file); +static void tmpfile_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_tmpfile = { + &tmpfile_init, + &tmpfile_open, + &tmpfile_write, + &tmpfile_close, + &dummy_remove, + &tmpfile_deinit +}; + + +static ds_ctxt_t * +tmpfile_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_tmpfile_ctxt_t *tmpfile_ctxt; + + ctxt = (ds_ctxt_t *)my_malloc(PSI_NOT_INSTRUMENTED, + sizeof(ds_ctxt_t) + sizeof(ds_tmpfile_ctxt_t), MYF(MY_FAE)); + tmpfile_ctxt = (ds_tmpfile_ctxt_t *) (ctxt + 1); + tmpfile_ctxt->file_list = NULL; + if (pthread_mutex_init(&tmpfile_ctxt->mutex, NULL)) { + + my_free(ctxt); + return NULL; + } + + ctxt->ptr = tmpfile_ctxt; + ctxt->root = my_strdup(PSI_NOT_INSTRUMENTED, root, MYF(MY_FAE)); + + return ctxt; +} + +static ds_file_t * +tmpfile_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat) +{ + ds_tmpfile_ctxt_t *tmpfile_ctxt; + char tmp_path[FN_REFLEN]; + ds_tmp_file_t *tmp_file; + ds_file_t *file; + size_t path_len; + File fd; + + /* Create a temporary file in tmpdir. The file will be automatically + removed on close. Code copied from mysql_tmpfile(). */ + fd = create_temp_file(tmp_path,xtrabackup_tmpdir, + "xbtemp", O_BINARY | O_SEQUENTIAL, + MYF(MY_WME | MY_TEMPORARY)); + + if (fd < 0) { + return NULL; + } + + path_len = strlen(path) + 1; /* terminating '\0' */ + + file = (ds_file_t *) my_malloc(PSI_NOT_INSTRUMENTED, + sizeof(ds_file_t) + sizeof(ds_tmp_file_t) + path_len, MYF(MY_FAE)); + + tmp_file = (ds_tmp_file_t *) (file + 1); + tmp_file->file = file; + memcpy(&tmp_file->mystat, mystat, sizeof(MY_STAT)); + /* Save a copy of 'path', since it may not be accessible later */ + tmp_file->orig_path = (char *) tmp_file + sizeof(ds_tmp_file_t); + + tmp_file->fd = fd; + memcpy(tmp_file->orig_path, path, path_len); + + /* Store the real temporary file name in file->path */ + file->path = my_strdup(PSI_NOT_INSTRUMENTED, tmp_path, MYF(MY_FAE)); + file->ptr = tmp_file; + + /* Store the file object in the list to be piped later */ + tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr; + tmp_file->list.data = tmp_file; + + pthread_mutex_lock(&tmpfile_ctxt->mutex); + tmpfile_ctxt->file_list = list_add(tmpfile_ctxt->file_list, + &tmp_file->list); + pthread_mutex_unlock(&tmpfile_ctxt->mutex); + + return file; +} + +static int +tmpfile_write(ds_file_t *file, const uchar *buf, size_t len) +{ + File fd = ((ds_tmp_file_t *) file->ptr)->fd; + + if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + return 0; + } + + return 1; +} + +static int +tmpfile_close(ds_file_t *file) +{ + /* Do nothing -- we will close (and thus remove) the file after piping + it to the destination datasink in tmpfile_deinit(). */ + + my_free(file->path); + + return 0; +} + +static void +tmpfile_deinit(ds_ctxt_t *ctxt) +{ + LIST *list; + ds_tmpfile_ctxt_t *tmpfile_ctxt; + MY_STAT mystat; + ds_tmp_file_t *tmp_file; + ds_file_t *dst_file; + ds_ctxt_t *pipe_ctxt; + void *buf = NULL; + const size_t buf_size = 10 * 1024 * 1024; + size_t bytes; + size_t offset; + + pipe_ctxt = ctxt->pipe_ctxt; + xb_a(pipe_ctxt != NULL); + + buf = my_malloc(PSI_NOT_INSTRUMENTED, buf_size, MYF(MY_FAE)); + + tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr; + list = tmpfile_ctxt->file_list; + + /* Walk the files in the order they have been added */ + list = list_reverse(list); + while (list != NULL) { + tmp_file = (ds_tmp_file_t *)list->data; + /* Stat the file to replace size and mtime on the original + * mystat struct */ + if (my_fstat(tmp_file->fd, &mystat, MYF(0))) { + die("my_fstat() failed."); + } + tmp_file->mystat.st_size = mystat.st_size; + tmp_file->mystat.st_mtime = mystat.st_mtime; + + dst_file = ds_open(pipe_ctxt, tmp_file->orig_path, + &tmp_file->mystat); + if (dst_file == NULL) { + die("could not stream a temporary file to " + "'%s'", tmp_file->orig_path); + } + + /* copy to the destination datasink */ + posix_fadvise(tmp_file->fd, 0, 0, POSIX_FADV_SEQUENTIAL); + if (my_seek(tmp_file->fd, 0, SEEK_SET, MYF(0)) == + MY_FILEPOS_ERROR) { + die("my_seek() failed for '%s', errno = %d.", + tmp_file->file->path, my_errno); + } + offset = 0; + while ((bytes = my_read(tmp_file->fd, (unsigned char *)buf, buf_size, + MYF(MY_WME))) > 0) { + posix_fadvise(tmp_file->fd, offset, buf_size, POSIX_FADV_DONTNEED); + offset += buf_size; + if (ds_write(dst_file, buf, bytes)) { + die("cannot write to stream for '%s'.", + tmp_file->orig_path); + } + } + if (bytes == (size_t) -1) { + die("my_read failed for %s", tmp_file->orig_path); + } + + my_close(tmp_file->fd, MYF(MY_WME)); + ds_close(dst_file); + + list = list_rest(list); + my_free(tmp_file->file); + } + + pthread_mutex_destroy(&tmpfile_ctxt->mutex); + + my_free(buf); + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_tmpfile.h b/extra/mariabackup/ds_tmpfile.h new file mode 100644 index 00000000..24fa9ad6 --- /dev/null +++ b/extra/mariabackup/ds_tmpfile.h @@ -0,0 +1,30 @@ +/****************************************************** +Copyright (c) 2012 Percona LLC and/or its affiliates. + +tmpfile datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef DS_TMPFILE_H +#define DS_TMPFILE_H + +#include "datasink.h" + +extern datasink_t datasink_tmpfile; + +extern MY_TMPDIR mysql_tmpdir_list; + +#endif diff --git a/extra/mariabackup/ds_xbstream.cc b/extra/mariabackup/ds_xbstream.cc new file mode 100644 index 00000000..3bf8bd08 --- /dev/null +++ b/extra/mariabackup/ds_xbstream.cc @@ -0,0 +1,229 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Streaming implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <my_base.h> +#include "common.h" +#include "datasink.h" +#include "xbstream.h" + +typedef struct { + xb_wstream_t *xbstream; + ds_file_t *dest_file; + pthread_mutex_t mutex; +} ds_stream_ctxt_t; + +typedef struct { + xb_wstream_file_t *xbstream_file; + ds_stream_ctxt_t *stream_ctxt; +} ds_stream_file_t; + +/*********************************************************************** +General streaming interface */ + +static ds_ctxt_t *xbstream_init(const char *root); +static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int xbstream_write(ds_file_t *file, const uchar *buf, size_t len); +static int xbstream_close(ds_file_t *file); +static void xbstream_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_xbstream = { + &xbstream_init, + &xbstream_open, + &xbstream_write, + &xbstream_close, + &dummy_remove, + &xbstream_deinit +}; + +static +ssize_t +my_xbstream_write_callback(xb_wstream_file_t *f __attribute__((unused)), + void *userdata, const void *buf, size_t len) +{ + ds_stream_ctxt_t *stream_ctxt; + + stream_ctxt = (ds_stream_ctxt_t *) userdata; + + xb_ad(stream_ctxt != NULL); + xb_ad(stream_ctxt->dest_file != NULL); + + if (!ds_write(stream_ctxt->dest_file, buf, len)) { + return len; + } + return -1; +} + +static +ds_ctxt_t * +xbstream_init(const char *root __attribute__((unused))) +{ + ds_ctxt_t *ctxt; + ds_stream_ctxt_t *stream_ctxt; + xb_wstream_t *xbstream; + + ctxt = (ds_ctxt_t *)my_malloc(PSI_NOT_INSTRUMENTED, + sizeof(ds_ctxt_t) + sizeof(ds_stream_ctxt_t), MYF(MY_FAE)); + stream_ctxt = (ds_stream_ctxt_t *)(ctxt + 1); + + if (pthread_mutex_init(&stream_ctxt->mutex, NULL)) { + msg("xbstream_init: pthread_mutex_init() failed."); + goto err; + } + + xbstream = xb_stream_write_new(); + if (xbstream == NULL) { + msg("xb_stream_write_new() failed."); + goto err; + } + stream_ctxt->xbstream = xbstream; + stream_ctxt->dest_file = NULL; + + ctxt->ptr = stream_ctxt; + + return ctxt; + +err: + my_free(ctxt); + return NULL; +} + +static +ds_file_t * +xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_file_t *file; + ds_stream_file_t *stream_file; + ds_stream_ctxt_t *stream_ctxt; + ds_ctxt_t *dest_ctxt; + xb_wstream_t *xbstream; + xb_wstream_file_t *xbstream_file; + + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr; + + pthread_mutex_lock(&stream_ctxt->mutex); + if (stream_ctxt->dest_file == NULL) { + stream_ctxt->dest_file = ds_open(dest_ctxt, path, mystat); + } + pthread_mutex_unlock(&stream_ctxt->mutex); + if (stream_ctxt->dest_file == NULL) { + return NULL; + } + + file = (ds_file_t *) my_malloc(PSI_NOT_INSTRUMENTED, + sizeof(ds_file_t) + + sizeof(ds_stream_file_t), + MYF(MY_FAE)); + if (!file) { + msg("my_malloc() failed."); + goto err; + } + stream_file = (ds_stream_file_t *) (file + 1); + + xbstream = stream_ctxt->xbstream; + + xbstream_file = xb_stream_write_open(xbstream, path, mystat, + stream_ctxt, + my_xbstream_write_callback); + + if (xbstream_file == NULL) { + msg("xb_stream_write_open() failed."); + goto err; + } + + stream_file->xbstream_file = xbstream_file; + stream_file->stream_ctxt = stream_ctxt; + file->ptr = stream_file; + file->path = stream_ctxt->dest_file->path; + + return file; + +err: + if (stream_ctxt->dest_file) { + ds_close(stream_ctxt->dest_file); + stream_ctxt->dest_file = NULL; + } + my_free(file); + + return NULL; +} + +static +int +xbstream_write(ds_file_t *file, const uchar *buf, size_t len) +{ + ds_stream_file_t *stream_file; + xb_wstream_file_t *xbstream_file; + + + stream_file = (ds_stream_file_t *) file->ptr; + + xbstream_file = stream_file->xbstream_file; + + if (xb_stream_write_data(xbstream_file, buf, len)) { + msg("xb_stream_write_data() failed."); + return 1; + } + + return 0; +} + +static +int +xbstream_close(ds_file_t *file) +{ + ds_stream_file_t *stream_file; + int rc = 0; + + stream_file = (ds_stream_file_t *)file->ptr; + + rc = xb_stream_write_close(stream_file->xbstream_file); + + my_free(file); + + return rc; +} + +static +void +xbstream_deinit(ds_ctxt_t *ctxt) +{ + ds_stream_ctxt_t *stream_ctxt; + + stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr; + + if (xb_stream_write_done(stream_ctxt->xbstream)) { + msg("xb_stream_done() failed."); + } + + if (stream_ctxt->dest_file) { + ds_close(stream_ctxt->dest_file); + stream_ctxt->dest_file = NULL; + } + + pthread_mutex_destroy(&stream_ctxt->mutex); + + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_xbstream.h b/extra/mariabackup/ds_xbstream.h new file mode 100644 index 00000000..acfbb33c --- /dev/null +++ b/extra/mariabackup/ds_xbstream.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Streaming interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef DS_XBSTREAM_H +#define DS_XBSTREAM_H + +#include "datasink.h" + +extern datasink_t datasink_xbstream; + +#endif diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc new file mode 100644 index 00000000..e0a4711a --- /dev/null +++ b/extra/mariabackup/fil_cur.cc @@ -0,0 +1,522 @@ +/****************************************************** +MariaBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Source file cursor implementation */ + +#include <my_global.h> +#include <my_base.h> +#include <fil0fil.h> +#include <fsp0fsp.h> +#include <srv0start.h> +#include <trx0sys.h> + +#include "fil_cur.h" +#include "fil0crypt.h" +#include "fil0pagecompress.h" +#include "common.h" +#include "read_filt.h" +#include "xtrabackup.h" +#include "backup_debug.h" + +/* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */ +#define XB_FIL_CUR_PAGES 640 + +/*********************************************************************** +Extracts the relative path ("database/table.ibd") of a tablespace from a +specified possibly absolute path. + +For user tablespaces both "./database/table.ibd" and +"/remote/dir/database/table.ibd" result in "database/table.ibd". + +For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1" +and "./ibdata1" yield "ibdata1" in the output. */ +const char * +xb_get_relative_path( +/*=================*/ + const char* path, /*!< in: tablespace path (either + relative or absolute) */ + ibool is_system) /*!< in: TRUE for system tablespaces, + i.e. when only the filename must be + returned. */ +{ + const char *next; + const char *cur; + const char *prev; + + prev = NULL; + cur = path; + +#ifdef _WIN32 + while ((next = strchr(cur, '\\')) != NULL) { + prev = cur; + cur = next + 1; + } +#endif + + while ((next = strchr(cur, '/')) != NULL) { + prev = cur; + cur = next + 1; + } + + if (is_system) { + return(cur); + } else { + return((prev == NULL) ? cur : prev); + } + +} + +/**********************************************************************//** +Closes a file. */ +static +void +xb_fil_node_close_file( +/*===================*/ + fil_node_t* node) /*!< in: file node */ +{ + ibool ret; + + mysql_mutex_lock(&fil_system.mutex); + + ut_ad(node); + ut_a(!node->being_extended); + + if (node->is_open()) { + ret = os_file_close(node->handle); + ut_a(ret); + node->handle = OS_FILE_CLOSED; + } + + mysql_mutex_unlock(&fil_system.mutex); +} + +/************************************************************************ +Open a source file cursor and initialize the associated read filter. + +@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must +be skipped and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_open( + /*============*/ + xb_fil_cur_t* cursor, /*!< out: source file cursor */ + xb_read_filt_t* read_filter, /*!< in/out: the read filter */ + fil_node_t* node, /*!< in: source tablespace node */ + uint thread_n, /*!< thread number for diagnostics */ + ulonglong max_file_size) +{ + bool success; + int err; + /* Initialize these first so xb_fil_cur_close() handles them correctly + in case of error */ + cursor->buf = NULL; + cursor->node = NULL; + cursor->n_process_batch = 0; + + cursor->space_id = node->space->id; + + strncpy(cursor->abs_path, node->name, (sizeof cursor->abs_path) - 1); + cursor->abs_path[(sizeof cursor->abs_path) - 1] = '\0'; + + /* Get the relative path for the destination tablespace name, i.e. the + one that can be appended to the backup root directory. Non-system + tablespaces may have absolute paths for DATA DIRECTORY. + We want to make "local" copies for the backup. */ + strncpy(cursor->rel_path, + xb_get_relative_path(cursor->abs_path, cursor->is_system()), + (sizeof cursor->rel_path) - 1); + cursor->rel_path[(sizeof cursor->rel_path) - 1] = '\0'; + + /* In the backup mode we should already have a tablespace handle created + by fil_ibd_load() unless it is a system + tablespace. Otherwise we open the file here. */ + if (!node->is_open()) { + ut_ad(cursor->is_system() + || srv_operation == SRV_OPERATION_RESTORE_DELTA + || xb_close_files); + + node->handle = os_file_create_simple_no_error_handling( + 0, node->name, + OS_FILE_OPEN, + OS_FILE_READ_ALLOW_DELETE, true, &success); + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + + msg(thread_n, "mariabackup: error: cannot open " + "tablespace %s", cursor->abs_path); + + return(XB_FIL_CUR_SKIP); + } + } + + ut_ad(node->is_open()); + + cursor->node = node; + cursor->file = node->handle; +#ifdef _WIN32 + HANDLE hDup; + DuplicateHandle(GetCurrentProcess(),cursor->file.m_file, + GetCurrentProcess(), &hDup, 0, FALSE, DUPLICATE_SAME_ACCESS); + int filenr = _open_osfhandle((intptr_t)hDup, 0); + if (filenr < 0) { + err = EINVAL; + } + else { + err = _fstat64(filenr, &cursor->statinfo); + close(filenr); + } +#else + err = fstat(cursor->file.m_file, &cursor->statinfo); +#endif + if (max_file_size < (ulonglong)cursor->statinfo.st_size) { + cursor->statinfo.st_size = (ulonglong)max_file_size; + } + if (err) { + msg(thread_n, "mariabackup: error: cannot fstat %s", + cursor->abs_path); + + xb_fil_cur_close(cursor); + + return(XB_FIL_CUR_SKIP); + } + + if (srv_file_flush_method == SRV_O_DIRECT + || srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) { + + os_file_set_nocache(cursor->file, node->name, "OPEN"); + } + + posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); + + cursor->page_size = node->space->physical_size(); + cursor->zip_size = node->space->zip_size(); + + /* Allocate read buffer */ + cursor->buf_size = XB_FIL_CUR_PAGES * cursor->page_size; + cursor->buf = static_cast<byte*>(aligned_malloc(cursor->buf_size, + srv_page_size)); + + cursor->buf_read = 0; + cursor->buf_npages = 0; + cursor->buf_offset = 0; + cursor->buf_page_no = 0; + cursor->thread_n = thread_n; + + if (!node->space->crypt_data + && os_file_read(IORequestRead, + node->handle, cursor->buf, 0, + cursor->page_size, nullptr) == DB_SUCCESS) { + mysql_mutex_lock(&fil_system.mutex); + if (!node->space->crypt_data) { + node->space->crypt_data = fil_space_read_crypt_data( + node->space->zip_size(), cursor->buf); + } + mysql_mutex_unlock(&fil_system.mutex); + } + + cursor->space_size = uint32_t(cursor->statinfo.st_size + / cursor->page_size); + + cursor->read_filter = read_filter; + cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, + node->space->id); + + return(XB_FIL_CUR_SUCCESS); +} + +static bool page_is_corrupted(const byte *page, ulint page_no, + const xb_fil_cur_t *cursor, + const fil_space_t *space) +{ + byte tmp_frame[UNIV_PAGE_SIZE_MAX]; + byte tmp_page[UNIV_PAGE_SIZE_MAX]; + const ulint page_size = cursor->page_size; + uint16_t page_type = fil_page_get_type(page); + + /* We ignore the doublewrite buffer pages.*/ + if (cursor->space_id == TRX_SYS_SPACE + && page_no >= FSP_EXTENT_SIZE + && page_no < FSP_EXTENT_SIZE * 3) { + return false; + } + + /* Validate page number. */ + if (mach_read_from_4(page + FIL_PAGE_OFFSET) != page_no + && cursor->space_id != TRX_SYS_SPACE) { + /* On pages that are not all zero, the + page number must match. + + There may be a mismatch on tablespace ID, + because files may be renamed during backup. + We disable the page number check + on the system tablespace, because it may consist + of multiple files, and here we count the pages + from the start of each file.) + + The first 38 and last 8 bytes are never encrypted. */ + const ulint* p = reinterpret_cast<const ulint*>(page); + const ulint* const end = reinterpret_cast<const ulint*>( + page + page_size); + do { + if (*p++) { + return true; + } + } while (p != end); + + /* Whole zero page is valid. */ + return false; + } + + if (space->full_crc32()) { + return buf_page_is_corrupted(true, page, space->flags); + } + + /* Validate encrypted pages. The first page is never encrypted. + In the system tablespace, the first page would be written with + FIL_PAGE_FILE_FLUSH_LSN at shutdown, and if the LSN exceeds + 4,294,967,295, the mach_read_from_4() below would wrongly + interpret the page as encrypted. We prevent that by checking + page_no first. */ + if (page_no + && mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) + && (opt_encrypted_backup + || (space->crypt_data + && space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED))) { + + if (!fil_space_verify_crypt_checksum(page, space->zip_size())) + return true; + + /* Compressed encrypted need to be decrypted + and decompressed for verification. */ + if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED + && !opt_extended_validation) + return false; + + memcpy(tmp_page, page, page_size); + + if (!space->crypt_data + || space->crypt_data->type == CRYPT_SCHEME_UNENCRYPTED + || !fil_space_decrypt(space, tmp_frame, tmp_page)) { + return true; + } + + if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + return buf_page_is_corrupted(true, tmp_page, + space->flags); + } + } + + if (page_type == FIL_PAGE_PAGE_COMPRESSED) { + memcpy(tmp_page, page, page_size); + } + + if (page_type == FIL_PAGE_PAGE_COMPRESSED + || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + ulint decomp = fil_page_decompress(tmp_frame, tmp_page, + space->flags); + page_type = fil_page_get_type(tmp_page); + + return (!decomp + || (decomp != srv_page_size + && cursor->zip_size) + || page_type == FIL_PAGE_PAGE_COMPRESSED + || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED + || buf_page_is_corrupted(true, tmp_page, + space->flags)); + } + + return buf_page_is_corrupted(true, page, space->flags); +} + +/** Reads and verifies the next block of pages from the source +file. Positions the cursor after the last read non-corrupted page. +@param[in,out] cursor source file cursor +@param[out] corrupted_pages adds corrupted pages if +opt_log_innodb_page_corruption is set +@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF +if there are no more pages to read and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t* cursor, + CorruptedPages &corrupted_pages) +{ + byte* page; + unsigned i; + ulint npages; + ulint retry_count; + xb_fil_cur_result_t ret; + ib_int64_t offset; + ib_int64_t to_read; + const ulint page_size = cursor->page_size; + bool defer = false; + xb_ad(!cursor->is_system() || page_size == srv_page_size); + + cursor->read_filter->get_next_batch(&cursor->read_filter_ctxt, + &offset, &to_read); + + if (to_read == 0LL) { + return(XB_FIL_CUR_EOF); + } + +reinit_buf: + cursor->n_process_batch++; + if (to_read > (ib_int64_t) cursor->buf_size) { + to_read = (ib_int64_t) cursor->buf_size; + } + + xb_a(to_read > 0 && to_read <= 0xFFFFFFFFLL); + + if ((to_read & ~(page_size - 1)) + && offset + to_read == cursor->statinfo.st_size) { + + if (to_read < (ib_int64_t) page_size) { + msg(cursor->thread_n, "Warning: junk at the end of " + "%s, offset = %llu, to_read = %llu",cursor->abs_path, (ulonglong) offset, (ulonglong) to_read); + return(XB_FIL_CUR_EOF); + } + + to_read = (ib_int64_t) (((ulint) to_read) & + ~(page_size - 1)); + } + + xb_a((to_read & (page_size - 1)) == 0); + + npages = (ulint) (to_read / page_size); + + retry_count = 10; + ret = XB_FIL_CUR_SUCCESS; + + fil_space_t *space = fil_space_t::get(cursor->space_id); + + if (!space) { + return XB_FIL_CUR_ERROR; + } + +read_retry: + xtrabackup_io_throttling(); + + cursor->buf_read = 0; + cursor->buf_npages = 0; + cursor->buf_offset = offset; + cursor->buf_page_no = static_cast<unsigned>(offset / page_size); + + if (os_file_read(IORequestRead, cursor->file, cursor->buf, offset, + (ulint) to_read, nullptr) != DB_SUCCESS) { + if (!srv_is_undo_tablespace(cursor->space_id)) { + ret = XB_FIL_CUR_ERROR; + goto func_exit; + } + + if (cursor->buf_page_no + >= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES) { + ret = XB_FIL_CUR_SKIP; + goto func_exit; + } + + to_read = SRV_UNDO_TABLESPACE_SIZE_IN_PAGES * page_size; + + if (cursor->n_process_batch > 1) { + ret = XB_FIL_CUR_ERROR; + goto func_exit; + } + + space->release(); + goto reinit_buf; + } + + defer = UT_LIST_GET_FIRST(space->chain)->deferred; + /* check pages for corruption and re-read if necessary. i.e. in case of + partially written pages */ + for (page = cursor->buf, i = 0; i < npages; + page += page_size, i++) { + unsigned page_no = cursor->buf_page_no + i; + + if (!defer && page_is_corrupted(page, page_no, cursor, space)) { + retry_count--; + + if (retry_count == 0) { + const char *ignore_corruption_warn = opt_log_innodb_page_corruption ? + " WARNING!!! The corruption is ignored due to" + " log-innodb-page-corruption option, the backup can contain" + " corrupted data." : ""; + msg(cursor->thread_n, + "Error: failed to read page after " + "10 retries. File %s seems to be " + "corrupted.%s", cursor->abs_path, ignore_corruption_warn); + ut_print_buf(stderr, page, page_size); + if (opt_log_innodb_page_corruption) { + corrupted_pages.add_page(cursor->node->name, + {cursor->node->space->id, page_no}); + retry_count = 1; + } + else { + ret = XB_FIL_CUR_ERROR; + break; + } + } + else { + msg(cursor->thread_n, "Database page corruption detected at page " + UINT32PF ", retrying...", + page_no); + std::this_thread::sleep_for( + std::chrono::milliseconds(100)); + goto read_retry; + } + } + DBUG_EXECUTE_FOR_KEY("add_corrupted_page_for", + cursor->node->space->name(), + { + unsigned corrupted_page_no = + static_cast<unsigned>(strtoul(dbug_val, NULL, 10)); + if (page_no == corrupted_page_no) + corrupted_pages.add_page(cursor->node->name, + {cursor->node->space->id, + corrupted_page_no}); + }); + cursor->buf_read += page_size; + cursor->buf_npages++; + } + + posix_fadvise(cursor->file, offset, to_read, POSIX_FADV_DONTNEED); +func_exit: + space->release(); + return(ret); +} + +/************************************************************************ +Close the source file cursor opened with xb_fil_cur_open() and its +associated read filter. */ +void +xb_fil_cur_close( +/*=============*/ + xb_fil_cur_t *cursor) /*!< in/out: source file cursor */ +{ + if (cursor->read_filter) { + cursor->read_filter->deinit(&cursor->read_filter_ctxt); + } + + aligned_free(cursor->buf); + cursor->buf = NULL; + + if (cursor->node != NULL) { + xb_fil_node_close_file(cursor->node); + cursor->file = OS_FILE_CLOSED; + } +} diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h new file mode 100644 index 00000000..b7812f65 --- /dev/null +++ b/extra/mariabackup/fil_cur.h @@ -0,0 +1,128 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Source file cursor interface */ + +#ifndef FIL_CUR_H +#define FIL_CUR_H + +#include <my_dir.h> +#include "read_filt.h" +#include "srv0start.h" +#include "srv0srv.h" +#include "xtrabackup.h" + +struct xb_fil_cur_t { + pfs_os_file_t file; /*!< source file handle */ + fil_node_t* node; /*!< source tablespace node */ + char rel_path[FN_REFLEN]; + /*!< normalized file path */ + char abs_path[FN_REFLEN]; + /*!< absolute file path */ + MY_STAT statinfo; /*!< information about the file */ + ulint zip_size; /*!< compressed page size in bytes or 0 + for uncompressed pages */ + ulint page_size; /*!< physical page size */ + xb_read_filt_t* read_filter; /*!< read filter */ + xb_read_filt_ctxt_t read_filter_ctxt; + /*!< read filter context */ + byte* buf; /*!< read buffer */ + size_t buf_size; /*!< buffer size in bytes */ + size_t buf_read; /*!< number of read bytes in buffer + after the last cursor read */ + size_t buf_npages; /*!< number of pages in buffer after the + last cursor read */ + ib_int64_t buf_offset; /*!< file offset of the first page in + buffer */ + unsigned buf_page_no; /*!< number of the first page in + buffer */ + uint thread_n; /*!< thread number for diagnostics */ + uint32_t space_id; /*!< ID of tablespace */ + uint32_t space_size; /*!< space size in pages */ + uint32_t n_process_batch;/*!< Number of batch processed */ + + /** @return whether this is not a file-per-table tablespace */ + bool is_system() const + { + ut_ad(space_id != SRV_TMP_SPACE_ID); + return(space_id == TRX_SYS_SPACE + || srv_is_undo_tablespace(space_id)); + } +}; + +typedef enum { + XB_FIL_CUR_SUCCESS, + XB_FIL_CUR_SKIP, + XB_FIL_CUR_ERROR, + XB_FIL_CUR_EOF +} xb_fil_cur_result_t; + +/************************************************************************ +Open a source file cursor and initialize the associated read filter. + +@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must +be skipped and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_open( +/*============*/ + xb_fil_cur_t* cursor, /*!< out: source file cursor */ + xb_read_filt_t* read_filter, /*!< in/out: the read filter */ + fil_node_t* node, /*!< in: source tablespace node */ + uint thread_n, /*!< thread number for diagnostics */ + ulonglong max_file_size = ULLONG_MAX); + +/** Reads and verifies the next block of pages from the source +file. Positions the cursor after the last read non-corrupted page. +@param[in,out] cursor source file cursor +@param[out] corrupted_pages adds corrupted pages if +opt_log_innodb_page_corruption is set +@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF +if there are no more pages to read and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t *cursor, + CorruptedPages &corrupted_pages); +/************************************************************************ +Close the source file cursor opened with xb_fil_cur_open() and its +associated read filter. */ +void +xb_fil_cur_close( +/*=============*/ + xb_fil_cur_t *cursor); /*!< in/out: source file cursor */ + +/*********************************************************************** +Extracts the relative path ("database/table.ibd") of a tablespace from a +specified possibly absolute path. + +For user tablespaces both "./database/table.ibd" and +"/remote/dir/database/table.ibd" result in "database/table.ibd". + +For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1" +and "./ibdata1" yield "ibdata1" in the output. */ +const char * +xb_get_relative_path( +/*=================*/ + const char* path, /*!< in: tablespace path (either + relative or absolute) */ + ibool is_system); /*!< in: TRUE for system tablespaces, + i.e. when only the filename must be + returned. */ + +#endif diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc new file mode 100644 index 00000000..b925b415 --- /dev/null +++ b/extra/mariabackup/innobackupex.cc @@ -0,0 +1,996 @@ +/****************************************************** +hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 51 Franklin +Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <stdio.h> +#include <string.h> +#include <mysql.h> +#include <my_dir.h> +#include <ut0mem.h> +#include <os0file.h> +#include <srv0start.h> +#include <algorithm> +#include <mysqld.h> +#include <my_default.h> +#include <my_getopt.h> +#include <string> +#include <sstream> +#include <set> +#include "common.h" +#include "innobackupex.h" +#include "xtrabackup.h" +#include "xbstream.h" +#include "fil_cur.h" +#include "write_filt.h" +#include "backup_copy.h" + +using std::min; +using std::max; + +/* options */ +my_bool opt_ibx_version = FALSE; +my_bool opt_ibx_help = FALSE; +my_bool opt_ibx_apply_log = FALSE; +my_bool opt_ibx_incremental = FALSE; +my_bool opt_ibx_notimestamp = FALSE; + +my_bool opt_ibx_copy_back = FALSE; +my_bool opt_ibx_move_back = FALSE; +my_bool opt_ibx_galera_info = FALSE; +my_bool opt_ibx_slave_info = FALSE; +my_bool opt_ibx_no_lock = FALSE; +my_bool opt_ibx_safe_slave_backup = FALSE; +my_bool opt_ibx_rsync = FALSE; +my_bool opt_ibx_force_non_empty_dirs = FALSE; +my_bool opt_ibx_noversioncheck = FALSE; +my_bool opt_ibx_no_backup_locks = FALSE; +my_bool opt_ibx_decompress = FALSE; + +char *opt_ibx_incremental_history_name = NULL; +char *opt_ibx_incremental_history_uuid = NULL; + +char *opt_ibx_user = NULL; +const char *opt_ibx_password = NULL; +char *opt_ibx_host = NULL; +char *opt_ibx_defaults_group = NULL; +char *opt_ibx_socket = NULL; +uint opt_ibx_port = 0; + +ulong opt_ibx_lock_wait_query_type; +ulong opt_ibx_kill_long_query_type; + +uint opt_ibx_kill_long_queries_timeout = 0; +uint opt_ibx_lock_wait_timeout = 0; +uint opt_ibx_lock_wait_threshold = 0; +uint opt_ibx_debug_sleep_before_unlock = 0; +uint opt_ibx_safe_slave_backup_timeout = 0; + +const char *opt_ibx_history = NULL; + +char *opt_ibx_include = NULL; +char *opt_ibx_databases = NULL; +bool ibx_partial_backup = false; + +char *ibx_position_arg = NULL; +char *ibx_backup_directory = NULL; + +extern bool xb_opt_destroy_password; + +/* copy of proxied xtrabackup options */ +my_bool ibx_xb_close_files; +const char *ibx_xtrabackup_compress_alg; +uint ibx_xtrabackup_compress_threads; +ulonglong ibx_xtrabackup_compress_chunk_size; +my_bool ibx_xtrabackup_export; +char *ibx_xtrabackup_extra_lsndir; +char *ibx_xtrabackup_incremental_basedir; +char *ibx_xtrabackup_incremental_dir; +my_bool ibx_xtrabackup_incremental_force_scan; +ulint ibx_xtrabackup_log_copy_interval; +char *ibx_xtrabackup_incremental; +int ibx_xtrabackup_parallel; +char *ibx_xtrabackup_stream_str; +char *ibx_xtrabackup_tables_file; +long ibx_xtrabackup_throttle; +char *ibx_opt_mysql_tmpdir; +longlong ibx_xtrabackup_use_memory; + + +static inline int ibx_msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2); +static inline int ibx_msg(const char *fmt, ...) +{ + int result; + time_t t = time(NULL); + char date[100]; + char *line; + va_list args; + + strftime(date, sizeof(date), "%y%m%d %H:%M:%S", localtime(&t)); + + va_start(args, fmt); + + result = vasprintf(&line, fmt, args); + + va_end(args); + + if (result != -1) { + result = fprintf(stderr, "%s %s: %s", + date, INNOBACKUPEX_BIN_NAME, line); + free(line); + } + + return result; +} + +enum innobackupex_options +{ + OPT_APPLY_LOG = 256, + OPT_COPY_BACK, + OPT_MOVE_BACK, + OPT_REDO_ONLY, + OPT_GALERA_INFO, + OPT_SLAVE_INFO, + OPT_INCREMENTAL, + OPT_INCREMENTAL_HISTORY_NAME, + OPT_INCREMENTAL_HISTORY_UUID, + OPT_LOCK_WAIT_QUERY_TYPE, + OPT_KILL_LONG_QUERY_TYPE, + OPT_KILL_LONG_QUERIES_TIMEOUT, + OPT_LOCK_WAIT_TIMEOUT, + OPT_LOCK_WAIT_THRESHOLD, + OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + OPT_NO_LOCK, + OPT_SAFE_SLAVE_BACKUP, + OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + OPT_RSYNC, + OPT_HISTORY, + OPT_INCLUDE, + OPT_FORCE_NON_EMPTY_DIRS, + OPT_NO_TIMESTAMP, + OPT_NO_VERSION_CHECK, + OPT_NO_BACKUP_LOCKS, + OPT_DATABASES, + OPT_DECOMPRESS, + + /* options wich are passed directly to xtrabackup */ + OPT_CLOSE_FILES, + OPT_COMPACT, + OPT_COMPRESS, + OPT_COMPRESS_THREADS, + OPT_COMPRESS_CHUNK_SIZE, + OPT_EXPORT, + OPT_EXTRA_LSNDIR, + OPT_INCREMENTAL_BASEDIR, + OPT_INCREMENTAL_DIR, + OPT_INCREMENTAL_FORCE_SCAN, + OPT_LOG_COPY_INTERVAL, + OPT_PARALLEL, + OPT_REBUILD_INDEXES, + OPT_REBUILD_THREADS, + OPT_STREAM, + OPT_TABLES_FILE, + OPT_THROTTLE, + OPT_USE_MEMORY, + OPT_INNODB_FORCE_RECOVERY, +}; + +ibx_mode_t ibx_mode = IBX_MODE_BACKUP; + +static struct my_option ibx_long_options[] = +{ + {"version", 'v', "print version information", + (uchar *) &opt_ibx_version, (uchar *) &opt_ibx_version, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"help", '?', "This option displays a help screen and exits.", + (uchar *) &opt_ibx_help, (uchar *) &opt_ibx_help, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"apply-log", OPT_APPLY_LOG, "Prepare a backup in BACKUP-DIR by " + "applying the redo log 'ib_logfile0' and creating new redo log. " + "The InnoDB configuration is read from the file \"backup-my.cnf\".", + (uchar*) &opt_ibx_apply_log, (uchar*) &opt_ibx_apply_log, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made " + "backup from the backup directory to their original locations.", + (uchar *) &opt_ibx_copy_back, (uchar *) &opt_ibx_copy_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"move-back", OPT_MOVE_BACK, "Move all the files in a previously made " + "backup from the backup directory to the actual datadir location. " + "Use with caution, as it removes backup files.", + (uchar *) &opt_ibx_move_back, (uchar *) &opt_ibx_move_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"galera-info", OPT_GALERA_INFO, "This options creates the " + "xtrabackup_galera_info file which contains the local node state at " + "the time of the backup. Option should be used when performing the " + "backup of MariaDB Galera Cluster. Has no effect when backup locks " + "are used to create the backup.", + (uchar *) &opt_ibx_galera_info, (uchar *) &opt_ibx_galera_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"slave-info", OPT_SLAVE_INFO, "This option is useful when backing " + "up a replication slave server. It prints the binary log position " + "and name of the master server. It also writes this information to " + "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. " + "A new slave for this master can be set up by starting a slave server " + "on this backup and issuing a \"CHANGE MASTER\" command with the " + "binary log position saved in the \"xtrabackup_slave_info\" file.", + (uchar *) &opt_ibx_slave_info, (uchar *) &opt_ibx_slave_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental", OPT_INCREMENTAL, + "Create an incremental backup, rather than a full one. When this option is specified, " + "either --incremental-lsn or --incremental-basedir can also be given. " + "If neither option is given, option --incremental-basedir is used " + "by default, set to the first timestamped backup " + "directory in the backup base directory.", + (uchar *) &opt_ibx_incremental, (uchar *) &opt_ibx_incremental, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock " + "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + "tables are InnoDB and you DO NOT CARE about the binary log " + "position of the backup. This option shouldn't be used if there " + "are any DDL statements being executed or if any updates are " + "happening on non-InnoDB tables (this includes the system MyISAM " + "tables in the mysql database), otherwise it could lead to an " + "inconsistent backup. If you are considering to use --no-lock " + "because your backups are failing to acquire the lock, this could " + "be because of incoming replication events preventing the lock " + "from succeeding. Please try using --safe-slave-backup to " + "momentarily stop the replication slave thread, this may help " + "the backup to succeed and you then don't need to resort to " + "using this option.", + (uchar *) &opt_ibx_no_lock, (uchar *) &opt_ibx_no_lock, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP, "Stop slave SQL thread " + "and wait to start backup until Slave_open_temp_tables in " + "\"SHOW STATUS\" is zero. If there are no open temporary tables, " + "the backup will take place, otherwise the SQL thread will be " + "started and stopped until there are no open temporary tables. " + "The backup will fail if Slave_open_temp_tables does not become " + "zero after --safe-slave-backup-timeout seconds. The slave SQL " + "thread will be restarted when the backup finishes.", + (uchar *) &opt_ibx_safe_slave_backup, + (uchar *) &opt_ibx_safe_slave_backup, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file " + "transfers. When this option is specified, innobackupex uses rsync " + "to copy all non-InnoDB files instead of spawning a separate cp for " + "each file, which can be much faster for servers with a large number " + "of databases or tables. This option cannot be used together with " + "--stream.", + (uchar *) &opt_ibx_rsync, (uchar *) &opt_ibx_rsync, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " + "option, when specified, makes --copy-back or --move-back transfer " + "files to non-empty directories. Note that no existing files will be " + "overwritten. If --copy-back or --move-back has to copy a file from " + "the backup directory which already exists in the destination " + "directory, it will still fail with an error.", + (uchar *) &opt_ibx_force_non_empty_dirs, + (uchar *) &opt_ibx_force_non_empty_dirs, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-timestamp", OPT_NO_TIMESTAMP, "This option prevents creation of a " + "time-stamped subdirectory of the BACKUP-ROOT-DIR given on the " + "command line. When it is specified, the backup is done in " + "BACKUP-ROOT-DIR instead.", + (uchar *) &opt_ibx_notimestamp, + (uchar *) &opt_ibx_notimestamp, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-version-check", OPT_NO_VERSION_CHECK, "This option disables the " + "version check which is enabled by the --version-check option.", + (uchar *) &opt_ibx_noversioncheck, + (uchar *) &opt_ibx_noversioncheck, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if " + "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " + "on the backup stage. The option has no effect when backup locks are " + "not supported by the server. This option is enabled by default, " + "disable with --no-backup-locks.", + (uchar *) &opt_ibx_no_backup_locks, + (uchar *) &opt_ibx_no_backup_locks, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " + "extension in a backup previously made with the --compress option.", + (uchar *) &opt_ibx_decompress, + (uchar *) &opt_ibx_decompress, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"user", 'u', "This option specifies the MySQL username used " + "when connecting to the server, if that's not the current user. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_ibx_user, (uchar*) &opt_ibx_user, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"host", 'H', "This option specifies the host to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + (uchar*) &opt_ibx_host, (uchar*) &opt_ibx_host, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"port", 'P', "This option specifies the port to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + &opt_ibx_port, &opt_ibx_port, 0, GET_UINT, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"password", 'p', "This option specifies the password to use " + "when connecting to the database. It accepts a string argument. " + "See mysql --help for details.", + 0, 0, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"socket", 'S', "This option specifies the socket to use when " + "connecting to the local database server with a UNIX domain socket. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_ibx_socket, (uchar*) &opt_ibx_socket, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME, + "This option specifies the name of the backup series stored in the " + XB_HISTORY_TABLE " history record to base an " + "incremental backup on. Backup will search the history table " + "looking for the most recent (highest innodb_to_lsn), successful " + "backup in the series and take the to_lsn value to use as the " + "starting lsn for the incremental backup. This will be mutually " + "exclusive with --incremental-history-uuid, --incremental-basedir " + "and --incremental-lsn. If no valid lsn can be found (no series by " + "that name, no successful backups by that name), " + "an error will be returned. It is used with the --incremental option.", + (uchar*) &opt_ibx_incremental_history_name, + (uchar*) &opt_ibx_incremental_history_name, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID, + "This option specifies the UUID of the specific history record " + "stored in the " XB_HISTORY_TABLE " table to base an " + "incremental backup on. --incremental-history-name, " + "--incremental-basedir and --incremental-lsn. If no valid lsn can be " + "found (no success record with that uuid), an error will be returned." + " It is used with the --incremental option.", + (uchar*) &opt_ibx_incremental_history_uuid, + (uchar*) &opt_ibx_incremental_history_uuid, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, + "This option specifies which types of queries are allowed to complete " + "before innobackupex will issue the global lock. Default is all.", + (uchar*) &opt_ibx_lock_wait_query_type, + (uchar*) &opt_ibx_lock_wait_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, + + {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, + "This option specifies which types of queries should be killed to " + "unblock the global lock. Default is \"all\".", + (uchar*) &opt_ibx_kill_long_query_type, + (uchar*) &opt_ibx_kill_long_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0, 0, 0}, + + {"history", OPT_HISTORY, + "This option enables the tracking of backup history in the " + XB_HISTORY_TABLE " table. An optional history " + "series name may be specified that will be placed with the history " + "record for the current backup being taken.", + NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"include", OPT_INCLUDE, + "This option is a regular expression to be matched against table " + "names in databasename.tablename format. It is passed directly to " + "--tables option. See the documentation for " + "details.", + (uchar*) &opt_ibx_include, + (uchar*) &opt_ibx_include, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"databases", OPT_DATABASES, + "This option specifies the list of databases that innobackupex should " + "back up. The option accepts a string argument or path to file that " + "contains the list of databases to back up. The list is of the form " + "\"databasename1[.table_name1] databasename2[.table_name2] . . .\". " + "If this option is not specified, all databases containing MyISAM and " + "InnoDB tables will be backed up. Please make sure that --databases " + "contains all of the InnoDB databases and tables, so that all of the " + "innodb.frm files are also backed up. In case the list is very long, " + "this can be specified in a file, and the full path of the file can " + "be specified instead of the list. (See option --tables-file.)", + (uchar*) &opt_ibx_databases, + (uchar*) &opt_ibx_databases, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, + "This option specifies the number of seconds innobackupex waits " + "between starting FLUSH TABLES WITH READ LOCK and killing those " + "queries that block it. Default is 0 seconds, which means " + "innobackupex will not attempt to kill any queries.", + (uchar*) &opt_ibx_kill_long_queries_timeout, + (uchar*) &opt_ibx_kill_long_queries_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that innobackupex should wait " + "for queries that would block FTWRL before running it. If there are " + "still such queries when the timeout expires, innobackupex terminates " + "with an error. Default is 0, in which case innobackupex does not " + "wait for queries to complete and starts FTWRL immediately.", + (uchar*) &opt_ibx_lock_wait_timeout, + (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, + "This option specifies the query run time threshold which is used by " + "innobackupex to detect long-running queries with a non-zero value " + "of --ftwrl-wait-timeout. FTWRL is not started until such " + "long-running queries exist. This option has no effect if " + "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", + (uchar*) &opt_ibx_lock_wait_threshold, + (uchar*) &opt_ibx_lock_wait_threshold, 0, GET_UINT, + REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, + + {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + "How many seconds --safe-slave-backup should wait for " + "Slave_open_temp_tables to become zero. (default 300)", + (uchar*) &opt_ibx_safe_slave_backup_timeout, + (uchar*) &opt_ibx_safe_slave_backup_timeout, 0, GET_UINT, + REQUIRED_ARG, 300, 0, 0, 0, 0, 0}, + + + /* Following command-line options are actually handled by xtrabackup. + We put them here with only purpose for them to showup in + innobackupex --help output */ + + {"close_files", OPT_CLOSE_FILES, "Do not keep files opened." + " Use at your own risk.", + (uchar*) &ibx_xb_close_files, (uchar*) &ibx_xb_close_files, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress", OPT_COMPRESS, "This option instructs backup to " + "compress backup copies of InnoDB data files." + , (uchar*) &ibx_xtrabackup_compress_alg, + (uchar*) &ibx_xtrabackup_compress_alg, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress-threads", OPT_COMPRESS_THREADS, + "This option specifies the number of worker threads that will be used " + "for parallel compression.", + (uchar*) &ibx_xtrabackup_compress_threads, + (uchar*) &ibx_xtrabackup_compress_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"compress-chunk-size", OPT_COMPRESS_CHUNK_SIZE, "Size of working " + "buffer(s) for compression threads in bytes. The default value " + "is 64K.", (uchar*) &ibx_xtrabackup_compress_chunk_size, + (uchar*) &ibx_xtrabackup_compress_chunk_size, + 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"export", OPT_EXPORT, " enables exporting individual tables for import " + "into another server.", + (uchar*) &ibx_xtrabackup_export, (uchar*) &ibx_xtrabackup_export, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"extra-lsndir", OPT_EXTRA_LSNDIR, "This option specifies the " + "directory in which to save an extra copy of the " + "\"xtrabackup_checkpoints\" file. The option accepts a string " + "argument.", + (uchar*) &ibx_xtrabackup_extra_lsndir, + (uchar*) &ibx_xtrabackup_extra_lsndir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-basedir", OPT_INCREMENTAL_BASEDIR, "This option " + "specifies the directory containing the full backup that is the base " + "dataset for the incremental backup. The option accepts a string " + "argument. It is used with the --incremental option.", + (uchar*) &ibx_xtrabackup_incremental_basedir, + (uchar*) &ibx_xtrabackup_incremental_basedir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-dir", OPT_INCREMENTAL_DIR, "This option specifies the " + "directory where the incremental backup will be combined with the " + "full backup to make a new full backup. The option accepts a string " + "argument. It is used with the --incremental option.", + (uchar*) &ibx_xtrabackup_incremental_dir, + (uchar*) &ibx_xtrabackup_incremental_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-force-scan", OPT_INCREMENTAL_FORCE_SCAN, + "Perform full scan of data files " + "for taking an incremental backup even if full changed page bitmap " + "data is available to enable the backup without the full scan.", + (uchar*)&ibx_xtrabackup_incremental_force_scan, + (uchar*)&ibx_xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {"log-copy-interval", OPT_LOG_COPY_INTERVAL, "This option specifies " + "time interval between checks done by log copying thread in " + "milliseconds.", (uchar*) &ibx_xtrabackup_log_copy_interval, + (uchar*) &ibx_xtrabackup_log_copy_interval, + 0, GET_LONG, REQUIRED_ARG, 1000, 0, LONG_MAX, 0, 1, 0}, + + {"incremental-lsn", OPT_INCREMENTAL, "This option specifies the log " + "sequence number (LSN) to use for the incremental backup. The option " + "accepts a string argument. It is used with the --incremental option. " + "It is used instead of specifying --incremental-basedir. For " + "databases created by MySQL and Percona Server 5.0-series versions, " + "specify the LSN as two 32-bit integers in high:low format. For " + "databases created in 5.1 and later, specify the LSN as a single " + "64-bit integer.", + (uchar*) &ibx_xtrabackup_incremental, + (uchar*) &ibx_xtrabackup_incremental, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"parallel", OPT_PARALLEL, "On backup, this option specifies the " + "number of threads to use to back " + "up files concurrently. The option accepts an integer argument.", + (uchar*) &ibx_xtrabackup_parallel, (uchar*) &ibx_xtrabackup_parallel, + 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, + + + {"stream", OPT_STREAM, "This option specifies the format in which to " + "do the streamed backup. The option accepts a string argument. The " + "backup will be done to STDOUT in the specified format. Currently, " + "the only supported formats are tar and mbstream/xbstream.", + (uchar*) &ibx_xtrabackup_stream_str, + (uchar*) &ibx_xtrabackup_stream_str, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"tables-file", OPT_TABLES_FILE, "This option specifies the file in " + "which there are a list of names of the form database. The option " + "accepts a string argument.table, one per line.", + (uchar*) &ibx_xtrabackup_tables_file, + (uchar*) &ibx_xtrabackup_tables_file, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"throttle", OPT_THROTTLE, "This option specifies a number of I/O " + "operations (pairs of read+write) per second. It accepts an integer " + "argument.", + (uchar*) &ibx_xtrabackup_throttle, (uchar*) &ibx_xtrabackup_throttle, + 0, GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0}, + + {"tmpdir", 't', "This option specifies the location where a temporary " + "files will be stored. If the option is not specified, the default is " + "to use the value of tmpdir read from the server configuration.", + (uchar*) &ibx_opt_mysql_tmpdir, + (uchar*) &ibx_opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"use-memory", OPT_USE_MEMORY, "This option accepts a string argument " + "that specifies the amount of memory in bytes to use " + "for crash recovery while preparing a backup. Multiples are supported " + "providing the unit (e.g. 1MB, 1GB). It is used only with the option " + "--apply-log.", + (uchar*) &ibx_xtrabackup_use_memory, + (uchar*) &ibx_xtrabackup_use_memory, + 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, + 1024*1024L, 0}, + + {"innodb-force-recovery", OPT_INNODB_FORCE_RECOVERY, + "This option starts up the embedded InnoDB instance in crash " + "recovery mode to ignore page corruption; should be used " + "with the \"--apply-log\" option, in emergencies only. The " + "default value is 0. Refer to \"innodb_force_recovery\" server " + "system variable documentation for more details.", + (uchar*)&xtrabackup_innodb_force_recovery, + (uchar*)&xtrabackup_innodb_force_recovery, + 0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0}, + + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + + +static void usage(void) +{ + puts("Open source backup tool\n\ +\n\ +Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\ +Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\ +\n\ +This program is free software; you can redistribute it and/or\n\ +modify it under the terms of the GNU General Public License\n\ +as published by the Free Software Foundation version 2\n\ +of the License.\n\ +\n\ +This program is distributed in the hope that it will be useful,\n\ +but WITHOUT ANY WARRANTY; without even the implied warranty of\n\ +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\ +GNU General Public License for more details.\n\ +\n\ +You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n\n"); + + puts("innobackupex - Non-blocking backup tool for InnoDB, XtraDB and HailDB databases\n\ +\n\ +SYNOPOSIS\n\ +\n\ +innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chunk-size=CHUNK-SIZE]\n\ + [--include=REGEXP] [--user=NAME]\n\ + [--password=WORD] [--port=PORT] [--socket=SOCKET]\n\ + [--no-timestamp] [--ibbackup=IBBACKUP-BINARY]\n\ + [--slave-info] [--galera-info] [--stream=tar|mbstream|xbstream]\n\ + [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME]\n\ + [--databases=LIST] [--no-lock] \n\ + [--tmpdir=DIRECTORY] [--tables-file=FILE]\n\ + [--history=NAME]\n\ + [--incremental] [--incremental-basedir]\n\ + [--incremental-dir] [--incremental-force-scan] [--incremental-lsn]\n\ + [--incremental-history-name=NAME] [--incremental-history-uuid=UUID]\n\ + [--close-files]\n\ + BACKUP-ROOT-DIR\n\ +\n\ +innobackupex --apply-log [--use-memory=B]\n\ + [--defaults-file=MY.CNF]\n\ + [--export] [--ibbackup=IBBACKUP-BINARY]\n\ + [--innodb-force-recovery=1]\n\ + BACKUP-DIR\n\ +\n\ +innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ +\n\ +innobackupex --move-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ +\n\ +innobackupex [--decompress]\n\ + [--parallel=NUMBER-OF-FORKS] BACKUP-DIR\n\ +\n\ +DESCRIPTION\n\ +\n\ +The first command line above makes a hot backup of a database.\n\ +By default it creates a backup directory (named by the current date\n\ + and time) in the given backup root directory. With the --no-timestamp\n\ +option it does not create a time-stamped backup directory, but it puts\n\ +the backup in the given directory (which must not exist). This\n\ +command makes a complete backup of all MyISAM and InnoDB tables and\n\ +indexes in all databases or in all of the databases specified with the\n\ +--databases option. The created backup contains .frm, .MRG, .MYD,\n\ +.MYI, .MAD, .MAI, .TRG, .TRN, .ARM, .ARZ, .CSM, CSV, .opt, .par, and\n\ +InnoDB data and log files. The MY.CNF options file defines the\n\ +location of the database.\n\ +\n\ +The --apply-log command prepares a backup for starting a MySQL\n\ +server on the backup. This command recovers InnoDB data files as specified\n\ +in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/ib_logfile0,\n\ +and creates new InnoDB log files as specified in BACKUP-DIR/backup-my.cnf.\n\ +The BACKUP-DIR should be the path to a backup directory\n\ +\n\ +The --copy-back command copies data, index, and log files\n\ +from the backup directory back to their original locations.\n\ +The MY.CNF options file defines the original location of the database.\n\ +The BACKUP-DIR is the path to a backup directory.\n\ +\n\ +The --move-back command is similar to --copy-back with the only difference that\n\ +it moves files to their original locations rather than copies them. As this\n\ +option removes backup files, it must be used with caution. It may be useful in\n\ +cases when there is not enough free disk space to copy files.\n\ +\n\ +The --decompress command will decompress a backup made\n\ +with the --compress option. The\n\ +--parallel option will allow multiple files to be decompressed\n\ +simultaneously. In order to decompress, the qpress utility MUST be installed\n\ +and accessible within the path. This process will remove the original\n\ +compressed files and leave the results in the same location.\n\ +\n\ +On success the exit code innobackupex is 0. A non-zero exit code \n\ +indicates an error.\n"); + printf("Usage: [%s [--defaults-file=#] --backup | %s [--defaults-file=#] --prepare] [OPTIONS]\n", my_progname, my_progname); + my_print_help(ibx_long_options); +} + + +static +my_bool +ibx_get_one_option(const struct my_option *opt, + const char *argument, const char *) +{ + switch(opt->id) { + case '?': + usage(); + exit(0); + break; + case 'v': + printf("innobackupex version %s %s (%s)", + MYSQL_SERVER_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); + exit(0); + break; + case OPT_HISTORY: + if (argument) { + opt_ibx_history = argument; + } else { + opt_ibx_history = ""; + } + break; + case OPT_STREAM: + if (!strcasecmp(argument, "mbstream") || + !strcasecmp(argument, "xbstream")) + xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM; + else { + ibx_msg("Invalid --stream argument: %s\n", argument); + return 1; + } + xtrabackup_stream = TRUE; + break; + case OPT_COMPRESS: + if (argument == NULL) + xtrabackup_compress_alg = "quicklz"; + else if (strcasecmp(argument, "quicklz")) + { + ibx_msg("Invalid --compress argument: %s\n", argument); + return 1; + } + xtrabackup_compress = TRUE; + break; + case 'p': + opt_ibx_password= argument; + break; + } + return(0); +} + +bool +make_backup_dir() +{ + time_t t = time(NULL); + char buf[100]; + + if (!opt_ibx_notimestamp && !ibx_xtrabackup_stream_str) { + strftime(buf, sizeof(buf), "%Y-%m-%d_%H-%M-%S", localtime(&t)); + ut_a(asprintf(&ibx_backup_directory, "%s/%s", + ibx_position_arg, buf) != -1); + } else { + ibx_backup_directory = strdup(ibx_position_arg); + } + + if (!directory_exists(ibx_backup_directory, true)) { + return(false); + } + + return(true); +} + +bool +ibx_handle_options(int *argc, char ***argv) +{ + int i, n_arguments; + + if (handle_options(argc, argv, ibx_long_options, ibx_get_one_option)) { + return(false); + } + + if (opt_ibx_apply_log) { + ibx_mode = IBX_MODE_APPLY_LOG; + } else if (opt_ibx_copy_back) { + ibx_mode = IBX_MODE_COPY_BACK; + } else if (opt_ibx_move_back) { + ibx_mode = IBX_MODE_MOVE_BACK; + } else if (opt_ibx_decompress) { + ibx_mode = IBX_MODE_DECRYPT_DECOMPRESS; + } else { + ibx_mode = IBX_MODE_BACKUP; + } + + /* find and save position argument */ + i = 0; + n_arguments = 0; + while (i < *argc) { + char *opt = (*argv)[i]; + + if (strncmp(opt, "--", 2) != 0 + && !(strlen(opt) == 2 && opt[0] == '-')) { + if (ibx_position_arg != NULL + && ibx_position_arg != opt) { + ibx_msg("Error: extra argument found %s\n", + opt); + } + ibx_position_arg = opt; + ++n_arguments; + } + ++i; + } + + *argc -= n_arguments; + if (n_arguments > 1) { + return(false); + } + + if (ibx_position_arg == NULL) { + ibx_msg("Missing argument\n"); + return(false); + } + + /* set argv[0] to be the program name */ + --(*argv); + ++(*argc); + + return(true); +} + +/*********************************************************************//** +Parse command-line options, connect to MySQL server, +detect server capabilities, etc. +@return true on success. */ +bool +ibx_init() +{ + const char *run; + + /*=====================*/ + xtrabackup_copy_back = opt_ibx_copy_back; + xtrabackup_move_back = opt_ibx_move_back; + opt_galera_info = opt_ibx_galera_info; + opt_slave_info = opt_ibx_slave_info; + opt_no_lock = opt_ibx_no_lock; + opt_safe_slave_backup = opt_ibx_safe_slave_backup; + opt_rsync = opt_ibx_rsync; + opt_force_non_empty_dirs = opt_ibx_force_non_empty_dirs; + opt_noversioncheck = opt_ibx_noversioncheck; + opt_no_backup_locks = opt_ibx_no_backup_locks; + opt_decompress = opt_ibx_decompress; + + opt_incremental_history_name = opt_ibx_incremental_history_name; + opt_incremental_history_uuid = opt_ibx_incremental_history_uuid; + + opt_user = opt_ibx_user; + opt_password = opt_ibx_password; + opt_host = opt_ibx_host; + opt_defaults_group = opt_ibx_defaults_group; + opt_socket = opt_ibx_socket; + opt_port = opt_ibx_port; + + opt_lock_wait_query_type = opt_ibx_lock_wait_query_type; + opt_kill_long_query_type = opt_ibx_kill_long_query_type; + + opt_kill_long_queries_timeout = opt_ibx_kill_long_queries_timeout; + opt_lock_wait_timeout = opt_ibx_lock_wait_timeout; + opt_lock_wait_threshold = opt_ibx_lock_wait_threshold; + opt_debug_sleep_before_unlock = opt_ibx_debug_sleep_before_unlock; + opt_safe_slave_backup_timeout = opt_ibx_safe_slave_backup_timeout; + + opt_history = opt_ibx_history; + + /* setup xtrabackup options */ + xb_close_files = ibx_xb_close_files; + xtrabackup_compress_alg = ibx_xtrabackup_compress_alg; + xtrabackup_compress_threads = ibx_xtrabackup_compress_threads; + xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size; + xtrabackup_export = ibx_xtrabackup_export; + xtrabackup_extra_lsndir = ibx_xtrabackup_extra_lsndir; + xtrabackup_incremental_basedir = ibx_xtrabackup_incremental_basedir; + xtrabackup_incremental_dir = ibx_xtrabackup_incremental_dir; + xtrabackup_incremental_force_scan = + ibx_xtrabackup_incremental_force_scan; + xtrabackup_log_copy_interval = ibx_xtrabackup_log_copy_interval; + xtrabackup_incremental = ibx_xtrabackup_incremental; + xtrabackup_parallel = ibx_xtrabackup_parallel; + xtrabackup_stream_str = ibx_xtrabackup_stream_str; + xtrabackup_tables_file = ibx_xtrabackup_tables_file; + xtrabackup_throttle = ibx_xtrabackup_throttle; + opt_mysql_tmpdir = ibx_opt_mysql_tmpdir; + xtrabackup_use_memory = ibx_xtrabackup_use_memory; + + if (!opt_ibx_incremental + && (xtrabackup_incremental + || xtrabackup_incremental_basedir + || opt_ibx_incremental_history_name + || opt_ibx_incremental_history_uuid)) { + ibx_msg("Error: --incremental-lsn, --incremental-basedir, " + "--incremental-history-name and " + "--incremental-history-uuid require the " + "--incremental option.\n"); + return(false); + } + + if (opt_ibx_databases != NULL) { + if (is_path_separator(*opt_ibx_databases)) { + xtrabackup_databases_file = opt_ibx_databases; + } else { + xtrabackup_databases = opt_ibx_databases; + } + } + + /* --tables and --tables-file options are xtrabackup only */ + ibx_partial_backup = (opt_ibx_include || opt_ibx_databases); + + if (ibx_mode == IBX_MODE_BACKUP) { + + if (!make_backup_dir()) { + return(false); + } + } + + /* --binlog-info is xtrabackup only, so force + --binlog-info=ON. i.e. behavior before the feature had been + implemented */ + opt_binlog_info = BINLOG_INFO_ON; + + switch (ibx_mode) { + case IBX_MODE_APPLY_LOG: + xtrabackup_prepare = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "apply-log"; + break; + case IBX_MODE_BACKUP: + xtrabackup_backup = TRUE; + xtrabackup_target_dir = ibx_backup_directory; + if (opt_ibx_include != NULL) { + xtrabackup_tables = opt_ibx_include; + } + run = "backup"; + break; + case IBX_MODE_COPY_BACK: + xtrabackup_copy_back = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "copy-back"; + break; + case IBX_MODE_MOVE_BACK: + xtrabackup_move_back = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "move-back"; + break; + case IBX_MODE_DECRYPT_DECOMPRESS: + xtrabackup_decrypt_decompress = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "decompress"; + break; + default: + ut_error; + } + + ibx_msg("Starting the %s operation\n\n" + "IMPORTANT: Please check that the %s run completes " + "successfully.\n" + " At the end of a successful %s run innobackupex\n" + " prints \"completed OK!\".\n\n", run, run, run); + + + return(true); +} + +void +ibx_cleanup() +{ + free(ibx_backup_directory); +} diff --git a/extra/mariabackup/innobackupex.h b/extra/mariabackup/innobackupex.h new file mode 100644 index 00000000..ba134741 --- /dev/null +++ b/extra/mariabackup/innobackupex.h @@ -0,0 +1,45 @@ +/****************************************************** +Copyright (c) 2011-2014 Percona LLC and/or its affiliates. + +Declarations for innobackupex.cc + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef INNOBACKUPEX_H +#define INNOBACKUPEX_H + +#define INNOBACKUPEX_BIN_NAME "innobackupex" + +enum ibx_mode_t { + IBX_MODE_BACKUP, + IBX_MODE_APPLY_LOG, + IBX_MODE_COPY_BACK, + IBX_MODE_MOVE_BACK, + IBX_MODE_DECRYPT_DECOMPRESS +}; + +extern ibx_mode_t ibx_mode; + +bool +ibx_handle_options(int *argc, char ***argv); + +bool +ibx_init(); + +void +ibx_cleanup(); + +#endif diff --git a/extra/mariabackup/quicklz/quicklz.c b/extra/mariabackup/quicklz/quicklz.c new file mode 100644 index 00000000..37421290 --- /dev/null +++ b/extra/mariabackup/quicklz/quicklz.c @@ -0,0 +1,848 @@ +// Fast data compression library +// Copyright (C) 2006-2011 Lasse Mikkel Reinhold +// lar@quicklz.com +// +// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything +// released into public must be open source) or under a commercial license if such +// has been acquired (see http://www.quicklz.com/order.html). The commercial license +// does not cover derived or ported versions created by third parties under GPL. + +// 1.5.0 final + +#include "quicklz.h" + +#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0 + #error quicklz.c and quicklz.h have different versions +#endif + +#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64)) + #define X86X64 +#endif + +#define MINOFFSET 2 +#define UNCONDITIONAL_MATCHLEN 6 +#define UNCOMPRESSED_END 4 +#define CWORD_LEN 4 + +#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 + #define OFFSET_BASE source + #define CAST (ui32)(size_t) +#else + #define OFFSET_BASE 0 + #define CAST +#endif + +int qlz_get_setting(int setting) +{ + switch (setting) + { + case 0: return QLZ_COMPRESSION_LEVEL; + case 1: return sizeof(qlz_state_compress); + case 2: return sizeof(qlz_state_decompress); + case 3: return QLZ_STREAMING_BUFFER; +#ifdef QLZ_MEMORY_SAFE + case 6: return 1; +#else + case 6: return 0; +#endif + case 7: return QLZ_VERSION_MAJOR; + case 8: return QLZ_VERSION_MINOR; + case 9: return QLZ_VERSION_REVISION; + } + return -1; +} + +#if QLZ_COMPRESSION_LEVEL == 1 +static int same(const unsigned char *src, size_t n) +{ + while(n > 0 && *(src + n) == *src) + n--; + return n == 0 ? 1 : 0; +} +#endif + +static void reset_table_compress(qlz_state_compress *state) +{ + int i; + for(i = 0; i < QLZ_HASH_VALUES; i++) + { +#if QLZ_COMPRESSION_LEVEL == 1 + state->hash[i].offset = 0; +#else + state->hash_counter[i] = 0; +#endif + } +} + +static void reset_table_decompress(qlz_state_decompress *state) +{ + int i; + (void)state; + (void)i; +#if QLZ_COMPRESSION_LEVEL == 2 + for(i = 0; i < QLZ_HASH_VALUES; i++) + { + state->hash_counter[i] = 0; + } +#endif +} + +static __inline ui32 hash_func(ui32 i) +{ +#if QLZ_COMPRESSION_LEVEL == 2 + return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1); +#else + return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1); +#endif +} + +static __inline ui32 fast_read(void const *src, ui32 bytes) +{ +#ifndef X86X64 + unsigned char *p = (unsigned char*)src; + switch (bytes) + { + case 4: + return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24); + case 3: + return(*p | *(p + 1) << 8 | *(p + 2) << 16); + case 2: + return(*p | *(p + 1) << 8); + case 1: + return(*p); + } + return 0; +#else + if (bytes >= 1 && bytes <= 4) + return *((ui32*)src); + else + return 0; +#endif +} + +static __inline ui32 hashat(const unsigned char *src) +{ + ui32 fetch, hash; + fetch = fast_read(src, 3); + hash = hash_func(fetch); + return hash; +} + +static __inline void fast_write(ui32 f, void *dst, size_t bytes) +{ +#ifndef X86X64 + unsigned char *p = (unsigned char*)dst; + + switch (bytes) + { + case 4: + *p = (unsigned char)f; + *(p + 1) = (unsigned char)(f >> 8); + *(p + 2) = (unsigned char)(f >> 16); + *(p + 3) = (unsigned char)(f >> 24); + return; + case 3: + *p = (unsigned char)f; + *(p + 1) = (unsigned char)(f >> 8); + *(p + 2) = (unsigned char)(f >> 16); + return; + case 2: + *p = (unsigned char)f; + *(p + 1) = (unsigned char)(f >> 8); + return; + case 1: + *p = (unsigned char)f; + return; + } +#else + switch (bytes) + { + case 4: + *((ui32*)dst) = f; + return; + case 3: + *((ui32*)dst) = f; + return; + case 2: + *((ui16 *)dst) = (ui16)f; + return; + case 1: + *((unsigned char*)dst) = (unsigned char)f; + return; + } +#endif +} + + +size_t qlz_size_decompressed(const char *source) +{ + ui32 n, r; + n = (((*source) & 2) == 2) ? 4 : 1; + r = fast_read(source + 1 + n, n); + r = r & (0xffffffff >> ((4 - n)*8)); + return r; +} + +size_t qlz_size_compressed(const char *source) +{ + ui32 n, r; + n = (((*source) & 2) == 2) ? 4 : 1; + r = fast_read(source + 1, n); + r = r & (0xffffffff >> ((4 - n)*8)); + return r; +} + +size_t qlz_size_header(const char *source) +{ + size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1; + return n; +} + + +static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n) +{ + // Caution if modifying memcpy_up! Overlap of dst and src must be special handled. +#ifndef X86X64 + unsigned char *end = dst + n; + while(dst < end) + { + *dst = *src; + dst++; + src++; + } +#else + ui32 f = 0; + do + { + *(ui32 *)(dst + f) = *(ui32 *)(src + f); + f += MINOFFSET + 1; + } + while (f < n); +#endif +} + +static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s) +{ +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 hash; + hash = hashat(s); + state->hash[hash].offset = s; + state->hash_counter[hash] = 1; +#elif QLZ_COMPRESSION_LEVEL == 2 + ui32 hash; + unsigned char c; + hash = hashat(s); + c = state->hash_counter[hash]; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s; + c++; + state->hash_counter[hash] = c; +#endif + (void)state; + (void)s; +} + +#if QLZ_COMPRESSION_LEVEL <= 2 +static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max) +{ + while(*lh < max) + { + (*lh)++; + update_hash(state, *lh); + } +} +#endif + +static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state) +{ + const unsigned char *last_byte = source + size - 1; + const unsigned char *src = source; + unsigned char *cword_ptr = destination; + unsigned char *dst = destination + CWORD_LEN; + ui32 cword_val = 1U << 31; + const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; + ui32 fetch = 0; + unsigned int lits = 0; + + (void) lits; + + if(src <= last_matchstart) + fetch = fast_read(src, 3); + + while(src <= last_matchstart) + { + if ((cword_val & 1) == 1) + { + // store uncompressed if compression ratio is too low + if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5)) + return 0; + + fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); + + cword_ptr = dst; + dst += CWORD_LEN; + cword_val = 1U << 31; + fetch = fast_read(src, 3); + } +#if QLZ_COMPRESSION_LEVEL == 1 + { + const unsigned char *o; + ui32 hash, cached; + + hash = hash_func(fetch); + cached = fetch ^ state->hash[hash].cache; + state->hash[hash].cache = fetch; + + o = state->hash[hash].offset + OFFSET_BASE; + state->hash[hash].offset = CAST(src - OFFSET_BASE); + +#ifdef X86X64 + if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) + { + if(cached != 0) + { +#else + if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) + { + if (*(o + 3) != *(src + 3)) + { +#endif + hash <<= 4; + cword_val = (cword_val >> 1) | (1U << 31); + fast_write((3 - 2) | hash, dst, 2); + src += 3; + dst += 2; + } + else + { + const unsigned char *old_src = src; + size_t matchlen; + hash <<= 4; + + cword_val = (cword_val >> 1) | (1U << 31); + src += 4; + + if(*(o + (src - old_src)) == *src) + { + src++; + if(*(o + (src - old_src)) == *src) + { + size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1; + size_t remaining = q > 255 ? 255 : q; + src++; + while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining) + src++; + } + } + + matchlen = src - old_src; + if (matchlen < 18) + { + fast_write((ui32)(matchlen - 2) | hash, dst, 2); + dst += 2; + } + else + { + fast_write((ui32)(matchlen << 16) | hash, dst, 3); + dst += 3; + } + } + fetch = fast_read(src, 3); + lits = 0; + } + else + { + lits++; + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); +#ifdef X86X64 + fetch = fast_read(src, 3); +#else + fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16); +#endif + } + } +#elif QLZ_COMPRESSION_LEVEL >= 2 + { + const unsigned char *o, *offset2; + ui32 hash, matchlen, k, m, best_k = 0; + unsigned char c; + size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1); + (void)best_k; + + + //hash = hashat(src); + fetch = fast_read(src, 3); + hash = hash_func(fetch); + + c = state->hash_counter[hash]; + + offset2 = state->hash[hash].offset[0]; + if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0) + { + matchlen = 3; + if(*(offset2 + matchlen) == *(src + matchlen)) + { + matchlen = 4; + while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining) + matchlen++; + } + } + else + matchlen = 0; + for(k = 1; k < QLZ_POINTERS && c > k; k++) + { + o = state->hash[hash].offset[k]; +#if QLZ_COMPRESSION_LEVEL == 3 + if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) +#elif QLZ_COMPRESSION_LEVEL == 2 + if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) +#endif + { + m = 3; + while(*(o + m) == *(src + m) && m < remaining) + m++; +#if QLZ_COMPRESSION_LEVEL == 3 + if ((m > matchlen) || (m == matchlen && o > offset2)) +#elif QLZ_COMPRESSION_LEVEL == 2 + if (m > matchlen) +#endif + { + offset2 = o; + matchlen = m; + best_k = k; + } + } + } + o = offset2; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; + c++; + state->hash_counter[hash] = c; + +#if QLZ_COMPRESSION_LEVEL == 3 + if(matchlen > 2 && src - o < 131071) + { + ui32 u; + size_t offset = src - o; + + for(u = 1; u < matchlen; u++) + { + hash = hashat(src + u); + c = state->hash_counter[hash]++; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u; + } + + cword_val = (cword_val >> 1) | (1U << 31); + src += matchlen; + + if(matchlen == 3 && offset <= 63) + { + *dst = (unsigned char)(offset << 2); + dst++; + } + else if (matchlen == 3 && offset <= 16383) + { + ui32 f = (ui32)((offset << 2) | 1); + fast_write(f, dst, 2); + dst += 2; + } + else if (matchlen <= 18 && offset <= 1023) + { + ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2; + fast_write(f, dst, 2); + dst += 2; + } + + else if(matchlen <= 33) + { + ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3; + fast_write(f, dst, 3); + dst += 3; + } + else + { + ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3; + fast_write(f, dst, 4); + dst += 4; + } + } + else + { + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); + } +#elif QLZ_COMPRESSION_LEVEL == 2 + + if(matchlen > 2) + { + cword_val = (cword_val >> 1) | (1U << 31); + src += matchlen; + + if (matchlen < 10) + { + ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5); + fast_write(f, dst, 2); + dst += 2; + } + else + { + ui32 f = best_k | (matchlen << 16) | (hash << 5); + fast_write(f, dst, 3); + dst += 3; + } + } + else + { + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); + } +#endif + } +#endif + } + while (src <= last_byte) + { + if ((cword_val & 1) == 1) + { + fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); + cword_ptr = dst; + dst += CWORD_LEN; + cword_val = 1U << 31; + } +#if QLZ_COMPRESSION_LEVEL < 3 + if (src <= last_byte - 3) + { +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 hash, fetch; + fetch = fast_read(src, 3); + hash = hash_func(fetch); + state->hash[hash].offset = CAST(src - OFFSET_BASE); + state->hash[hash].cache = fetch; +#elif QLZ_COMPRESSION_LEVEL == 2 + ui32 hash; + unsigned char c; + hash = hashat(src); + c = state->hash_counter[hash]; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; + c++; + state->hash_counter[hash] = c; +#endif + } +#endif + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); + } + + while((cword_val & 1) != 1) + cword_val = (cword_val >> 1); + + fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); + + // min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument + return dst - destination < 9 ? 9 : dst - destination; +} + +static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history) +{ + const unsigned char *src = source + qlz_size_header((const char *)source); + unsigned char *dst = destination; + const unsigned char *last_destination_byte = destination + size - 1; + ui32 cword_val = 1; + const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; + unsigned char *last_hashed = destination - 1; + const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1; + static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0}; + + (void) last_source_byte; + (void) last_hashed; + (void) state; + (void) history; + + for(;;) + { + ui32 fetch; + + if (cword_val == 1) + { +#ifdef QLZ_MEMORY_SAFE + if(src + CWORD_LEN - 1 > last_source_byte) + return 0; +#endif + cword_val = fast_read(src, CWORD_LEN); + src += CWORD_LEN; + } + +#ifdef QLZ_MEMORY_SAFE + if(src + 4 - 1 > last_source_byte) + return 0; +#endif + + fetch = fast_read(src, 4); + + if ((cword_val & 1) == 1) + { + ui32 matchlen; + const unsigned char *offset2; + +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 hash; + cword_val = cword_val >> 1; + hash = (fetch >> 4) & 0xfff; + offset2 = (const unsigned char *)(size_t)state->hash[hash].offset; + + if((fetch & 0xf) != 0) + { + matchlen = (fetch & 0xf) + 2; + src += 2; + } + else + { + matchlen = *(src + 2); + src += 3; + } + +#elif QLZ_COMPRESSION_LEVEL == 2 + ui32 hash; + unsigned char c; + cword_val = cword_val >> 1; + hash = (fetch >> 5) & 0x7ff; + c = (unsigned char)(fetch & 0x3); + offset2 = state->hash[hash].offset[c]; + + if((fetch & (28)) != 0) + { + matchlen = ((fetch >> 2) & 0x7) + 2; + src += 2; + } + else + { + matchlen = *(src + 2); + src += 3; + } + +#elif QLZ_COMPRESSION_LEVEL == 3 + ui32 offset; + cword_val = cword_val >> 1; + if ((fetch & 3) == 0) + { + offset = (fetch & 0xff) >> 2; + matchlen = 3; + src++; + } + else if ((fetch & 2) == 0) + { + offset = (fetch & 0xffff) >> 2; + matchlen = 3; + src += 2; + } + else if ((fetch & 1) == 0) + { + offset = (fetch & 0xffff) >> 6; + matchlen = ((fetch >> 2) & 15) + 3; + src += 2; + } + else if ((fetch & 127) != 3) + { + offset = (fetch >> 7) & 0x1ffff; + matchlen = ((fetch >> 2) & 0x1f) + 2; + src += 3; + } + else + { + offset = (fetch >> 15); + matchlen = ((fetch >> 7) & 255) + 3; + src += 4; + } + + offset2 = dst - offset; +#endif + +#ifdef QLZ_MEMORY_SAFE + if(offset2 < history || offset2 > dst - MINOFFSET - 1) + return 0; + + if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1)) + return 0; +#endif + + memcpy_up(dst, offset2, matchlen); + dst += matchlen; + +#if QLZ_COMPRESSION_LEVEL <= 2 + update_hash_upto(state, &last_hashed, dst - matchlen); + last_hashed = dst - 1; +#endif + } + else + { + if (dst < last_matchstart) + { + unsigned int n = bitlut[cword_val & 0xf]; +#ifdef X86X64 + *(ui32 *)dst = *(ui32 *)src; +#else + memcpy_up(dst, src, 4); +#endif + cword_val = cword_val >> n; + dst += n; + src += n; +#if QLZ_COMPRESSION_LEVEL <= 2 + update_hash_upto(state, &last_hashed, dst - 3); +#endif + } + else + { + while(dst <= last_destination_byte) + { + if (cword_val == 1) + { + src += CWORD_LEN; + cword_val = 1U << 31; + } +#ifdef QLZ_MEMORY_SAFE + if(src >= last_source_byte + 1) + return 0; +#endif + *dst = *src; + dst++; + src++; + cword_val = cword_val >> 1; + } + +#if QLZ_COMPRESSION_LEVEL <= 2 + update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant +#endif + return size; + } + + } + } +} + +size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state) +{ + size_t r; + ui32 compressed; + size_t base; + + if(size == 0 || size > 0xffffffff - 400) + return 0; + + if(size < 216) + base = 3; + else + base = 9; + +#if QLZ_STREAMING_BUFFER > 0 + if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER) +#endif + { + reset_table_compress(state); + r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state); +#if QLZ_STREAMING_BUFFER > 0 + reset_table_compress(state); +#endif + if(r == base) + { + memcpy(destination + base, source, size); + r = size + base; + compressed = 0; + } + else + { + compressed = 1; + } + state->stream_counter = 0; + } +#if QLZ_STREAMING_BUFFER > 0 + else + { + unsigned char *src = state->stream_buffer + state->stream_counter; + + memcpy(src, source, size); + r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state); + + if(r == base) + { + memcpy(destination + base, src, size); + r = size + base; + compressed = 0; + reset_table_compress(state); + } + else + { + compressed = 1; + } + state->stream_counter += size; + } +#endif + if(base == 3) + { + *destination = (unsigned char)(0 | compressed); + *(destination + 1) = (unsigned char)r; + *(destination + 2) = (unsigned char)size; + } + else + { + *destination = (unsigned char)(2 | compressed); + fast_write((ui32)r, destination + 1, 4); + fast_write((ui32)size, destination + 5, 4); + } + + *destination |= (QLZ_COMPRESSION_LEVEL << 2); + *destination |= (1 << 6); + *destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4); + +// 76543210 +// 01SSLLHC + + return r; +} + +size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state) +{ + size_t dsiz = qlz_size_decompressed(source); + +#if QLZ_STREAMING_BUFFER > 0 + if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER) +#endif + { + if((*source & 1) == 1) + { + reset_table_decompress(state); + dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination); + } + else + { + memcpy(destination, source + qlz_size_header(source), dsiz); + } + state->stream_counter = 0; + reset_table_decompress(state); + } +#if QLZ_STREAMING_BUFFER > 0 + else + { + unsigned char *dst = state->stream_buffer + state->stream_counter; + if((*source & 1) == 1) + { + dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer); + } + else + { + memcpy(dst, source + qlz_size_header(source), dsiz); + reset_table_decompress(state); + } + memcpy(destination, dst, dsiz); + state->stream_counter += dsiz; + } +#endif + return dsiz; +} + diff --git a/extra/mariabackup/quicklz/quicklz.h b/extra/mariabackup/quicklz/quicklz.h new file mode 100644 index 00000000..6ffe00f3 --- /dev/null +++ b/extra/mariabackup/quicklz/quicklz.h @@ -0,0 +1,144 @@ +#ifndef QLZ_HEADER +#define QLZ_HEADER + +// Fast data compression library +// Copyright (C) 2006-2011 Lasse Mikkel Reinhold +// lar@quicklz.com +// +// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything +// released into public must be open source) or under a commercial license if such +// has been acquired (see http://www.quicklz.com/order.html). The commercial license +// does not cover derived or ported versions created by third parties under GPL. + +// You can edit following user settings. Data must be decompressed with the same +// setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed +// (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially +// zeroed out (see manual). First #ifndef makes it possible to define settings from +// the outside like the compiler command line. + +// 1.5.0 final + +#ifndef QLZ_COMPRESSION_LEVEL + #define QLZ_COMPRESSION_LEVEL 1 + //#define QLZ_COMPRESSION_LEVEL 2 + //#define QLZ_COMPRESSION_LEVEL 3 + + #define QLZ_STREAMING_BUFFER 0 + //#define QLZ_STREAMING_BUFFER 100000 + //#define QLZ_STREAMING_BUFFER 1000000 + + //#define QLZ_MEMORY_SAFE +#endif + +#define QLZ_VERSION_MAJOR 1 +#define QLZ_VERSION_MINOR 5 +#define QLZ_VERSION_REVISION 0 + +// Using size_t, memset() and memcpy() +#include <string.h> + +// Verify compression level +#if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3 +#error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3 +#endif + +typedef unsigned int ui32; +typedef unsigned short int ui16; + +// Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values! +#if QLZ_COMPRESSION_LEVEL == 1 +#define QLZ_POINTERS 1 +#define QLZ_HASH_VALUES 4096 +#elif QLZ_COMPRESSION_LEVEL == 2 +#define QLZ_POINTERS 4 +#define QLZ_HASH_VALUES 2048 +#elif QLZ_COMPRESSION_LEVEL == 3 +#define QLZ_POINTERS 16 +#define QLZ_HASH_VALUES 4096 +#endif + +// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization. +#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__ + #define QLZ_PTR_64 +#endif + +// hash entry +typedef struct +{ +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 cache; +#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 + unsigned int offset; +#else + const unsigned char *offset; +#endif +#else + const unsigned char *offset[QLZ_POINTERS]; +#endif + +} qlz_hash_compress; + +typedef struct +{ +#if QLZ_COMPRESSION_LEVEL == 1 + const unsigned char *offset; +#else + const unsigned char *offset[QLZ_POINTERS]; +#endif +} qlz_hash_decompress; + + +// states +typedef struct +{ + #if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; + #endif + size_t stream_counter; + qlz_hash_compress hash[QLZ_HASH_VALUES]; + unsigned char hash_counter[QLZ_HASH_VALUES]; +} qlz_state_compress; + + +#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2 + typedef struct + { +#if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; +#endif + qlz_hash_decompress hash[QLZ_HASH_VALUES]; + unsigned char hash_counter[QLZ_HASH_VALUES]; + size_t stream_counter; + } qlz_state_decompress; +#elif QLZ_COMPRESSION_LEVEL == 3 + typedef struct + { +#if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; +#endif +#if QLZ_COMPRESSION_LEVEL <= 2 + qlz_hash_decompress hash[QLZ_HASH_VALUES]; +#endif + size_t stream_counter; + } qlz_state_decompress; +#endif + + +#if defined (__cplusplus) +extern "C" { +#endif + +// Public functions of QuickLZ +size_t qlz_size_decompressed(const char *source); +size_t qlz_size_compressed(const char *source); +size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state); +size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state); +int qlz_get_setting(int setting); +size_t qlz_size_header(const char *source); + +#if defined (__cplusplus) +} +#endif + +#endif + diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc new file mode 100644 index 00000000..58920055 --- /dev/null +++ b/extra/mariabackup/read_filt.cc @@ -0,0 +1,207 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Data file read filter implementation */ + +#include "read_filt.h" +#include "common.h" +#include "fil_cur.h" +#include "xtrabackup.h" + +/****************************************************************//** +Perform read filter context initialization that is common to all read +filters. */ +static +void +common_init( +/*========*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter context */ + const xb_fil_cur_t* cursor) /*!<in: file cursor */ +{ + ctxt->offset = 0; + ctxt->data_file_size = cursor->statinfo.st_size; + ctxt->buffer_capacity = cursor->buf_size; + ctxt->page_size = cursor->page_size; +} + +/****************************************************************//** +Initialize the pass-through read filter. */ +static +void +rf_pass_through_init( +/*=================*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter context */ + const xb_fil_cur_t* cursor, /*!<in: file cursor */ + ulint space_id __attribute__((unused))) + /*!<in: space id we are reading */ +{ + common_init(ctxt, cursor); +} + +/****************************************************************//** +Get the next batch of pages for the pass-through read filter. */ +static +void +rf_pass_through_get_next_batch( +/*===========================*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter + context */ + ib_int64_t* read_batch_start, /*!<out: starting read + offset in bytes for the + next batch of pages */ + ib_int64_t* read_batch_len) /*!<out: length in + bytes of the next batch + of pages */ +{ + *read_batch_start = ctxt->offset; + *read_batch_len = ctxt->data_file_size - ctxt->offset; + + if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { + *read_batch_len = ctxt->buffer_capacity; + } + + ctxt->offset += *read_batch_len; +} + +/****************************************************************//** +Deinitialize the pass-through read filter. */ +static +void +rf_pass_through_deinit( +/*===================*/ + xb_read_filt_ctxt_t* ctxt __attribute__((unused))) + /*!<in: read filter context */ +{ +} + +/****************************************************************//** +Initialize the changed page bitmap-based read filter. Assumes that +the bitmap is already set up in changed_page_bitmap. */ +static +void +rf_bitmap_init( +/*===========*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter + context */ + const xb_fil_cur_t* cursor, /*!<in: read cursor */ + ulint space_id) /*!<in: space id */ +{ + common_init(ctxt, cursor); + ctxt->bitmap_range = xb_page_bitmap_range_init(changed_page_bitmap, + space_id); + ctxt->filter_batch_end = 0; +} + +/****************************************************************//** +Get the next batch of pages for the bitmap read filter. */ +static +void +rf_bitmap_get_next_batch( +/*=====================*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter + context */ + ib_int64_t* read_batch_start, /*!<out: starting read + offset in bytes for the + next batch of pages */ + ib_int64_t* read_batch_len) /*!<out: length in + bytes of the next batch + of pages */ +{ + ulint start_page_id; + const ulint page_size = ctxt->page_size; + + start_page_id = (ulint)(ctxt->offset / page_size); + + xb_a (ctxt->offset % page_size == 0); + + if (start_page_id == ctxt->filter_batch_end) { + + /* Used up all the previous bitmap range, get some more */ + ulint next_page_id; + + /* Find the next changed page using the bitmap */ + next_page_id = xb_page_bitmap_range_get_next_bit + (ctxt->bitmap_range, TRUE); + + if (next_page_id == ULINT_UNDEFINED) { + *read_batch_len = 0; + return; + } + + ctxt->offset = next_page_id * page_size; + + /* Find the end of the current changed page block by searching + for the next cleared bitmap bit */ + ctxt->filter_batch_end + = xb_page_bitmap_range_get_next_bit(ctxt->bitmap_range, + FALSE); + xb_a(next_page_id < ctxt->filter_batch_end); + } + + *read_batch_start = ctxt->offset; + if (ctxt->filter_batch_end == ULINT_UNDEFINED) { + /* No more cleared bits in the bitmap, need to copy all the + remaining pages. */ + *read_batch_len = ctxt->data_file_size - ctxt->offset; + } else { + *read_batch_len = ctxt->filter_batch_end * page_size + - ctxt->offset; + } + + /* If the page block is larger than the buffer capacity, limit it to + buffer capacity. The subsequent invocations will continue returning + the current block in buffer-sized pieces until ctxt->filter_batch_end + is reached, trigerring the next bitmap query. */ + if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { + *read_batch_len = ctxt->buffer_capacity; + } + + ctxt->offset += *read_batch_len; + xb_a (ctxt->offset % page_size == 0); + xb_a (*read_batch_start % page_size == 0); + xb_a (*read_batch_len % page_size == 0); +} + +/****************************************************************//** +Deinitialize the changed page bitmap-based read filter. */ +static +void +rf_bitmap_deinit( +/*=============*/ + xb_read_filt_ctxt_t* ctxt) /*!<in/out: read filter context */ +{ + xb_page_bitmap_range_deinit(ctxt->bitmap_range); +} + +/* The pass-through read filter */ +xb_read_filt_t rf_pass_through = { + &rf_pass_through_init, + &rf_pass_through_get_next_batch, + &rf_pass_through_deinit +}; + +/* The changed page bitmap-based read filter */ +xb_read_filt_t rf_bitmap = { + &rf_bitmap_init, + &rf_bitmap_get_next_batch, + &rf_bitmap_deinit +}; diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h new file mode 100644 index 00000000..51150705 --- /dev/null +++ b/extra/mariabackup/read_filt.h @@ -0,0 +1,66 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Data file read filter interface */ + +#ifndef XB_READ_FILT_H +#define XB_READ_FILT_H + +#include "changed_page_bitmap.h" + +typedef uint32_t space_id_t; + +struct xb_fil_cur_t; + +/* The read filter context */ +struct xb_read_filt_ctxt_t { + ib_int64_t offset; /*!< current file offset */ + ib_int64_t data_file_size; /*!< data file size */ + size_t buffer_capacity;/*!< read buffer capacity */ + space_id_t space_id; /*!< space id */ + /* The following fields used only in bitmap filter */ + /* Move these to union if any other filters are added in future */ + xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range + iterator for space_id */ + ulint page_size; /*!< page size */ + ulint filter_batch_end;/*!< the ending page id of the + current changed page block in + the bitmap */ + /** TODO: remove this default constructor */ + xb_read_filt_ctxt_t() : page_size(0) {} +}; + +/* The read filter */ +struct xb_read_filt_t { + void (*init)(xb_read_filt_ctxt_t* ctxt, + const xb_fil_cur_t* cursor, + ulint space_id); + void (*get_next_batch)(xb_read_filt_ctxt_t* ctxt, + ib_int64_t* read_batch_start, + ib_int64_t* read_batch_len); + void (*deinit)(xb_read_filt_ctxt_t* ctxt); +}; + +extern xb_read_filt_t rf_pass_through; +extern xb_read_filt_t rf_bitmap; + +#endif diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc new file mode 100644 index 00000000..052cea26 --- /dev/null +++ b/extra/mariabackup/write_filt.cc @@ -0,0 +1,236 @@ +/****************************************************** +MariaBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Page write filters implementation */ + +#include <my_global.h> +#include <my_base.h> +#include "common.h" +#include "write_filt.h" +#include "fil_cur.h" +#include "xtrabackup.h" + +/************************************************************************ +Write-through page write filter. */ +static my_bool wf_wt_init(ds_ctxt *ds_meta, + xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages); +static my_bool wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile); + +xb_write_filt_t wf_write_through = { + &wf_wt_init, + &wf_wt_process, + NULL, + NULL +}; + +/************************************************************************ +Incremental page write filter. */ +static my_bool wf_incremental_init(ds_ctxt *ds_meta, + xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages); +static my_bool wf_incremental_process(xb_write_filt_ctxt_t *ctxt, + ds_file_t *dstfile); +static my_bool wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, + ds_file_t *dstfile); +static void wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt); + +xb_write_filt_t wf_incremental = { + &wf_incremental_init, + &wf_incremental_process, + &wf_incremental_finalize, + &wf_incremental_deinit +}; + +/************************************************************************ +Initialize incremental page write filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_incremental_init(ds_ctxt *ds_meta, + xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages) +{ + char meta_name[FN_REFLEN]; + xb_wf_incremental_ctxt_t *cp = + &(ctxt->wf_incremental_ctxt); + + ctxt->cursor = cursor; + + /* allocate buffer for incremental backup (4096 pages) */ + cp->delta_buf_size = (cursor->page_size / 4) * cursor->page_size; + cp->delta_buf = (unsigned char *)my_large_malloc(&cp->delta_buf_size, MYF(0)); + + if (!cp->delta_buf) { + msg(cursor->thread_n,"Can't allocate %zu bytes", + (size_t) cp->delta_buf_size); + return (FALSE); + } + + /* write delta meta info */ + snprintf(meta_name, sizeof(meta_name), "%s%s", dst_name, + XB_DELTA_INFO_SUFFIX); + const xb_delta_info_t info(cursor->page_size, cursor->zip_size, + cursor->space_id); + if (!xb_write_delta_metadata(ds_meta, meta_name, &info)) { + msg(cursor->thread_n,"Error: " + "failed to write meta info for %s", + cursor->rel_path); + return(FALSE); + } + + /* change the target file name, since we are only going to write + delta pages */ + strcat(dst_name, ".delta"); + + mach_write_to_4(cp->delta_buf, 0x78747261UL); /*"xtra"*/ + + cp->npages = 1; + cp->corrupted_pages = corrupted_pages; + + return(TRUE); +} + +/************************************************************************ +Run the next batch of pages through incremental page write filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + unsigned i; + xb_fil_cur_t *cursor = ctxt->cursor; + byte *page; + const ulint page_size = cursor->page_size; + xb_wf_incremental_ctxt_t *cp = &(ctxt->wf_incremental_ctxt); + + for (i = 0, page = cursor->buf; i < cursor->buf_npages; + i++, page += page_size) { + + if ((!cp->corrupted_pages || + !cp->corrupted_pages->contains({cursor->node->space->id, + cursor->buf_page_no + i})) && + incremental_lsn >= mach_read_from_8(page + FIL_PAGE_LSN)) + continue; + + /* Check whether TRX_SYS page has been changed */ + if (mach_read_from_4(page + FIL_PAGE_SPACE_ID) + == TRX_SYS_SPACE + && mach_read_from_4(page + FIL_PAGE_OFFSET) + == TRX_SYS_PAGE_NO) { + msg(cursor->thread_n, + "--incremental backup is impossible if " + "the server had been restarted with " + "different innodb_undo_tablespaces."); + return false; + } + + /* updated page */ + if (cp->npages == page_size / 4) { + /* flush buffer */ + if (ds_write(dstfile, cp->delta_buf, + cp->npages * page_size)) { + return(FALSE); + } + + /* clear buffer */ + memset(cp->delta_buf, 0, page_size / 4 * page_size); + /*"xtra"*/ + mach_write_to_4(cp->delta_buf, 0x78747261UL); + cp->npages = 1; + } + + mach_write_to_4(cp->delta_buf + cp->npages * 4, + cursor->buf_page_no + i); + memcpy(cp->delta_buf + cp->npages * page_size, page, + page_size); + + cp->npages++; + } + + return(TRUE); +} + +/************************************************************************ +Flush the incremental page write filter's buffer. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + xb_fil_cur_t *cursor = ctxt->cursor; + const ulint page_size = cursor->page_size; + xb_wf_incremental_ctxt_t *cp = &(ctxt->wf_incremental_ctxt); + + if (cp->npages != page_size / 4) { + mach_write_to_4(cp->delta_buf + cp->npages * 4, 0xFFFFFFFFUL); + } + + /* Mark the final block */ + mach_write_to_4(cp->delta_buf, 0x58545241UL); /*"XTRA"*/ + + /* flush buffer */ + if (ds_write(dstfile, cp->delta_buf, cp->npages * page_size)) { + return(FALSE); + } + + return(TRUE); +} + +/************************************************************************ +Free the incremental page write filter's buffer. */ +static void +wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt) +{ + xb_wf_incremental_ctxt_t *cp = &(ctxt->wf_incremental_ctxt); + my_large_free(cp->delta_buf, cp->delta_buf_size); +} + +/************************************************************************ +Initialize the write-through page write filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_wt_init(ds_ctxt *ds_meta __attribute__((unused)), + xb_write_filt_ctxt_t *ctxt, char *dst_name __attribute__((unused)), + xb_fil_cur_t *cursor, CorruptedPages *) +{ + ctxt->cursor = cursor; + + return(TRUE); +} + +/************************************************************************ +Write the next batch of pages to the destination datasink. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + xb_fil_cur_t *cursor = ctxt->cursor; + + if (ds_write(dstfile, cursor->buf, cursor->buf_read)) { + return(FALSE); + } + + return(TRUE); +} diff --git a/extra/mariabackup/write_filt.h b/extra/mariabackup/write_filt.h new file mode 100644 index 00000000..a0ce0778 --- /dev/null +++ b/extra/mariabackup/write_filt.h @@ -0,0 +1,59 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* Page write filter interface */ + +#ifndef XB_WRITE_FILT_H +#define XB_WRITE_FILT_H + +#include "fil_cur.h" +#include "datasink.h" +#include "xtrabackup.h" + +/* Incremental page filter context */ +typedef struct { + ulint delta_buf_size; + byte *delta_buf; + ulint npages; + CorruptedPages *corrupted_pages; +} xb_wf_incremental_ctxt_t; + +/* Page filter context used as an opaque structure by callers */ +typedef struct { + xb_fil_cur_t *cursor; + xb_wf_incremental_ctxt_t wf_incremental_ctxt; +} xb_write_filt_ctxt_t; + + +typedef struct { + my_bool (*init)(ds_ctxt *ds_meta, + xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages); + my_bool (*process)(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile); + my_bool (*finalize)(xb_write_filt_ctxt_t *, ds_file_t *dstfile); + void (*deinit)(xb_write_filt_ctxt_t *); +} xb_write_filt_t; + +extern xb_write_filt_t wf_write_through; +extern xb_write_filt_t wf_incremental; + +#endif /* XB_WRITE_FILT_H */ diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc new file mode 100644 index 00000000..1b93e9ed --- /dev/null +++ b/extra/mariabackup/wsrep.cc @@ -0,0 +1,117 @@ +/****************************************************** +Percona XtraBackup: hot backup tool for InnoDB +(c) 2009-2014 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + + Copyright 2010 Codership Oy <http://www.codership.com> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <my_base.h> +#include <handler.h> +#include <trx0rseg.h> +#include <mysql/service_wsrep.h> + +#include "common.h" +#ifdef WITH_WSREP + +#include <wsrep_api.h> + +/*! Name of file where Galera info is stored on recovery */ +#define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info" + +/*********************************************************************** +Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that +information is present in the trx system header. Otherwise, do nothing. */ +void +xb_write_galera_info(bool incremental_prepare) +/*==================*/ +{ + FILE* fp; + XID xid; + char uuid_str[40]; + long long seqno; + MY_STAT statinfo; + + /* Do not overwrite existing an existing file to be compatible with + servers with older server versions */ + if (!incremental_prepare && + my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) { + + return; + } + + xid.null(); + + if (!trx_rseg_read_wsrep_checkpoint(xid)) { + + return; + } + + wsrep_uuid_t uuid; + memcpy(uuid.data, wsrep_xid_uuid(&xid), sizeof(uuid.data)); + if (wsrep_uuid_print(&uuid, uuid_str, + sizeof(uuid_str)) < 0) { + return; + } + + fp = fopen(XB_GALERA_INFO_FILENAME, "w"); + if (fp == NULL) { + + die( + "could not create " XB_GALERA_INFO_FILENAME + ", errno = %d\n", + errno); + exit(EXIT_FAILURE); + } + + seqno = wsrep_xid_seqno(&xid); + + msg("mariabackup: Recovered WSREP position: %s:%lld\n", + uuid_str, (long long) seqno); + + if (fprintf(fp, "%s:%lld", uuid_str, (long long) seqno) < 0) { + + die( + "could not write to " XB_GALERA_INFO_FILENAME + ", errno = %d\n", + errno);; + } + + fclose(fp); +} +#endif diff --git a/extra/mariabackup/xb_plugin.cc b/extra/mariabackup/xb_plugin.cc new file mode 100644 index 00000000..7470d376 --- /dev/null +++ b/extra/mariabackup/xb_plugin.cc @@ -0,0 +1,229 @@ +/* Copyright (c) 2017, 2022, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ + +#include <my_global.h> +#include <mysqld.h> +#include <mysql.h> +#include <xtrabackup.h> +#include <xb_plugin.h> +#include <sql_plugin.h> +#include <sstream> +#include <vector> +#include <common.h> +#include <backup_mysql.h> +#include <srv0srv.h> + + +extern struct st_maria_plugin *mysql_optional_plugins[]; +extern struct st_maria_plugin *mysql_mandatory_plugins[]; +static void xb_plugin_init(int argc, char **argv); + +extern char *xb_plugin_load; +extern char *xb_plugin_dir; + +const int PLUGIN_MAX_ARGS = 1024; +std::vector<std::string> backup_plugins_args; + +const char *QUERY_PLUGIN = +"SELECT plugin_name, plugin_library, @@plugin_dir" +" FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'" +" OR (plugin_type = 'DAEMON' AND plugin_name LIKE 'provider\\_%')" +" AND plugin_status='ACTIVE'"; + +std::string xb_plugin_config; + +static void add_to_plugin_load_list(const char *plugin_def) +{ + opt_plugin_load_list_ptr->push_back(new i_string(plugin_def)); +} + +static char XTRABACKUP_EXE[] = "xtrabackup"; + +/* + Read "plugin-load" value from backup-my.cnf during prepare phase. + The value is stored during backup phase. +*/ +static std::string get_plugin_from_cnf(const char *dir) +{ + std::string path = dir + std::string("/backup-my.cnf"); + FILE *f = fopen(path.c_str(), "r"); + if (!f) + { + die("Can't open %s for reading", path.c_str()); + } + char line[512]; + std::string plugin_load; + while (fgets(line, sizeof(line), f)) + { + if (strncmp(line, "plugin_load=", 12) == 0) + { + plugin_load = line + 12; + // remote \n at the end of string + plugin_load.resize(plugin_load.size() - 1); + break; + } + } + fclose(f); + return plugin_load; +} + + +void xb_plugin_backup_init(MYSQL *mysql) +{ + MYSQL_RES *result; + MYSQL_ROW row; + std::ostringstream oss; + char *argv[PLUGIN_MAX_ARGS]; + char show_query[1024] = ""; + std::string plugin_load; + int argc; + + result = xb_mysql_query(mysql, QUERY_PLUGIN, true, true); + while ((row = mysql_fetch_row(result))) + { + char *name= row[0]; + char *library= row[1]; + char *dir= row[2]; + + if (!plugin_load.length()) + { +#ifdef _WIN32 + for (char *p = dir; *p; p++) + if (*p == '\\') *p = '/'; +#endif + strncpy(opt_plugin_dir, dir, FN_REFLEN - 1); + opt_plugin_dir[FN_REFLEN - 1] = '\0'; + oss << "plugin_dir=" << '"' << dir << '"' << std::endl; + } + + plugin_load += std::string(";") + name; + + if (library) + { + /* Remove shared library suffixes, in case we'll prepare on different OS.*/ + const char *extensions[] = { ".dll", ".so", 0 }; + for (size_t i = 0; extensions[i]; i++) + { + const char *ext = extensions[i]; + if (ends_with(library, ext)) + library[strlen(library) - strlen(ext)] = 0; + } + plugin_load += std::string("=") + library; + } + + if (strncmp(name, "provider_", 9) == 0) + continue; + + /* Read plugin variables. */ + snprintf(show_query, sizeof(show_query), "SHOW variables like '%s_%%'", name); + } + mysql_free_result(result); + if (!plugin_load.length()) + return; + + oss << "plugin_load=" << plugin_load.c_str() + 1 << std::endl; + + /* Required to load the plugin later.*/ + add_to_plugin_load_list(plugin_load.c_str() + 1); + + + if (*show_query) + { + result = xb_mysql_query(mysql, show_query, true, true); + while ((row = mysql_fetch_row(result))) + { + std::string arg("--"); + arg += row[0]; + arg += "="; + arg += row[1]; + backup_plugins_args.push_back(arg); + oss << row[0] << "=" << row[1] << std::endl; + } + + mysql_free_result(result); + + /* Check whether to encrypt logs. */ + result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true); + row = mysql_fetch_row(result); + srv_encrypt_log = (row != 0 && row[0][0] == '1'); + oss << "innodb_encrypt_log=" << row[0] << std::endl; + + mysql_free_result(result); + } + + xb_plugin_config = oss.str(); + + argc = 0; + argv[argc++] = XTRABACKUP_EXE; + for(size_t i = 0; i < backup_plugins_args.size(); i++) + { + argv[argc++] = (char *)backup_plugins_args[i].c_str(); + if (argc == PLUGIN_MAX_ARGS - 2) + break; + } + argv[argc] = 0; + + xb_plugin_init(argc, argv); +} + +const char *xb_plugin_get_config() +{ + return xb_plugin_config.c_str(); +} + +extern int finalize_encryption_plugin(st_plugin_int *plugin); + + +void xb_plugin_prepare_init(int argc, char **argv, const char *dir) +{ + std::string plugin_load= get_plugin_from_cnf(dir ? dir : "."); + if (plugin_load.size()) + { + msg("Loading plugins from %s", plugin_load.c_str()); + } + else + { + finalize_encryption_plugin(0); + return; + } + + add_to_plugin_load_list(plugin_load.c_str()); + + if (xb_plugin_dir) + { + strncpy(opt_plugin_dir, xb_plugin_dir, FN_REFLEN - 1); + opt_plugin_dir[FN_REFLEN - 1] = '\0'; + } + + char **new_argv = new char *[argc + 2]; + new_argv[0] = XTRABACKUP_EXE; + memcpy(&new_argv[1], argv, argc*sizeof(char *)); + + xb_plugin_init(argc+1, new_argv); + + delete[] new_argv; +} + +static void xb_plugin_init(int argc, char **argv) +{ + /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */ + mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0; + plugin_maturity = MariaDB_PLUGIN_MATURITY_UNKNOWN; /* mariabackup accepts all plugins */ + msg("Loading plugins"); + for (int i= 1; i < argc; i++) + msg("\t Plugin parameter : '%s'", argv[i]); + plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE); +} + diff --git a/extra/mariabackup/xb_plugin.h b/extra/mariabackup/xb_plugin.h new file mode 100644 index 00000000..fea24b6b --- /dev/null +++ b/extra/mariabackup/xb_plugin.h @@ -0,0 +1,5 @@ +#include <mysql.h> +#include <string> +extern void xb_plugin_backup_init(MYSQL *mysql); +extern const char* xb_plugin_get_config(); +extern void xb_plugin_prepare_init(int argc, char **argv, const char *dir); diff --git a/extra/mariabackup/xb_regex.h b/extra/mariabackup/xb_regex.h new file mode 100644 index 00000000..8f2f0908 --- /dev/null +++ b/extra/mariabackup/xb_regex.h @@ -0,0 +1,49 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +/* This file is required to abstract away regex(3) calls so that +my_regex is used on Windows and native calls are used on POSIX platforms. */ + +#ifndef XB_REGEX_H +#define XB_REGEX_H + +#ifdef HAVE_SYSTEM_REGEX +#include <regex.h> +#else +#define PCRE2_STATIC 1 /* Important on Windows */ +#include <pcre2posix.h> +#endif + +typedef regex_t* xb_regex_t; + +#define xb_regex_init() + +#define xb_regexec(preg,string,nmatch,pmatch,eflags) \ + regexec(preg, string, nmatch, pmatch, eflags) + +#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ + regerror(errcode, preg, errbuf, errbuf_size) + +#define xb_regcomp(preg,regex,cflags) \ + regcomp(preg, regex, cflags) + +#define xb_regfree(preg) regfree(preg) + +#define xb_regex_end() + +#endif /* XB_REGEX_H */ diff --git a/extra/mariabackup/xbcloud.cc b/extra/mariabackup/xbcloud.cc new file mode 100644 index 00000000..588a15eb --- /dev/null +++ b/extra/mariabackup/xbcloud.cc @@ -0,0 +1,2722 @@ +/****************************************************** +Copyright (c) 2014 Percona LLC and/or its affiliates. + +The xbstream utility: serialize/deserialize files in the XBSTREAM format. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <my_default.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <curl/curl.h> +#include <ev.h> +#include <unistd.h> +#include <errno.h> +#include <gcrypt.h> +#include <assert.h> +#include <my_sys.h> +#include <my_dir.h> +#include <my_getopt.h> +#include <algorithm> +#include <map> +#include <string> +#include <jsmn.h> +#include "xbstream.h" + +using std::min; +using std::max; +using std::map; +using std::string; + +#define XBCLOUD_VERSION "1.0" + +#define SWIFT_MAX_URL_SIZE 8192 +#define SWIFT_MAX_HDR_SIZE 8192 + +#define SWIFT_CHUNK_SIZE 11 * 1024 * 1024 + +#if ((LIBCURL_VERSION_MAJOR >= 7) && (LIBCURL_VERSION_MINOR >= 16)) +#define OLD_CURL_MULTI 0 +#else +#define OLD_CURL_MULTI 1 +#endif + +/*****************************************************************************/ + +typedef struct swift_auth_info_struct swift_auth_info; +typedef struct connection_info_struct connection_info; +typedef struct socket_info_struct socket_info; +typedef struct global_io_info_struct global_io_info; +typedef struct slo_chunk_struct slo_chunk; +typedef struct container_list_struct container_list; +typedef struct object_info_struct object_info; + +struct swift_auth_info_struct { + char url[SWIFT_MAX_URL_SIZE]; + char token[SWIFT_MAX_HDR_SIZE]; +}; + +struct global_io_info_struct { + struct ev_loop *loop; + struct ev_io input_event; + struct ev_timer timer_event; + CURLM *multi; + int still_running; + int eof; + curl_socket_t input_fd; + connection_info **connections; + long chunk_no; + connection_info *current_connection; + const char *url; + const char *container; + const char *token; + const char *backup_name; +}; + +struct socket_info_struct { + curl_socket_t sockfd; + CURL *easy; + int action; + long timeout; + struct ev_io ev; + int evset; + global_io_info *global; +}; + +struct connection_info_struct { + CURL *easy; + global_io_info *global; + char *buffer; + size_t buffer_size; + size_t filled_size; + size_t upload_size; + bool chunk_uploaded; + bool chunk_acked; + char error[CURL_ERROR_SIZE]; + struct curl_slist *slist; + char *name; + size_t name_len; + char hash[33]; + size_t chunk_no; + bool magic_verified; + size_t chunk_path_len; + xb_chunk_type_t chunk_type; + size_t payload_size; + size_t chunk_size; + int retry_count; + bool upload_started; + ulong global_idx; +}; + +struct slo_chunk_struct { + char name[SWIFT_MAX_URL_SIZE]; + char md5[33]; + int idx; + size_t size; +}; + +struct object_info_struct { + char hash[33]; + char name[SWIFT_MAX_URL_SIZE]; + size_t bytes; +}; + +struct container_list_struct { + size_t content_length; + size_t content_bufsize; + char *content_json; + size_t object_count; + size_t idx; + object_info *objects; + bool final; +}; + +enum {SWIFT, S3}; +const char *storage_names[] = +{ "SWIFT", "S3", NullS}; + +static my_bool opt_verbose = 0; +static ulong opt_storage = SWIFT; +static const char *opt_swift_user = NULL; +static const char *opt_swift_user_id = NULL; +static const char *opt_swift_password = NULL; +static const char *opt_swift_tenant = NULL; +static const char *opt_swift_tenant_id = NULL; +static const char *opt_swift_project = NULL; +static const char *opt_swift_project_id = NULL; +static const char *opt_swift_domain = NULL; +static const char *opt_swift_domain_id = NULL; +static const char *opt_swift_region = NULL; +static const char *opt_swift_container = NULL; +static const char *opt_swift_storage_url = NULL; +static const char *opt_swift_auth_url = NULL; +static const char *opt_swift_key = NULL; +static const char *opt_swift_auth_version = NULL; +static const char *opt_name = NULL; +static const char *opt_cacert = NULL; +static ulong opt_parallel = 1; +static my_bool opt_insecure = 0; +static enum {MODE_GET, MODE_PUT, MODE_DELETE} opt_mode; + +static char **file_list = NULL; +static int file_list_size = 0; + +TYPELIB storage_typelib = +{array_elements(storage_names)-1, "", storage_names, NULL}; + +enum { + OPT_STORAGE = 256, + OPT_SWIFT_CONTAINER, + OPT_SWIFT_AUTH_URL, + OPT_SWIFT_KEY, + OPT_SWIFT_USER, + OPT_SWIFT_USER_ID, + OPT_SWIFT_PASSWORD, + OPT_SWIFT_TENANT, + OPT_SWIFT_TENANT_ID, + OPT_SWIFT_PROJECT, + OPT_SWIFT_PROJECT_ID, + OPT_SWIFT_DOMAIN, + OPT_SWIFT_DOMAIN_ID, + OPT_SWIFT_REGION, + OPT_SWIFT_STORAGE_URL, + OPT_SWIFT_AUTH_VERSION, + OPT_PARALLEL, + OPT_CACERT, + OPT_INSECURE, + OPT_VERBOSE +}; + + +static struct my_option my_long_options[] = +{ + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"storage", OPT_STORAGE, "Specify storage type S3/SWIFT.", + &opt_storage, &opt_storage, &storage_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"swift-auth-version", OPT_SWIFT_AUTH_VERSION, + "Swift authentication verison to use.", + &opt_swift_auth_version, &opt_swift_auth_version, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-container", OPT_SWIFT_CONTAINER, + "Swift container to store backups into.", + &opt_swift_container, &opt_swift_container, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-user", OPT_SWIFT_USER, + "Swift user name.", + &opt_swift_user, &opt_swift_user, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-user-id", OPT_SWIFT_USER_ID, + "Swift user ID.", + &opt_swift_user_id, &opt_swift_user_id, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-auth-url", OPT_SWIFT_AUTH_URL, + "Base URL of SWIFT authentication service.", + &opt_swift_auth_url, &opt_swift_auth_url, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-storage-url", OPT_SWIFT_STORAGE_URL, + "URL of object-store endpoint. Usually received from authentication " + "service. Specify to override this value.", + &opt_swift_storage_url, &opt_swift_storage_url, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-key", OPT_SWIFT_KEY, + "Swift key.", + &opt_swift_key, &opt_swift_key, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-tenant", OPT_SWIFT_TENANT, + "The tenant name. Both the --swift-tenant and --swift-tenant-id " + "options are optional, but should not be specified together.", + &opt_swift_tenant, &opt_swift_tenant, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-tenant-id", OPT_SWIFT_TENANT_ID, + "The tenant ID. Both the --swift-tenant and --swift-tenant-id " + "options are optional, but should not be specified together.", + &opt_swift_tenant_id, &opt_swift_tenant_id, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-project", OPT_SWIFT_PROJECT, + "The project name.", + &opt_swift_project, &opt_swift_project, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-project-id", OPT_SWIFT_PROJECT_ID, + "The project ID.", + &opt_swift_project_id, &opt_swift_project_id, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-domain", OPT_SWIFT_DOMAIN, + "The domain name.", + &opt_swift_domain, &opt_swift_domain, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-domain-id", OPT_SWIFT_DOMAIN_ID, + "The domain ID.", + &opt_swift_domain_id, &opt_swift_domain_id, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-password", OPT_SWIFT_PASSWORD, + "The password of the user.", + &opt_swift_password, &opt_swift_password, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-region", OPT_SWIFT_REGION, + "The region object-store endpoint.", + &opt_swift_region, &opt_swift_region, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"parallel", OPT_PARALLEL, + "Number of parallel chunk uploads.", + &opt_parallel, &opt_parallel, 0, GET_ULONG, REQUIRED_ARG, + 1, 0, 0, 0, 0, 0}, + + {"cacert", OPT_CACERT, + "CA certificate file.", + &opt_cacert, &opt_cacert, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"insecure", OPT_INSECURE, + "Do not verify server SSL certificate.", + &opt_insecure, &opt_insecure, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {"verbose", OPT_VERBOSE, + "Turn ON cURL tracing.", + &opt_verbose, &opt_verbose, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +/* The values of these arguments should be masked + on the command line */ +static const char * const masked_args[] = { + "--swift-password", + "--swift-key", + "--swift-auth-url", + "--swift-storage-url", + "--swift-container", + "--swift-user", + "--swift-tenant", + "--swift-user-id", + "--swift-tenant-id", + 0 +}; + +static map<string, ulonglong> file_chunk_count; + +static +void +print_version() +{ + printf("%s Ver %s for %s (%s)\n", my_progname, XBCLOUD_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); +} + +static +void +usage() +{ + print_version(); + puts("Copyright (C) 2015 Percona LLC and/or its affiliates."); + puts("This software comes with ABSOLUTELY NO WARRANTY. " + "This is free software,\nand you are welcome to modify and " + "redistribute it under the GPL license.\n"); + + puts("Manage backups on Cloud services.\n"); + + puts("Usage: "); + printf(" %s -c put [OPTIONS...] <NAME> upload backup from STDIN into " + "the cloud service with given name.\n", my_progname); + printf(" %s -c get [OPTIONS...] <NAME> [FILES...] stream specified " + "backup or individual files from cloud service into STDOUT.\n", + my_progname); + + puts("\nOptions:"); + my_print_help(my_long_options); +} + +static +my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))) +{ + switch (optid) { + case '?': + usage(); + exit(0); + } + + return(FALSE); +} + +static const char *load_default_groups[]= + { "xbcloud", 0 }; + +/*********************************************************************//** +mask sensitive values on the command line */ +static +void +mask_args(int argc, char **argv) +{ + int i; + for (i = 0; i < argc-1; i++) { + int j = 0; + if (argv[i]) while (masked_args[j]) { + char *p; + if ((p = strstr(argv[i], masked_args[j]))) { + p += strlen(masked_args[j]); + while (*p && *p != '=') { + p++; + } + if (*p == '=') { + p++; + while (*p) { + *p++ = 'x'; + } + } + } + j++; + } + } +} + +static +int parse_args(int argc, char **argv) +{ + const char *command; + + if (argc < 2) { + fprintf(stderr, "Command isn't specified. " + "Supported commands are put and get\n"); + usage(); + exit(EXIT_FAILURE); + } + + command = argv[1]; + argc--; argv++; + + if (strcasecmp(command, "put") == 0) { + opt_mode = MODE_PUT; + } else if (strcasecmp(command, "get") == 0) { + opt_mode = MODE_GET; + } else if (strcasecmp(command, "delete") == 0) { + opt_mode = MODE_DELETE; + } else { + fprintf(stderr, "Unknown command %s. " + "Supported commands are put and get\n", command); + usage(); + exit(EXIT_FAILURE); + } + + load_defaults_or_exit("my", load_default_groups, &argc, &argv); + + if (handle_options(&argc, &argv, my_long_options, get_one_option)) { + exit(EXIT_FAILURE); + } + + /* make sure name is specified */ + if (argc < 1) { + fprintf(stderr, "Backup name is required argument\n"); + exit(EXIT_FAILURE); + } + opt_name = argv[0]; + argc--; argv++; + + /* validate arguments */ + if (opt_storage == SWIFT) { + if (opt_swift_user == NULL) { + fprintf(stderr, "Swift user is not specified\n"); + exit(EXIT_FAILURE); + } + if (opt_swift_container == NULL) { + fprintf(stderr, + "Swift container is not specified\n"); + exit(EXIT_FAILURE); + } + if (opt_swift_auth_url == NULL) { + fprintf(stderr, "Swift auth URL is not specified\n"); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "Swift is only supported storage API\n"); + } + + if (argc > 0) { + file_list = argv; + file_list_size = argc; + } + + return(0); +} + +static char *hex_md5(const unsigned char *hash, char *out) +{ + enum { hash_len = 16 }; + char *p; + int i; + + for (i = 0, p = out; i < hash_len; i++, p+=2) { + sprintf(p, "%02x", hash[i]); + } + + return out; +} + +/* If header starts with prefix it's value will be copied into output buffer */ +static +int get_http_header(const char *prefix, const char *buffer, + char *out, size_t out_size) +{ + const char *beg, *end; + size_t len, prefix_len; + + prefix_len = strlen(prefix); + + if (strncasecmp(buffer, prefix, prefix_len) == 0) { + beg = buffer + prefix_len; + end = strchr(beg, '\r'); + + len = min<size_t>(end - beg, out_size - 1); + + strncpy(out, beg, len); + + out[len] = 0; + + return 1; + } + + return 0; +} + +static +size_t swift_auth_header_read_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + swift_auth_info *info = (swift_auth_info*)(data); + + get_http_header("X-Storage-Url: ", ptr, + info->url, array_elements(info->url)); + get_http_header("X-Auth-Token: ", ptr, + info->token, array_elements(info->token)); + + return nmemb * size; +} + +/*********************************************************************//** +Authenticate against Swift TempAuth. Fills swift_auth_info struct. +Uses creadentials privided as global variables. +@returns true if access is granted and token received. */ +static +bool +swift_temp_auth(const char *auth_url, swift_auth_info *info) +{ + CURL *curl; + CURLcode res; + long http_code; + char *hdr_buf = NULL; + struct curl_slist *slist = NULL; + + if (opt_swift_user == NULL) { + fprintf(stderr, "Swift user must be specified for TempAuth.\n"); + return(false); + } + + if (opt_swift_key == NULL) { + fprintf(stderr, "Swift key must be specified for TempAuth.\n"); + return(false); + } + + curl = curl_easy_init(); + + if (curl != NULL) { + + hdr_buf = (char *)(calloc(14 + max(strlen(opt_swift_user), + strlen(opt_swift_key)), 1)); + + if (!hdr_buf) { + res = CURLE_FAILED_INIT; + goto cleanup; + } + + sprintf(hdr_buf, "X-Auth-User: %s", opt_swift_user); + slist = curl_slist_append(slist, hdr_buf); + + sprintf(hdr_buf, "X-Auth-Key: %s", opt_swift_key); + slist = curl_slist_append(slist, hdr_buf); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, auth_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + swift_auth_header_read_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, info); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, "error: authentication failed: " + "curl_easy_perform(): %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 200 && + http_code != 204) { + fprintf(stderr, "error: authentication failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (hdr_buf) { + free(hdr_buf); + } + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + if (res == CURLE_OK) { + /* check that we received token and storage URL */ + if (*info->url == 0) { + fprintf(stderr, "error: malformed response: " + "X-Storage-Url is missing\n"); + return(false); + } + if (*info->token == 0) { + fprintf(stderr, "error: malformed response: " + "X-Auth-Token is missing\n"); + return(false); + } + return(true); + } + + return(false); +} + +static +size_t +write_null_cb(char *buffer, size_t size, size_t nmemb, void *stream) +{ + return fwrite(buffer, size, nmemb, stderr); +} + + +static +size_t +read_null_cb(char *ptr, size_t size, size_t nmemb, void *data) +{ + return 0; +} + + +static +int +swift_create_container(swift_auth_info *info, const char *name) +{ + char url[SWIFT_MAX_URL_SIZE]; + char auth_token[SWIFT_MAX_HDR_SIZE]; + CURLcode res; + long http_code; + CURL *curl; + struct curl_slist *slist = NULL; + + snprintf(url, array_elements(url), "%s/%s", info->url, name); + snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s", + info->token); + + curl = curl_easy_init(); + + if (curl != NULL) { + slist = curl_slist_append(slist, auth_token); + slist = curl_slist_append(slist, "Content-Length: 0"); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_null_cb); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_null_cb); + curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0L); + curl_easy_setopt(curl, CURLOPT_PUT, 1L); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 201 && /* created */ + http_code != 202 /* accepted (already exists) */) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + return res; +} + + +/*********************************************************************//** +Delete object with given url. +@returns true if object deleted successfully. */ +static +bool +swift_delete_object(swift_auth_info *info, const char *url) +{ + char auth_token[SWIFT_MAX_HDR_SIZE]; + CURLcode res; + long http_code; + CURL *curl; + struct curl_slist *slist = NULL; + bool ret = false; + + snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s", + info->token); + + curl = curl_easy_init(); + + if (curl != NULL) { + slist = curl_slist_append(slist, auth_token); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 200 && /* OK */ + http_code != 204 /* no content */) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + goto cleanup; + } + ret = true; + } else { + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + return ret; +} + +static int conn_upload_init(connection_info *conn); +static void conn_buffer_updated(connection_info *conn); +static connection_info *conn_new(global_io_info *global, ulong global_idx); +static void conn_cleanup(connection_info *conn); +static void conn_upload_retry(connection_info *conn); + +/* Check for completed transfers, and remove their easy handles */ +static void check_multi_info(global_io_info *g) +{ + char *eff_url; + CURLMsg *msg; + int msgs_left; + connection_info *conn; + CURL *easy; + + while ((msg = curl_multi_info_read(g->multi, &msgs_left))) { + if (msg->msg == CURLMSG_DONE) { + easy = msg->easy_handle; + curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn); + curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, + &eff_url); + curl_multi_remove_handle(g->multi, easy); + curl_easy_cleanup(easy); + conn->easy = NULL; + if (conn->chunk_acked) { + conn->chunk_uploaded = true; + fprintf(stderr, "%s is done\n", conn->hash); + } else { + fprintf(stderr, "error: chunk %zu '%s' %s " + "is not uploaded, but socket closed " + "(%zu bytes of %zu left to upload)\n", + conn->chunk_no, + conn->name, + conn->hash, + conn->chunk_size - conn->upload_size, + conn->chunk_size); + conn_upload_retry(conn); + } + } + } +} + +/* Die if we get a bad CURLMcode somewhere */ +static void mcode_or_die(const char *where, CURLMcode code) +{ + if (code != CURLM_OK) + { + const char *s; + switch (code) + { + case CURLM_BAD_HANDLE: + s = "CURLM_BAD_HANDLE"; + break; + case CURLM_BAD_EASY_HANDLE: + s = "CURLM_BAD_EASY_HANDLE"; + break; + case CURLM_OUT_OF_MEMORY: + s = "CURLM_OUT_OF_MEMORY"; + break; + case CURLM_INTERNAL_ERROR: + s = "CURLM_INTERNAL_ERROR"; + break; + case CURLM_UNKNOWN_OPTION: + s = "CURLM_UNKNOWN_OPTION"; + break; + case CURLM_LAST: + s = "CURLM_LAST"; + break; + default: + s = "CURLM_unknown"; + break; + case CURLM_BAD_SOCKET: + s = "CURLM_BAD_SOCKET"; + fprintf(stderr, "error: %s returns (%d) %s\n", + where, code, s); + /* ignore this error */ + return; + } + fprintf(stderr, "error: %s returns (%d) %s\n", + where, code, s); + assert(0); + } +} + +/* Called by libev when we get action on a multi socket */ +static void event_cb(EV_P_ struct ev_io *w, int revents) +{ + global_io_info *global = (global_io_info*)(w->data); + CURLMcode rc; + +#if !(OLD_CURL_MULTI) + int action = (revents & EV_READ ? CURL_POLL_IN : 0) | + (revents & EV_WRITE ? CURL_POLL_OUT : 0); + + do { + rc = curl_multi_socket_action(global->multi, w->fd, action, + &global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#else + do { + rc = curl_multi_socket(global->multi, w->fd, + &global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#endif + mcode_or_die("error: event_cb: curl_multi_socket_action", rc); + check_multi_info(global); + if (global->still_running <= 0) { + ev_timer_stop(global->loop, &global->timer_event); + } +} + +static void remsock(curl_socket_t s, socket_info *fdp, global_io_info *global) +{ + if (fdp) { + if (fdp->evset) { + ev_io_stop(global->loop, &fdp->ev); + } + free(fdp); + } +} + +static void setsock(socket_info *fdp, curl_socket_t s, CURL *easy, int action, + global_io_info *global) +{ + int kind = (action & CURL_POLL_IN ? (int)(EV_READ) : 0) | + (action & CURL_POLL_OUT ? (int)(EV_WRITE) : 0); + + fdp->sockfd = s; + fdp->action = action; + fdp->easy = easy; + if (fdp->evset) + ev_io_stop(global->loop, &fdp->ev); + ev_io_init(&fdp->ev, event_cb, fdp->sockfd, kind); + fdp->ev.data = global; + fdp->evset = 1; + ev_io_start(global->loop, &fdp->ev); +} + +static void addsock(curl_socket_t s, CURL *easy, int action, + global_io_info *global) +{ + socket_info *fdp = (socket_info *)(calloc(sizeof(socket_info), 1)); + + fdp->global = global; + setsock(fdp, s, easy, action, global); + curl_multi_assign(global->multi, s, fdp); +} + +static int sock_cb(CURL *easy, curl_socket_t s, int what, void *cbp, + void *sockp) +{ + global_io_info *global = (global_io_info*)(cbp); + socket_info *fdp = (socket_info*)(sockp); + + if (what == CURL_POLL_REMOVE) { + remsock(s, fdp, global); + } else { + if (!fdp) { + addsock(s, easy, what, global); + } else { + setsock(fdp, s, easy, what, global); + } + } + return 0; +} + +/* Called by libev when our timeout expires */ +static void timer_cb(EV_P_ struct ev_timer *w, int revents) +{ + global_io_info *io_global = (global_io_info*)(w->data); + CURLMcode rc; + +#if !(OLD_CURL_MULTI) + do { + rc = curl_multi_socket_action(io_global->multi, + CURL_SOCKET_TIMEOUT, 0, + &io_global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#else + do { + rc = curl_multi_socket_all(io_global->multi, + &io_global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#endif + mcode_or_die("timer_cb: curl_multi_socket_action", rc); + check_multi_info(io_global); +} + +static connection_info *get_current_connection(global_io_info *global) +{ + connection_info *conn = global->current_connection; + ulong i; + + if (conn && conn->filled_size < conn->chunk_size) + return conn; + + for (i = 0; i < opt_parallel; i++) { + conn = global->connections[i]; + if (conn->chunk_uploaded || conn->filled_size == 0) { + global->current_connection = conn; + conn_upload_init(conn); + return conn; + } + } + + return NULL; +} + +/* This gets called whenever data is received from the input */ +static void input_cb(EV_P_ struct ev_io *w, int revents) +{ + global_io_info *io_global = (global_io_info *)(w->data); + connection_info *conn = get_current_connection(io_global); + + if (conn == NULL) + return; + + if (conn->filled_size < conn->chunk_size) { + if (revents & EV_READ) { + ssize_t nbytes = read(io_global->input_fd, + conn->buffer + conn->filled_size, + conn->chunk_size - + conn->filled_size); + if (nbytes > 0) { + conn->filled_size += nbytes; + conn_buffer_updated(conn); + } else if (nbytes < 0) { + if (errno != EAGAIN && errno != EINTR) { + char error[200]; + my_strerror(error, sizeof(error), + errno); + fprintf(stderr, "error: failed to read " + "input stream (%s)\n", error); + /* failed to read input */ + exit(1); + } + } else { + io_global->eof = 1; + ev_io_stop(io_global->loop, w); + } + } + } + + assert(conn->filled_size <= conn->chunk_size); +} + +static int swift_upload_read_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + size_t realsize; + + connection_info *conn = (connection_info*)(data); + + if (conn->filled_size == conn->upload_size && + conn->upload_size < conn->chunk_size && !conn->global->eof) { + ssize_t nbytes; + assert(conn->global->current_connection == conn); + do { + nbytes = read(conn->global->input_fd, + conn->buffer + conn->filled_size, + conn->chunk_size - conn->filled_size); + } while (nbytes == -1 && errno == EAGAIN); + if (nbytes > 0) { + conn->filled_size += nbytes; + conn_buffer_updated(conn); + } else { + conn->global->eof = 1; + } + } + + realsize = min(size * nmemb, conn->filled_size - conn->upload_size); + + memcpy(ptr, conn->buffer + conn->upload_size, realsize); + conn->upload_size += realsize; + + assert(conn->filled_size <= conn->chunk_size); + assert(conn->upload_size <= conn->filled_size); + + return realsize; +} + +static +size_t upload_header_read_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + connection_info *conn = (connection_info *)(data); + char etag[33]; + + if (get_http_header("Etag: ", ptr, etag, array_elements(etag))) { + if (strcmp(conn->hash, etag) != 0) { + fprintf(stderr, "error: ETag mismatch\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "acked chunk %s\n", etag); + conn->chunk_acked = true; + } + + return nmemb * size; +} + +static int conn_upload_init(connection_info *conn) +{ + conn->filled_size = 0; + conn->upload_size = 0; + conn->chunk_uploaded = false; + conn->chunk_acked = false; + conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN; + conn->magic_verified = false; + conn->chunk_path_len = 0; + conn->chunk_type = XB_CHUNK_TYPE_UNKNOWN; + conn->payload_size = 0; + conn->upload_started = false; + conn->retry_count = 0; + if (conn->name != NULL) { + conn->name[0] = 0; + } + + if (conn->easy != NULL) { + conn->easy = 0; + } + + if (conn->slist != NULL) { + curl_slist_free_all(conn->slist); + conn->slist = NULL; + } + + return 0; +} + +static void conn_upload_prepare(connection_info *conn) +{ + gcry_md_hd_t md5; + + gcry_md_open(&md5, GCRY_MD_MD5, 0); + gcry_md_write(md5, conn->buffer, conn->chunk_size); + hex_md5(gcry_md_read(md5, GCRY_MD_MD5), conn->hash); + gcry_md_close(md5); +} + +static int conn_upload_start(connection_info *conn) +{ + char token_header[SWIFT_MAX_HDR_SIZE]; + char object_url[SWIFT_MAX_URL_SIZE]; + char content_len[200], etag[200]; + global_io_info *global; + CURLMcode rc; + + global = conn->global; + + fprintf(stderr, "uploading chunk %s/%s/%s.%020zu " + "(md5: %s, size: %zu)\n", + global->container, global->backup_name, conn->name, + conn->chunk_no, conn->hash, conn->chunk_size); + + snprintf(object_url, array_elements(object_url), "%s/%s/%s/%s.%020zu", + global->url, global->container, global->backup_name, + conn->name, conn->chunk_no); + + snprintf(content_len, sizeof(content_len), "Content-Length: %lu", + (ulong)(conn->chunk_size)); + + snprintf(etag, sizeof(etag), "ETag: %s", conn->hash); + + snprintf(token_header, array_elements(token_header), + "X-Auth-Token: %s", global->token); + + conn->slist = curl_slist_append(conn->slist, token_header); + conn->slist = curl_slist_append(conn->slist, + "Connection: keep-alive"); + conn->slist = curl_slist_append(conn->slist, + "Content-Type: " + "application/octet-stream"); + conn->slist = curl_slist_append(conn->slist, content_len); + conn->slist = curl_slist_append(conn->slist, etag); + + conn->easy = curl_easy_init(); + if (!conn->easy) { + fprintf(stderr, "error: curl_easy_init() failed\n"); + return 1; + } + curl_easy_setopt(conn->easy, CURLOPT_URL, object_url); + curl_easy_setopt(conn->easy, CURLOPT_READFUNCTION, + swift_upload_read_cb); + curl_easy_setopt(conn->easy, CURLOPT_READDATA, conn); + curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error); + curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn); + curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 1L); + curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_TIME, 5L); + curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_LIMIT, 1024L); + curl_easy_setopt(conn->easy, CURLOPT_PUT, 1L); + curl_easy_setopt(conn->easy, CURLOPT_HTTPHEADER, conn->slist); + curl_easy_setopt(conn->easy, CURLOPT_HEADERFUNCTION, + upload_header_read_cb); + curl_easy_setopt(conn->easy, CURLOPT_HEADERDATA, conn); + curl_easy_setopt(conn->easy, CURLOPT_INFILESIZE, + (long) conn->chunk_size); + if (opt_cacert != NULL) + curl_easy_setopt(conn->easy, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(conn->easy, CURLOPT_SSL_VERIFYPEER, FALSE); + + rc = curl_multi_add_handle(conn->global->multi, conn->easy); + mcode_or_die("conn_upload_init: curl_multi_add_handle", rc); + +#if (OLD_CURL_MULTI) + do { + rc = curl_multi_socket_all(global->multi, + &global->still_running); + } while(rc == CURLM_CALL_MULTI_PERFORM); +#endif + + conn->upload_started = true; + + return 0; +} + +static void conn_cleanup(connection_info *conn) +{ + if (conn) { + free(conn->name); + free(conn->buffer); + if (conn->slist) { + curl_slist_free_all(conn->slist); + conn->slist = NULL; + } + if (conn->easy) { + curl_easy_cleanup(conn->easy); + conn->easy = NULL; + } + } + free(conn); +} + +static void conn_upload_retry(connection_info *conn) +{ + /* already closed by cURL */ + conn->easy = NULL; + + if (conn->slist != NULL) { + curl_slist_free_all(conn->slist); + conn->slist = NULL; + } + + if (conn->retry_count++ > 3) { + fprintf(stderr, "error: retry count limit reached\n"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "warning: retrying to upload chunk %zu of '%s'\n", + conn->chunk_no, conn->name); + + conn->upload_size = 0; + + conn_upload_start(conn); +} + +static connection_info *conn_new(global_io_info *global, ulong global_idx) +{ + connection_info *conn; + + conn = (connection_info *)(calloc(1, sizeof(connection_info))); + if (conn == NULL) { + goto error; + } + + conn->global = global; + conn->global_idx = global_idx; + conn->buffer_size = SWIFT_CHUNK_SIZE; + if ((conn->buffer = (char *)(calloc(conn->buffer_size, 1))) == + NULL) { + goto error; + } + + return conn; + +error: + if (conn != NULL) { + conn_cleanup(conn); + } + + fprintf(stderr, "error: out of memory\n"); + exit(EXIT_FAILURE); + + return NULL; +} + +/*********************************************************************//** +Handle input buffer updates. Parse chunk header and set appropriate +buffer size. */ +static +void +conn_buffer_updated(connection_info *conn) +{ + bool ready_for_upload = false; + + /* chunk header */ + if (!conn->magic_verified && + conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN) { + if (strncmp(XB_STREAM_CHUNK_MAGIC, conn->buffer, + sizeof(XB_STREAM_CHUNK_MAGIC) - 1) != 0) { + + fprintf(stderr, "Error: magic expected\n"); + exit(EXIT_FAILURE); + } + conn->magic_verified = true; + conn->chunk_path_len = uint4korr(conn->buffer + + PATH_LENGTH_OFFSET); + conn->chunk_type = (xb_chunk_type_t) + (conn->buffer[CHUNK_TYPE_OFFSET]); + conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len; + if (conn->chunk_type != XB_CHUNK_TYPE_EOF) { + conn->chunk_size += 16; + } + } + + /* ordinary chunk */ + if (conn->magic_verified && + conn->payload_size == 0 && + conn->chunk_type != XB_CHUNK_TYPE_EOF && + conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len + 16) { + + conn->payload_size = uint8korr(conn->buffer + + CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len); + + conn->chunk_size = conn->payload_size + 4 + 16 + + conn->chunk_path_len + + CHUNK_HEADER_CONSTANT_LEN; + + if (conn->name == NULL) { + conn->name = (char*)(malloc(conn->chunk_path_len + 1)); + } else if (conn->name_len < conn->chunk_path_len + 1) { + conn->name = (char*)(realloc(conn->name, + conn->chunk_path_len + 1)); + } + conn->name_len = conn->chunk_path_len + 1; + + memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN, + conn->chunk_path_len); + conn->name[conn->chunk_path_len] = 0; + + if (conn->buffer_size < conn->chunk_size) { + conn->buffer = + (char *)(realloc(conn->buffer, conn->chunk_size)); + conn->buffer_size = conn->chunk_size; + } + } + + /* EOF chunk has no payload */ + if (conn->magic_verified && + conn->chunk_type == XB_CHUNK_TYPE_EOF && + conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len) { + + if (conn->name == NULL) { + conn->name = (char*)(malloc(conn->chunk_path_len + 1)); + } else if (conn->name_len < conn->chunk_path_len + 1) { + conn->name = (char*)(realloc(conn->name, + conn->chunk_path_len + 1)); + } + conn->name_len = conn->chunk_path_len + 1; + + memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN, + conn->chunk_path_len); + conn->name[conn->chunk_path_len] = 0; + } + + if (conn->filled_size > 0 && conn->filled_size == conn->chunk_size) { + ready_for_upload = true; + } + + /* start upload once recieved the size of the chunk */ + if (!conn->upload_started && ready_for_upload) { + conn->chunk_no = file_chunk_count[conn->name]++; + conn_upload_prepare(conn); + conn_upload_start(conn); + } +} + +static int init_input(global_io_info *io_global) +{ + ev_io_init(&io_global->input_event, input_cb, STDIN_FILENO, EV_READ); + io_global->input_event.data = io_global; + ev_io_start(io_global->loop, &io_global->input_event); + + return 0; +} + +/* Update the event timer after curl_multi library calls */ +static int multi_timer_cb(CURLM *multi, long timeout_ms, global_io_info *global) +{ + ev_timer_stop(global->loop, &global->timer_event); + if (timeout_ms > 0) { + double t = timeout_ms / 1000.0; + ev_timer_init(&global->timer_event, timer_cb, t, 0.); + ev_timer_start(global->loop, &global->timer_event); + } else { + timer_cb(global->loop, &global->timer_event, 0); + } + return 0; +} + +static +int swift_upload_parts(swift_auth_info *auth, const char *container, + const char *name) +{ + global_io_info io_global; + ulong i; +#if (OLD_CURL_MULTI) + long timeout; +#endif + CURLMcode rc; + int n_dirty_buffers; + + memset(&io_global, 0, sizeof(io_global)); + + io_global.loop = ev_default_loop(0); + init_input(&io_global); + io_global.multi = curl_multi_init(); + ev_timer_init(&io_global.timer_event, timer_cb, 0., 0.); + io_global.timer_event.data = &io_global; + io_global.connections = (connection_info **) + (calloc(opt_parallel, sizeof(connection_info))); + io_global.url = auth->url; + io_global.container = container; + io_global.backup_name = name; + io_global.token = auth->token; + for (i = 0; i < opt_parallel; i++) { + io_global.connections[i] = conn_new(&io_global, i); + } + + /* setup the generic multi interface options we want */ + curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb); + curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETDATA, &io_global); +#if !(OLD_CURL_MULTI) + curl_multi_setopt(io_global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb); + curl_multi_setopt(io_global.multi, CURLMOPT_TIMERDATA, &io_global); + do { + rc = curl_multi_socket_action(io_global.multi, + CURL_SOCKET_TIMEOUT, 0, + &io_global.still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#else + curl_multi_timeout(io_global.multi, &timeout); + if (timeout >= 0) { + multi_timer_cb(io_global.multi, timeout, &io_global); + } + do { + rc = curl_multi_socket_all(io_global.multi, &io_global.still_running); + } while(rc == CURLM_CALL_MULTI_PERFORM); +#endif + + ev_loop(io_global.loop, 0); + check_multi_info(&io_global); + curl_multi_cleanup(io_global.multi); + + n_dirty_buffers = 0; + for (i = 0; i < opt_parallel; i++) { + connection_info *conn = io_global.connections[i]; + if (conn && conn->upload_size != conn->filled_size) { + fprintf(stderr, "error: upload failed: %lu bytes left " + "in the buffer %s (uploaded = %d)\n", + (ulong)(conn->filled_size - conn->upload_size), + conn->name, conn->chunk_uploaded); + ++n_dirty_buffers; + } + } + + for (i = 0; i < opt_parallel; i++) { + if (io_global.connections[i] != NULL) { + conn_cleanup(io_global.connections[i]); + } + } + free(io_global.connections); + + if (n_dirty_buffers > 0) { + return(EXIT_FAILURE); + } + + return 0; +} + +struct download_buffer_info { + off_t offset; + size_t size; + size_t result_len; + char *buf; + curl_read_callback custom_header_callback; + void *custom_header_callback_data; +}; + +/*********************************************************************//** +Callback to parse header of GET request on swift contaier. */ +static +size_t fetch_buffer_header_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + download_buffer_info *buffer_info = (download_buffer_info*)(data); + size_t buf_size; + char content_length_str[100]; + char *endptr; + + if (get_http_header("Content-Length: ", ptr, + content_length_str, sizeof(content_length_str))) { + + buf_size = strtoull(content_length_str, &endptr, 10); + + if (buffer_info->buf == NULL) { + buffer_info->buf = (char*)(malloc(buf_size)); + buffer_info->size = buf_size; + } + + if (buf_size > buffer_info->size) { + buffer_info->buf = (char*) + (realloc(buffer_info->buf, buf_size)); + buffer_info->size = buf_size; + } + + buffer_info->result_len = buf_size; + } + + if (buffer_info->custom_header_callback) { + buffer_info->custom_header_callback(ptr, size, nmemb, + buffer_info->custom_header_callback_data); + } + + return nmemb * size; +} + +/*********************************************************************//** +Write contents into string buffer */ +static +size_t +fetch_buffer_cb(char *buffer, size_t size, size_t nmemb, void *out_buffer) +{ + download_buffer_info *buffer_info = (download_buffer_info*)(out_buffer); + + assert(buffer_info->size >= buffer_info->offset + size * nmemb); + + memcpy(buffer_info->buf + buffer_info->offset, buffer, size * nmemb); + buffer_info->offset += size * nmemb; + + return size * nmemb; +} + + +/*********************************************************************//** +Downloads contents of URL into buffer. Caller is responsible for +deallocating the buffer. +@return pointer to a buffer or NULL */ +static +char * +swift_fetch_into_buffer(swift_auth_info *auth, const char *url, + char **buf, size_t *buf_size, size_t *result_len, + curl_read_callback header_callback, + void *header_callback_data) +{ + char auth_token[SWIFT_MAX_HDR_SIZE]; + download_buffer_info buffer_info; + struct curl_slist *slist = NULL; + long http_code; + CURL *curl; + CURLcode res; + + memset(&buffer_info, 0, sizeof(buffer_info)); + buffer_info.buf = *buf; + buffer_info.size = *buf_size; + buffer_info.custom_header_callback = header_callback; + buffer_info.custom_header_callback_data = header_callback_data; + + snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s", + auth->token); + + curl = curl_easy_init(); + + if (curl != NULL) { + slist = curl_slist_append(slist, auth_token); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer_info); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + fetch_buffer_header_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, + &buffer_info); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 300) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + if (res == CURLE_OK) { + *buf = buffer_info.buf; + *buf_size = buffer_info.size; + *result_len = buffer_info.result_len; + return(buffer_info.buf); + } + + free(buffer_info.buf); + *buf = NULL; + *buf_size = 0; + *result_len = 0; + + return(NULL); +} + +static +container_list * +container_list_new() +{ + container_list *list = + (container_list *)(calloc(1, sizeof(container_list))); + + list->object_count = 1000; + list->objects = (object_info*) + (calloc(list->object_count, sizeof(object_info))); + + if (list->objects == NULL) { + fprintf(stderr, "error: out of memory\n"); + free(list); + return(NULL); + } + + return(list); +} + +static +void +container_list_free(container_list *list) +{ + free(list->content_json); + free(list->objects); + free(list); +} + +static +void +container_list_add_object(container_list *list, const char *name, + const char *hash, size_t bytes) +{ + const size_t object_count_step = 1000; + + if (list->idx >= list->object_count) { + list->objects = (object_info*) + realloc(list->objects, + (list->object_count + object_count_step) * + sizeof(object_info)); + memset(list->objects + list->object_count, 0, + object_count_step * sizeof(object_info)); + list->object_count += object_count_step; + } + assert(list->idx <= list->object_count); + safe_strcpy(list->objects[list->idx].name, + sizeof(list->objects[list->idx].name), name); + safe_strcpy(list->objects[list->idx].hash, + sizeof(list->objects[list->idx].hash), hash); + + list->objects[list->idx].bytes = bytes; + ++list->idx; +} + + +/*********************************************************************//** +Tokenize json string. Return array of tokens. Caller is responsoble for +deallocating the array. */ +jsmntok_t * +json_tokenise(char *json, size_t len, int initial_tokens) +{ + jsmn_parser parser; + jsmn_init(&parser); + + unsigned int n = initial_tokens; + jsmntok_t *tokens = (jsmntok_t *)(malloc(sizeof(jsmntok_t) * n)); + + int ret = jsmn_parse(&parser, json, len, tokens, n); + + while (ret == JSMN_ERROR_NOMEM) + { + n = n * 2 + 1; + tokens = (jsmntok_t*)(realloc(tokens, sizeof(jsmntok_t) * n)); + ret = jsmn_parse(&parser, json, len, tokens, n); + } + + if (ret == JSMN_ERROR_INVAL) { + fprintf(stderr, "error: invalid JSON string\n"); + + } + if (ret == JSMN_ERROR_PART) { + fprintf(stderr, "error: truncated JSON string\n"); + } + + return tokens; +} + +/*********************************************************************//** +Return true if token representation equal to given string. */ +static +bool +json_token_eq(const char *buf, jsmntok_t *t, const char *s) +{ + size_t len = strlen(s); + + assert(t->end > t->start); + + return((size_t)(t->end - t->start) == len && + (strncmp(buf + t->start, s, len) == 0)); +} + +/*********************************************************************//** +Copy given token as string. */ +static +bool +json_token_str(const char *buf, jsmntok_t *t, char *out, int out_size) +{ + size_t len = min(t->end - t->start, out_size - 1); + + memcpy(out, buf + t->start, len); + out[len] = 0; + + return(true); +} + +/*********************************************************************//** +Parse SWIFT container list response and fill output array with values +sorted by object name. */ +static +bool +swift_parse_container_list(container_list *list) +{ + enum {MAX_DEPTH=20}; + enum label_t {NONE, OBJECT}; + + char name[SWIFT_MAX_URL_SIZE]; + char hash[33]; + char bytes[30]; + char *response = list->content_json; + + struct stack_t { + jsmntok_t *t; + int n_items; + label_t label; + }; + + stack_t stack[MAX_DEPTH]; + jsmntok_t *tokens; + int level; + size_t count = 0; + + tokens = json_tokenise(list->content_json, list->content_length, 200); + + stack[0].t = &tokens[0]; + stack[0].label = NONE; + stack[0].n_items = 1; + level = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) { + jsmntok_t *t = &tokens[i]; + + assert(t->start != -1 && t->end != -1); + assert(level >= 0); + + --stack[level].n_items; + + switch (t->type) { + case JSMN_ARRAY: + case JSMN_OBJECT: + if (level < MAX_DEPTH - 1) { + level++; + } + stack[level].t = t; + stack[level].label = NONE; + if (t->type == JSMN_ARRAY) { + stack[level].n_items = t->size; + j += t->size; + } else { + stack[level].n_items = t->size * 2; + j += t->size * 2; + } + break; + case JSMN_PRIMITIVE: + case JSMN_STRING: + if (stack[level].t->type == JSMN_OBJECT && + stack[level].n_items % 2 == 1) { + /* key */ + if (json_token_eq(response, t, "name")) { + json_token_str(response, &tokens[i + 1], + name, sizeof(name)); + } + if (json_token_eq(response, t, "hash")) { + json_token_str(response, &tokens[i + 1], + hash, sizeof(hash)); + } + if (json_token_eq(response, t, "bytes")) { + json_token_str(response, &tokens[i + 1], + bytes, sizeof(bytes)); + } + } + break; + } + + while (stack[level].n_items == 0 && level > 0) { + if (stack[level].t->type == JSMN_OBJECT + && level == 2) { + char *endptr; + container_list_add_object(list, name, hash, + strtoull(bytes, &endptr, 10)); + ++count; + } + --level; + } + } + + if (count == 0) { + list->final = true; + } + + free(tokens); + + return(true); +} + +/*********************************************************************//** +List swift container with given name. Return list of objects sorted by +object name. */ +static +container_list * +swift_list(swift_auth_info *auth, const char *container, const char *path) +{ + container_list *list; + char url[SWIFT_MAX_URL_SIZE]; + + list = container_list_new(); + + while (!list->final) { + + /* download the list in json format */ + snprintf(url, array_elements(url), + "%s/%s?format=json&limit=1000%s%s%s%s", + auth->url, container, path ? "&prefix=" : "", + path ? path : "", list->idx > 0 ? "&marker=" : "", + list->idx > 0 ? + list->objects[list->idx - 1].name : ""); + + list->content_json = swift_fetch_into_buffer(auth, url, + &list->content_json, &list->content_bufsize, + &list->content_length, NULL, NULL); + + if (list->content_json == NULL) { + container_list_free(list); + return(NULL); + } + + /* parse downloaded list */ + if (!swift_parse_container_list(list)) { + fprintf(stderr, "error: unable to parse " + "container list\n"); + container_list_free(list); + return(NULL); + } + } + + return(list); +} + + +/*********************************************************************//** +Return true if chunk is a part of backup with given name. */ +static +bool +chunk_belongs_to(const char *chunk_name, const char *backup_name) +{ + size_t backup_name_len = strlen(backup_name); + + return((strlen(chunk_name) > backup_name_len) + && (chunk_name[backup_name_len] == '/') + && strncmp(chunk_name, backup_name, backup_name_len) == 0); +} + +/*********************************************************************//** +Return true if chunk is in given list. */ +static +bool +chunk_in_list(const char *chunk_name, char **list, int list_size) +{ + size_t chunk_name_len; + + if (list_size == 0) { + return(true); + } + + chunk_name_len = strlen(chunk_name); + if (chunk_name_len < 20) { + return(false); + } + + for (int i = 0; i < list_size; i++) { + size_t item_len = strlen(list[i]); + + if ((strncmp(chunk_name - item_len + chunk_name_len - 21, + list[i], item_len) == 0) + && (chunk_name[chunk_name_len - 21] == '.') + && (chunk_name[chunk_name_len - item_len - 22] == '/')) { + return(true); + } + } + + return(false); +} + +static +int swift_download(swift_auth_info *auth, const char *container, + const char *name) +{ + container_list *list; + char *buf = NULL; + size_t buf_size = 0; + size_t result_len = 0; + + if ((list = swift_list(auth, container, name)) == NULL) { + return(CURLE_FAILED_INIT); + } + + for (size_t i = 0; i < list->idx; i++) { + const char *chunk_name = list->objects[i].name; + + if (chunk_belongs_to(chunk_name, name) + && chunk_in_list(chunk_name, file_list, file_list_size)) { + char url[SWIFT_MAX_URL_SIZE]; + + snprintf(url, sizeof(url), "%s/%s/%s", + auth->url, container, chunk_name); + + if ((buf = swift_fetch_into_buffer( + auth, url, &buf, &buf_size, &result_len, + NULL, NULL)) == NULL) { + fprintf(stderr, "error: failed to download " + "chunk %s\n", chunk_name); + container_list_free(list); + return(CURLE_FAILED_INIT); + } + + fwrite(buf, 1, result_len, stdout); + } + } + + free(buf); + + container_list_free(list); + + return(CURLE_OK); +} + + +/*********************************************************************//** +Delete backup with given name from given container. +@return true if backup deleted successfully */ +static +bool swift_delete(swift_auth_info *auth, const char *container, + const char *name) +{ + container_list *list; + + if ((list = swift_list(auth, container, name)) == NULL) { + return(CURLE_FAILED_INIT); + } + + for (size_t i = 0; i < list->object_count; i++) { + const char *chunk_name = list->objects[i].name; + + if (chunk_belongs_to(chunk_name, name)) { + char url[SWIFT_MAX_URL_SIZE]; + + snprintf(url, sizeof(url), "%s/%s/%s", + auth->url, container, chunk_name); + + fprintf(stderr, "delete %s\n", chunk_name); + if (!swift_delete_object(auth, url)) { + fprintf(stderr, "error: failed to delete " + "chunk %s\n", chunk_name); + container_list_free(list); + return(CURLE_FAILED_INIT); + } + } + } + + container_list_free(list); + + return(CURLE_OK); +} + +/*********************************************************************//** +Check if backup with given name exists. +@return true if backup exists */ +static +bool swift_backup_exists(swift_auth_info *auth, const char *container, + const char *backup_name) +{ + container_list *list; + + if ((list = swift_list(auth, container, backup_name)) == NULL) { + fprintf(stderr, "error: unable to list container %s\n", + container); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; i < list->object_count; i++) { + if (chunk_belongs_to(list->objects[i].name, backup_name)) { + container_list_free(list); + return(true); + } + } + + container_list_free(list); + + return(false); +} + +/*********************************************************************//** +Fills auth_info with response from keystone response. +@return true is response parsed successfully */ +static +bool +swift_parse_keystone_response_v2(char *response, size_t response_length, + swift_auth_info *auth_info) +{ + enum {MAX_DEPTH=20}; + enum label_t {NONE, ACCESS, CATALOG, ENDPOINTS, TOKEN}; + + char filtered_url[SWIFT_MAX_URL_SIZE]; + char public_url[SWIFT_MAX_URL_SIZE]; + char region[SWIFT_MAX_URL_SIZE]; + char id[SWIFT_MAX_URL_SIZE]; + char token_id[SWIFT_MAX_URL_SIZE]; + char type[SWIFT_MAX_URL_SIZE]; + + struct stack_t { + jsmntok_t *t; + int n_items; + label_t label; + }; + + stack_t stack[MAX_DEPTH]; + jsmntok_t *tokens; + int level; + + tokens = json_tokenise(response, response_length, 200); + + stack[0].t = &tokens[0]; + stack[0].label = NONE; + stack[0].n_items = 1; + level = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) { + jsmntok_t *t = &tokens[i]; + + assert(t->start != -1 && t->end != -1); + assert(level >= 0); + + --stack[level].n_items; + + switch (t->type) { + case JSMN_ARRAY: + case JSMN_OBJECT: + if (level < MAX_DEPTH - 1) { + level++; + } + stack[level].t = t; + stack[level].label = NONE; + if (t->type == JSMN_ARRAY) { + stack[level].n_items = t->size; + j += t->size; + } else { + stack[level].n_items = t->size * 2; + j += t->size * 2; + } + break; + case JSMN_PRIMITIVE: + case JSMN_STRING: + if (stack[level].t->type == JSMN_OBJECT && + stack[level].n_items % 2 == 1) { + /* key */ + if (json_token_eq(response, t, "access")) { + stack[level].label = ACCESS; + } + if (json_token_eq(response, t, + "serviceCatalog")) { + stack[level].label = CATALOG; + } + if (json_token_eq(response, t, "endpoints")) { + stack[level].label = ENDPOINTS; + } + if (json_token_eq(response, t, "token")) { + stack[level].label = TOKEN; + } + if (json_token_eq(response, t, "id")) { + json_token_str(response, &tokens[i + 1], + id, sizeof(id)); + } + if (json_token_eq(response, t, "id") + && stack[level - 1].label == TOKEN) { + json_token_str(response, &tokens[i + 1], + token_id, sizeof(token_id)); + } + if (json_token_eq(response, t, "region")) { + json_token_str(response, &tokens[i + 1], + region, sizeof(region)); + } + if (json_token_eq(response, t, "publicURL")) { + json_token_str(response, &tokens[i + 1], + public_url, sizeof(public_url)); + } + if (json_token_eq(response, t, "type")) { + json_token_str(response, &tokens[i + 1], + type, sizeof(type)); + } + } + break; + } + + while (stack[level].n_items == 0 && level > 0) { + if (stack[level].t->type == JSMN_OBJECT + && level == 6 + && stack[level - 1].t->type == JSMN_ARRAY + && stack[level - 2].label == ENDPOINTS) { + if (opt_swift_region == NULL + || strcmp(opt_swift_region, region) == 0) { + strncpy(filtered_url, public_url, + sizeof(filtered_url)); + } + } + if (stack[level].t->type == JSMN_OBJECT && + level == 4 && + stack[level - 1].t->type == JSMN_ARRAY && + stack[level - 2].label == CATALOG) { + if (strcmp(type, "object-store") == 0) { + strncpy(auth_info->url, filtered_url, + sizeof(auth_info->url)); + } + } + --level; + } + } + + free(tokens); + + strncpy(auth_info->token, token_id, sizeof(auth_info->token)); + + assert(level == 0); + + if (*auth_info->token == 0) { + fprintf(stderr, "error: can not receive token from response\n"); + return(false); + } + + if (*auth_info->url == 0) { + fprintf(stderr, "error: can not get URL from response\n"); + return(false); + } + + return(true); +} + +/*********************************************************************//** +Authenticate against Swift TempAuth. Fills swift_auth_info struct. +Uses creadentials privided as global variables. +@returns true if access is granted and token received. */ +static +bool +swift_keystone_auth_v2(const char *auth_url, swift_auth_info *info) +{ + char tenant_arg[SWIFT_MAX_URL_SIZE]; + char payload[SWIFT_MAX_URL_SIZE]; + struct curl_slist *slist = NULL; + download_buffer_info buf_info; + long http_code; + CURLcode res; + CURL *curl; + bool auth_res = false; + + memset(&buf_info, 0, sizeof(buf_info)); + + if (opt_swift_user == NULL) { + fprintf(stderr, "error: both --swift-user is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_password == NULL) { + fprintf(stderr, "error: both --swift-password is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_tenant != NULL && opt_swift_tenant_id != NULL) { + fprintf(stderr, "error: both --swift-tenant and " + "--swift-tenant-id specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_tenant != NULL) { + snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"", + "tenantName", opt_swift_tenant); + } else if (opt_swift_tenant_id != NULL) { + snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"", + "tenantId", opt_swift_tenant_id); + } else { + *tenant_arg = 0; + } + + snprintf(payload, sizeof(payload), "{\"auth\": " + "{\"passwordCredentials\": {\"username\":\"%s\"," + "\"password\":\"%s\"}%s}}", + opt_swift_user, opt_swift_password, tenant_arg); + + curl = curl_easy_init(); + + if (curl != NULL) { + + slist = curl_slist_append(slist, + "Content-Type: application/json"); + slist = curl_slist_append(slist, + "Accept: application/json"); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_URL, auth_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + fetch_buffer_header_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, + &buf_info); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 300) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + + if (!swift_parse_keystone_response_v2(buf_info.buf, + buf_info.size, info)) { + goto cleanup; + } + + auth_res = true; + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + free(buf_info.buf); + + return(auth_res); +} + + +/*********************************************************************//** +Fills auth_info with response from keystone response. +@return true is response parsed successfully */ +static +bool +swift_parse_keystone_response_v3(char *response, size_t response_length, + swift_auth_info *auth_info) +{ + enum {MAX_DEPTH=20}; + enum label_t {NONE, TOKEN, CATALOG, ENDPOINTS}; + + char url[SWIFT_MAX_URL_SIZE]; + char filtered_url[SWIFT_MAX_URL_SIZE]; + char region[SWIFT_MAX_URL_SIZE]; + char interface[SWIFT_MAX_URL_SIZE]; + char type[SWIFT_MAX_URL_SIZE]; + + struct stack_t { + jsmntok_t *t; + int n_items; + label_t label; + }; + + stack_t stack[MAX_DEPTH]; + jsmntok_t *tokens; + int level; + + tokens = json_tokenise(response, response_length, 200); + + stack[0].t = &tokens[0]; + stack[0].label = NONE; + stack[0].n_items = 1; + level = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) { + jsmntok_t *t = &tokens[i]; + + assert(t->start != -1 && t->end != -1); + assert(level >= 0); + + --stack[level].n_items; + + switch (t->type) { + case JSMN_ARRAY: + case JSMN_OBJECT: + if (level < MAX_DEPTH - 1) { + level++; + } + stack[level].t = t; + stack[level].label = NONE; + if (t->type == JSMN_ARRAY) { + stack[level].n_items = t->size; + j += t->size; + } else { + stack[level].n_items = t->size * 2; + j += t->size * 2; + } + break; + case JSMN_PRIMITIVE: + case JSMN_STRING: + if (stack[level].t->type == JSMN_OBJECT && + stack[level].n_items % 2 == 1) { + /* key */ + if (json_token_eq(response, t, "token")) { + stack[level].label = TOKEN; + fprintf(stderr, "token\n"); + } + if (json_token_eq(response, t, + "catalog")) { + stack[level].label = CATALOG; + fprintf(stderr, "catalog\n"); + } + if (json_token_eq(response, t, "endpoints")) { + stack[level].label = ENDPOINTS; + } + if (json_token_eq(response, t, "region")) { + json_token_str(response, &tokens[i + 1], + region, sizeof(region)); + } + if (json_token_eq(response, t, "url")) { + json_token_str(response, &tokens[i + 1], + url, sizeof(url)); + } + if (json_token_eq(response, t, "interface")) { + json_token_str(response, &tokens[i + 1], + interface, sizeof(interface)); + } + if (json_token_eq(response, t, "type")) { + json_token_str(response, &tokens[i + 1], + type, sizeof(type)); + } + } + break; + } + + while (stack[level].n_items == 0 && level > 0) { + if (stack[level].t->type == JSMN_OBJECT + && level == 6 + && stack[level - 1].t->type == JSMN_ARRAY + && stack[level - 2].label == ENDPOINTS) { + if ((opt_swift_region == NULL + || strcmp(opt_swift_region, region) == 0) + && strcmp(interface, "public") == 0) { + strncpy(filtered_url, url, + sizeof(filtered_url)); + } + } + if (stack[level].t->type == JSMN_OBJECT && + level == 4 && + stack[level - 1].t->type == JSMN_ARRAY && + stack[level - 2].label == CATALOG) { + if (strcmp(type, "object-store") == 0) { + strncpy(auth_info->url, filtered_url, + sizeof(auth_info->url)); + } + } + --level; + } + } + + free(tokens); + + assert(level == 0); + + if (*auth_info->url == 0) { + fprintf(stderr, "error: can not get URL from response\n"); + return(false); + } + + return(true); +} + +/*********************************************************************//** +Captures X-Subject-Token header. */ +static +size_t keystone_v3_header_cb(char *ptr, size_t size, size_t nmemb, void *data) +{ + swift_auth_info *info = (swift_auth_info*)(data); + + get_http_header("X-Subject-Token: ", ptr, + info->token, array_elements(info->token)); + + return nmemb * size; +} + +/*********************************************************************//** +Authenticate against Swift TempAuth. Fills swift_auth_info struct. +Uses creadentials privided as global variables. +@returns true if access is granted and token received. */ +static +bool +swift_keystone_auth_v3(const char *auth_url, swift_auth_info *info) +{ + char scope[SWIFT_MAX_URL_SIZE]; + char domain[SWIFT_MAX_URL_SIZE]; + char payload[SWIFT_MAX_URL_SIZE]; + struct curl_slist *slist = NULL; + download_buffer_info buf_info; + long http_code; + CURLcode res; + CURL *curl; + bool auth_res = false; + + memset(&buf_info, 0, sizeof(buf_info)); + buf_info.custom_header_callback = keystone_v3_header_cb; + buf_info.custom_header_callback_data = info; + + if (opt_swift_user == NULL) { + fprintf(stderr, "error: both --swift-user is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_password == NULL) { + fprintf(stderr, "error: both --swift-password is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_project_id != NULL && opt_swift_project != NULL) { + fprintf(stderr, "error: both --swift-project and " + "--swift-project-id specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_domain_id != NULL && opt_swift_domain != NULL) { + fprintf(stderr, "error: both --swift-domain and " + "--swift-domain-id specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_project_id != NULL && opt_swift_domain != NULL) { + fprintf(stderr, "error: both --swift-project-id and " + "--swift-domain specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_project_id != NULL && opt_swift_domain_id != NULL) { + fprintf(stderr, "error: both --swift-project-id and " + "--swift-domain-id specified for keystone " + "authentication.\n"); + return(false); + } + + scope[0] = 0; domain[0] = 0; + + if (opt_swift_domain != NULL) { + snprintf(domain, sizeof(domain), + ",{\"domain\":{\"name\":\"%s\"}}", + opt_swift_domain); + } else if (opt_swift_domain_id != NULL) { + snprintf(domain, sizeof(domain), + ",{\"domain\":{\"id\":\"%s\"}}", + opt_swift_domain_id); + } + + if (opt_swift_project_id != NULL) { + snprintf(scope, sizeof(scope), + ",\"scope\":{\"project\":{\"id\":\"%s\"}}", + opt_swift_project_id); + } else if (opt_swift_project != NULL) { + snprintf(scope, sizeof(scope), + ",\"scope\":{\"project\":{\"name\":\"%s\"%s}}", + opt_swift_project, domain); + } + + snprintf(payload, sizeof(payload), "{\"auth\":{\"identity\":" + "{\"methods\":[\"password\"],\"password\":{\"user\":" + "{\"name\":\"%s\",\"password\":\"%s\"%s}}}%s}}", + opt_swift_user, opt_swift_password, + *scope ? "" : ",\"domain\":{\"id\":\"default\"}", + scope); + + curl = curl_easy_init(); + + if (curl != NULL) { + + slist = curl_slist_append(slist, + "Content-Type: application/json"); + slist = curl_slist_append(slist, + "Accept: application/json"); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_URL, auth_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + fetch_buffer_header_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, + &buf_info); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 300) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + + if (!swift_parse_keystone_response_v3(buf_info.buf, + buf_info.size, info)) { + goto cleanup; + } + + auth_res = true; + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + free(buf_info.buf); + + return(auth_res); +} + +int main(int argc, char **argv) +{ + swift_auth_info info; + char auth_url[SWIFT_MAX_URL_SIZE]; + + MY_INIT(argv[0]); + + /* handle_options in parse_args is destructive so + * we make a copy of our argument pointers so we can + * mask the sensitive values afterwards */ + char **mask_argv = (char **)malloc(sizeof(char *) * (argc - 1)); + memcpy(mask_argv, argv + 1, sizeof(char *) * (argc - 1)); + + if (parse_args(argc, argv)) { + return(EXIT_FAILURE); + } + + mask_args(argc, mask_argv); /* mask args on cmdline */ + + curl_global_init(CURL_GLOBAL_ALL); + + if (opt_swift_auth_version == NULL || *opt_swift_auth_version == '1') { + /* TempAuth */ + snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sauth/v%s/", + opt_swift_auth_url, opt_swift_auth_version ? + opt_swift_auth_version : "1.0"); + + if (!swift_temp_auth(auth_url, &info)) { + fprintf(stderr, "error: failed to authenticate\n"); + return(EXIT_FAILURE); + } + + } else if (*opt_swift_auth_version == '2') { + /* Keystone v2 */ + snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/tokens", + opt_swift_auth_url, opt_swift_auth_version); + + if (!swift_keystone_auth_v2(auth_url, &info)) { + fprintf(stderr, "error: failed to authenticate\n"); + return(EXIT_FAILURE); + } + + } else if (*opt_swift_auth_version == '3') { + /* Keystone v3 */ + snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/auth/tokens", + opt_swift_auth_url, opt_swift_auth_version); + + if (!swift_keystone_auth_v3(auth_url, &info)) { + fprintf(stderr, "error: failed to authenticate\n"); + exit(EXIT_FAILURE); + } + + } + + if (opt_swift_storage_url != NULL) { + snprintf(info.url, sizeof(info.url), "%s", + opt_swift_storage_url); + } + + fprintf(stderr, "Object store URL: %s\n", info.url); + + if (opt_mode == MODE_PUT) { + + if (swift_create_container(&info, opt_swift_container) != 0) { + fprintf(stderr, "error: failed to create " + "container %s\n", + opt_swift_container); + return(EXIT_FAILURE); + } + + if (swift_backup_exists(&info, opt_swift_container, opt_name)) { + fprintf(stderr, "error: backup named '%s' " + "already exists!\n", + opt_name); + return(EXIT_FAILURE); + } + + if (swift_upload_parts(&info, opt_swift_container, + opt_name) != 0) { + fprintf(stderr, "error: upload failed\n"); + return(EXIT_FAILURE); + } + + } else if (opt_mode == MODE_GET) { + + if (swift_download(&info, opt_swift_container, opt_name) + != CURLE_OK) { + fprintf(stderr, "error: download failed\n"); + return(EXIT_FAILURE); + } + + } else if (opt_mode == MODE_DELETE) { + + if (swift_delete(&info, opt_swift_container, opt_name) + != CURLE_OK) { + fprintf(stderr, "error: delete failed\n"); + return(EXIT_FAILURE); + } + + } else { + fprintf(stderr, "Unknown command supplied.\n"); + exit(EXIT_FAILURE); + } + + curl_global_cleanup(); + + return(EXIT_SUCCESS); +} diff --git a/extra/mariabackup/xbstream.cc b/extra/mariabackup/xbstream.cc new file mode 100644 index 00000000..6306806b --- /dev/null +++ b/extra/mariabackup/xbstream.cc @@ -0,0 +1,560 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +The xbstream utility: serialize/deserialize files in the XBSTREAM format. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <my_base.h> +#include <my_getopt.h> +#include <hash.h> +#include <my_pthread.h> +#include "common.h" +#include "xbstream.h" +#include "datasink.h" + +#define XBSTREAM_VERSION "1.0" +#define XBSTREAM_BUFFER_SIZE (10 * 1024 * 1024UL) + +#define START_FILE_HASH_SIZE 16 + +typedef enum { + RUN_MODE_NONE, + RUN_MODE_CREATE, + RUN_MODE_EXTRACT +} run_mode_t; + +/* Need the following definitions to avoid linking with ds_*.o and their link +dependencies */ +datasink_t datasink_archive; +datasink_t datasink_xbstream; +datasink_t datasink_compress; +datasink_t datasink_tmpfile; + +static run_mode_t opt_mode; +static char * opt_directory = NULL; +static my_bool opt_verbose = 0; +static int opt_parallel = 1; + +static struct my_option my_long_options[] = +{ + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"create", 'c', "Stream the specified files to the standard output.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"extract", 'x', "Extract to disk files from the stream on the " + "standard input.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"directory", 'C', "Change the current directory to the specified one " + "before streaming or extracting.", &opt_directory, &opt_directory, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Print verbose output.", &opt_verbose, &opt_verbose, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"parallel", 'p', "Number of worker threads for reading / writing.", + &opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG, + 1, 1, INT_MAX, 0, 0, 0}, + + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +typedef struct { + HASH *filehash; + xb_rstream_t *stream; + ds_ctxt_t *ds_ctxt; + pthread_mutex_t *mutex; +} extract_ctxt_t; + +typedef struct { + char *path; + uint pathlen; + my_off_t offset; + ds_file_t *file; + pthread_mutex_t mutex; +} file_entry_t; + +static int get_options(int *argc, char ***argv); +static int mode_create(int argc, char **argv); +static int mode_extract(int n_threads, int argc, char **argv); +static my_bool get_one_option(const struct my_option *opt, + const char *argument, const char *filename); + +int +main(int argc, char **argv) +{ + MY_INIT(argv[0]); + + if (get_options(&argc, &argv)) { + goto err; + } + + if (opt_mode == RUN_MODE_NONE) { + msg("%s: either -c or -x must be specified.", my_progname); + goto err; + } + + /* Change the current directory if -C is specified */ + if (opt_directory && my_setwd(opt_directory, MYF(MY_WME))) { + goto err; + } + + if (opt_mode == RUN_MODE_CREATE && mode_create(argc, argv)) { + goto err; + } else if (opt_mode == RUN_MODE_EXTRACT && + mode_extract(opt_parallel, argc, argv)) { + goto err; + } + + my_cleanup_options(my_long_options); + + my_end(0); + + return EXIT_SUCCESS; +err: + my_cleanup_options(my_long_options); + + my_end(0); + + exit(EXIT_FAILURE); +} + +static +int +get_options(int *argc, char ***argv) +{ + int ho_error; + + if ((ho_error= handle_options(argc, argv, my_long_options, + get_one_option))) + { + exit(EXIT_FAILURE); + } + + return 0; +} + +static +void +print_version(void) +{ + printf("%s Ver %s for %s (%s)\n", my_progname, XBSTREAM_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); +} + +static +void +usage(void) +{ + print_version(); + puts("Copyright (C) 2011-2013 Percona LLC and/or its affiliates."); + puts("This software comes with ABSOLUTELY NO WARRANTY. " + "This is free software,\nand you are welcome to modify and " + "redistribute it under the GPL license.\n"); + + puts("Serialize/deserialize files in the XBSTREAM format.\n"); + + puts("Usage: "); + printf(" %s -c [OPTIONS...] FILES... # stream specified files to " + "standard output.\n", my_progname); + printf(" %s -x [OPTIONS...] # extract files from the stream" + "on the standard input.\n", my_progname); + + puts("\nOptions:"); + my_print_help(my_long_options); +} + +static +int +set_run_mode(run_mode_t mode) +{ + if (opt_mode != RUN_MODE_NONE) { + msg("%s: can't set specify both -c and -x.", my_progname); + return 1; + } + + opt_mode = mode; + + return 0; +} + +static +my_bool +get_one_option(const struct my_option *opt, const char *, const char *) +{ + switch (opt->id) { + case 'c': + if (set_run_mode(RUN_MODE_CREATE)) { + return TRUE; + } + break; + case 'x': + if (set_run_mode(RUN_MODE_EXTRACT)) { + return TRUE; + } + break; + case '?': + usage(); + exit(0); + } + + return FALSE; +} + +static +int +stream_one_file(File file, xb_wstream_file_t *xbfile) +{ + uchar *buf; + ssize_t bytes; + my_off_t offset; + + posix_fadvise(file, 0, 0, POSIX_FADV_SEQUENTIAL); + offset = my_tell(file, MYF(MY_WME)); + + buf = (uchar*)(my_malloc(PSI_NOT_INSTRUMENTED, XBSTREAM_BUFFER_SIZE, MYF(MY_FAE))); + + while ((bytes = (ssize_t)my_read(file, buf, XBSTREAM_BUFFER_SIZE, + MYF(MY_WME))) > 0) { + if (xb_stream_write_data(xbfile, buf, bytes)) { + msg("%s: xb_stream_write_data() failed.", + my_progname); + my_free(buf); + return 1; + } + posix_fadvise(file, offset, XBSTREAM_BUFFER_SIZE, + POSIX_FADV_DONTNEED); + offset += XBSTREAM_BUFFER_SIZE; + + } + + my_free(buf); + + if (bytes < 0) { + return 1; + } + + return 0; +} + +static +int +mode_create(int argc, char **argv) +{ + int i; + MY_STAT mystat; + xb_wstream_t *stream; + + if (argc < 1) { + msg("%s: no files are specified.", my_progname); + return 1; + } + + stream = xb_stream_write_new(); + if (stream == NULL) { + msg("%s: xb_stream_write_new() failed.", my_progname); + return 1; + } + + for (i = 0; i < argc; i++) { + char *filepath = argv[i]; + File src_file; + xb_wstream_file_t *file; + + if (my_stat(filepath, &mystat, MYF(MY_WME)) == NULL) { + goto err; + } + if (!MY_S_ISREG(mystat.st_mode)) { + msg("%s: %s is not a regular file, exiting.", + my_progname, filepath); + goto err; + } + + if ((src_file = my_open(filepath, O_RDONLY, MYF(MY_WME))) < 0) { + msg("%s: failed to open %s.", my_progname, filepath); + goto err; + } + + file = xb_stream_write_open(stream, filepath, &mystat, NULL, NULL); + if (file == NULL) { + goto err; + } + + if (opt_verbose) { + msg("%s", filepath); + } + + if (stream_one_file(src_file, file) || + xb_stream_write_close(file) || + my_close(src_file, MYF(MY_WME))) { + goto err; + } + } + + xb_stream_write_done(stream); + + return 0; +err: + xb_stream_write_done(stream); + + return 1; +} + +static +file_entry_t * +file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) +{ + file_entry_t *entry; + ds_file_t *file; + + entry = (file_entry_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(file_entry_t), + MYF(MY_WME | MY_ZEROFILL)); + if (entry == NULL) { + return NULL; + } + + entry->path = my_strndup(PSI_NOT_INSTRUMENTED, path, pathlen, MYF(MY_WME)); + if (entry->path == NULL) { + goto err; + } + entry->pathlen = pathlen; + + file = ds_open(ctxt->ds_ctxt, path, NULL); + + if (file == NULL) { + msg("%s: failed to create file.", my_progname); + goto err; + } + + if (opt_verbose) { + msg("%s", entry->path); + } + + entry->file = file; + + pthread_mutex_init(&entry->mutex, NULL); + + return entry; + +err: + if (entry->path != NULL) { + my_free(entry->path); + } + my_free(entry); + + return NULL; +} + +static +uchar * +get_file_entry_key(file_entry_t *entry, size_t *length, + my_bool not_used __attribute__((unused))) +{ + *length = entry->pathlen; + return (uchar *) entry->path; +} + +static +void +file_entry_free(file_entry_t *entry) +{ + pthread_mutex_destroy(&entry->mutex); + ds_close(entry->file); + my_free(entry->path); + my_free(entry); +} + +static +void * +extract_worker_thread_func(void *arg) +{ + xb_rstream_chunk_t chunk; + file_entry_t *entry; + xb_rstream_result_t res; + + extract_ctxt_t *ctxt = (extract_ctxt_t *) arg; + + my_thread_init(); + + memset(&chunk, 0, sizeof(chunk)); + + while (1) { + + pthread_mutex_lock(ctxt->mutex); + res = xb_stream_read_chunk(ctxt->stream, &chunk); + + if (res != XB_STREAM_READ_CHUNK) { + pthread_mutex_unlock(ctxt->mutex); + break; + } + + /* If unknown type and ignorable flag is set, skip this chunk */ + if (chunk.type == XB_CHUNK_TYPE_UNKNOWN && \ + !(chunk.flags & XB_STREAM_FLAG_IGNORABLE)) { + pthread_mutex_unlock(ctxt->mutex); + continue; + } + + /* See if we already have this file open */ + entry = (file_entry_t *) my_hash_search(ctxt->filehash, + (uchar *) chunk.path, + chunk.pathlen); + + if (entry == NULL) { + entry = file_entry_new(ctxt, + chunk.path, + chunk.pathlen); + if (entry == NULL) { + pthread_mutex_unlock(ctxt->mutex); + break; + } + if (my_hash_insert(ctxt->filehash, (uchar *) entry)) { + msg("%s: my_hash_insert() failed.", + my_progname); + pthread_mutex_unlock(ctxt->mutex); + break; + } + } + + pthread_mutex_lock(&entry->mutex); + + pthread_mutex_unlock(ctxt->mutex); + + res = xb_stream_validate_checksum(&chunk); + + if (res != XB_STREAM_READ_CHUNK) { + pthread_mutex_unlock(&entry->mutex); + break; + } + + if (chunk.type == XB_CHUNK_TYPE_EOF) { + pthread_mutex_unlock(&entry->mutex); + pthread_mutex_lock(ctxt->mutex); + my_hash_delete(ctxt->filehash, (uchar *) entry); + pthread_mutex_unlock(ctxt->mutex); + + continue; + } + + if (entry->offset != chunk.offset) { + msg("%s: out-of-order chunk: real offset = 0x%llx, " + "expected offset = 0x%llx", my_progname, + chunk.offset, entry->offset); + pthread_mutex_unlock(&entry->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + + if (ds_write(entry->file, chunk.data, chunk.length)) { + msg("%s: my_write() failed.", my_progname); + pthread_mutex_unlock(&entry->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + + entry->offset += chunk.length; + + pthread_mutex_unlock(&entry->mutex); + } + + if (chunk.data) + my_free(chunk.data); + + my_thread_end(); + + return (void *)(res); +} + + +static +int +mode_extract(int n_threads, int argc __attribute__((unused)), + char **argv __attribute__((unused))) +{ + xb_rstream_t *stream = NULL; + HASH filehash; + ds_ctxt_t *ds_ctxt = NULL; + extract_ctxt_t ctxt; + int i; + pthread_t *tids = NULL; + void **retvals = NULL; + pthread_mutex_t mutex; + int ret = 0; + + if (my_hash_init(PSI_NOT_INSTRUMENTED, &filehash, &my_charset_bin, + START_FILE_HASH_SIZE, 0, 0, (my_hash_get_key) get_file_entry_key, + (my_hash_free_key) file_entry_free, MYF(0))) { + msg("%s: failed to initialize file hash.", my_progname); + return 1; + } + + if (pthread_mutex_init(&mutex, NULL)) { + msg("%s: failed to initialize mutex.", my_progname); + my_hash_free(&filehash); + return 1; + } + + /* If --directory is specified, it is already set as CWD by now. */ + ds_ctxt = ds_create(".", DS_TYPE_LOCAL); + if (ds_ctxt == NULL) { + ret = 1; + goto exit; + } + + + stream = xb_stream_read_new(); + if (stream == NULL) { + msg("%s: xb_stream_read_new() failed.", my_progname); + pthread_mutex_destroy(&mutex); + ret = 1; + goto exit; + } + + ctxt.stream = stream; + ctxt.filehash = &filehash; + ctxt.ds_ctxt = ds_ctxt; + ctxt.mutex = &mutex; + + tids = (pthread_t *)calloc(n_threads, sizeof(pthread_t)); + retvals = (void **)calloc(n_threads, sizeof(void*)); + + for (i = 0; i < n_threads; i++) + pthread_create(tids + i, NULL, extract_worker_thread_func, + &ctxt); + + for (i = 0; i < n_threads; i++) + pthread_join(tids[i], retvals + i); + + for (i = 0; i < n_threads; i++) { + if ((size_t)retvals[i] == XB_STREAM_READ_ERROR) { + ret = 1; + goto exit; + } + } + +exit: + pthread_mutex_destroy(&mutex); + + free(tids); + free(retvals); + + my_hash_free(&filehash); + if (ds_ctxt != NULL) { + ds_destroy(ds_ctxt); + } + xb_stream_read_done(stream); + + return ret; +} diff --git a/extra/mariabackup/xbstream.h b/extra/mariabackup/xbstream.h new file mode 100644 index 00000000..1b36ec24 --- /dev/null +++ b/extra/mariabackup/xbstream.h @@ -0,0 +1,106 @@ +/****************************************************** +Copyright (c) 2011-2017 Percona LLC and/or its affiliates. + +The xbstream format interface. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef XBSTREAM_H +#define XBSTREAM_H + +#include <my_base.h> + +/* Magic value in a chunk header */ +#define XB_STREAM_CHUNK_MAGIC "XBSTCK01" + +/* Chunk flags */ +/* Chunk can be ignored if unknown version/format */ +#define XB_STREAM_FLAG_IGNORABLE 0x01 + +/* Magic + flags + type + path len */ +#define CHUNK_HEADER_CONSTANT_LEN ((sizeof(XB_STREAM_CHUNK_MAGIC) - 1) + \ + 1 + 1 + 4) +#define CHUNK_TYPE_OFFSET (sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1) +#define PATH_LENGTH_OFFSET (sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1) + +typedef struct xb_wstream_struct xb_wstream_t; + +typedef struct xb_wstream_file_struct xb_wstream_file_t; + +typedef enum { + XB_STREAM_FMT_NONE, + XB_STREAM_FMT_XBSTREAM +} xb_stream_fmt_t; + +/************************************************************************ +Write interface. */ + +typedef ssize_t xb_stream_write_callback(xb_wstream_file_t *file, + void *userdata, + const void *buf, size_t len); + +xb_wstream_t *xb_stream_write_new(void); + +xb_wstream_file_t *xb_stream_write_open(xb_wstream_t *stream, const char *path, + MY_STAT *mystat, void *userdata, + xb_stream_write_callback *onwrite); + +int xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len); + +int xb_stream_write_close(xb_wstream_file_t *file); + +int xb_stream_write_done(xb_wstream_t *stream); + +/************************************************************************ +Read interface. */ + +typedef enum { + XB_STREAM_READ_CHUNK, + XB_STREAM_READ_EOF, + XB_STREAM_READ_ERROR +} xb_rstream_result_t; + +typedef enum { + XB_CHUNK_TYPE_UNKNOWN = '\0', + XB_CHUNK_TYPE_PAYLOAD = 'P', + XB_CHUNK_TYPE_EOF = 'E' +} xb_chunk_type_t; + +typedef struct xb_rstream_struct xb_rstream_t; + +typedef struct { + uchar flags; + xb_chunk_type_t type; + uint pathlen; + char path[FN_REFLEN]; + size_t length; + my_off_t offset; + my_off_t checksum_offset; + void *data; + ulong checksum; + size_t buflen; +} xb_rstream_chunk_t; + +xb_rstream_t *xb_stream_read_new(void); + +xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream, + xb_rstream_chunk_t *chunk); + +int xb_stream_read_done(xb_rstream_t *stream); + +xb_rstream_result_t xb_stream_validate_checksum(xb_rstream_chunk_t *chunk); + +#endif diff --git a/extra/mariabackup/xbstream_read.cc b/extra/mariabackup/xbstream_read.cc new file mode 100644 index 00000000..b54a9815 --- /dev/null +++ b/extra/mariabackup/xbstream_read.cc @@ -0,0 +1,226 @@ +/****************************************************** +Copyright (c) 2011-2017 Percona LLC and/or its affiliates. + +The xbstream format reader implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <my_base.h> +#include <zlib.h> +#include "common.h" +#include "xbstream.h" + +/* Allocate 1 MB for the payload buffer initially */ +#define INIT_BUFFER_LEN (1024 * 1024) + +#ifndef MY_OFF_T_MAX +#define MY_OFF_T_MAX (~(my_off_t)0UL) +#endif + +struct xb_rstream_struct { + my_off_t offset; + File fd; +}; + +xb_rstream_t * +xb_stream_read_new(void) +{ + xb_rstream_t *stream; + + stream = (xb_rstream_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(xb_rstream_t), MYF(MY_FAE)); + +#ifdef _WIN32 + setmode(fileno(stdin), _O_BINARY); +#endif + + stream->fd = my_fileno(stdin); + stream->offset = 0; + + return stream; +} + +static inline +xb_chunk_type_t +validate_chunk_type(uchar code) +{ + switch ((xb_chunk_type_t) code) { + case XB_CHUNK_TYPE_PAYLOAD: + case XB_CHUNK_TYPE_EOF: + return (xb_chunk_type_t) code; + default: + return XB_CHUNK_TYPE_UNKNOWN; + } +} + +xb_rstream_result_t +xb_stream_validate_checksum(xb_rstream_chunk_t *chunk) +{ + ulong checksum; + checksum = my_checksum(0, chunk->data, chunk->length); + if (checksum != chunk->checksum) { + msg("xb_stream_read_chunk(): invalid checksum at offset " + "0x%llx: expected 0x%lx, read 0x%lx.", + (ulonglong) chunk->checksum_offset, chunk->checksum, + checksum); + return XB_STREAM_READ_ERROR; + } + + return XB_STREAM_READ_CHUNK; +} + +#define F_READ(buf,len) \ + do { \ + if (xb_read_full(fd, (uchar *)buf, len) < len) { \ + msg("xb_stream_read_chunk(): my_read() failed."); \ + goto err; \ + } \ + } while (0) + +xb_rstream_result_t +xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) +{ + uchar tmpbuf[16]; + uchar *ptr = tmpbuf; + uint pathlen; + size_t tbytes; + ulonglong ullval; + File fd = stream->fd; + + xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN); + + /* This is the only place where we expect EOF, so read with + xb_read_full() rather than F_READ() */ + tbytes = xb_read_full(fd, ptr, CHUNK_HEADER_CONSTANT_LEN); + if (tbytes == 0) { + return XB_STREAM_READ_EOF; + } else if (tbytes < CHUNK_HEADER_CONSTANT_LEN) { + msg("xb_stream_read_chunk(): unexpected end of stream at " + "offset 0x%llx.", stream->offset); + goto err; + } + + ptr = tmpbuf; + + /* Chunk magic value */ + if (memcmp(tmpbuf, XB_STREAM_CHUNK_MAGIC, 8)) { + msg("xb_stream_read_chunk(): wrong chunk magic at offset " + "0x%llx.", (ulonglong) stream->offset); + goto err; + } + ptr += 8; + stream->offset += 8; + + /* Chunk flags */ + chunk->flags = *ptr++; + stream->offset++; + + /* Chunk type, ignore unknown ones if ignorable flag is set */ + chunk->type = validate_chunk_type(*ptr); + if (chunk->type == XB_CHUNK_TYPE_UNKNOWN && + !(chunk->flags & XB_STREAM_FLAG_IGNORABLE)) { + msg("xb_stream_read_chunk(): unknown chunk type 0x%lu at " + "offset 0x%llx.", (ulong) *ptr, + (ulonglong) stream->offset); + goto err; + } + ptr++; + stream->offset++; + + /* Path length */ + pathlen = uint4korr(ptr); + if (pathlen >= FN_REFLEN) { + msg("xb_stream_read_chunk(): path length (%lu) is too large at " + "offset 0x%llx.", (ulong) pathlen, stream->offset); + goto err; + } + chunk->pathlen = pathlen; + stream->offset +=4; + + xb_ad((ptr + 4 - tmpbuf) == CHUNK_HEADER_CONSTANT_LEN); + + /* Path */ + if (chunk->pathlen > 0) { + F_READ((uchar *) chunk->path, pathlen); + stream->offset += pathlen; + } + chunk->path[pathlen] = '\0'; + + if (chunk->type == XB_CHUNK_TYPE_EOF) { + return XB_STREAM_READ_CHUNK; + } + + /* Payload length */ + F_READ(tmpbuf, 16); + ullval = uint8korr(tmpbuf); + if (ullval > (ulonglong) SIZE_T_MAX) { + msg("xb_stream_read_chunk(): chunk length is too large at " + "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->length = (size_t) ullval; + stream->offset += 8; + + /* Payload offset */ + ullval = uint8korr(tmpbuf + 8); + if (ullval > (ulonglong) MY_OFF_T_MAX) { + msg("xb_stream_read_chunk(): chunk offset is too large at " + "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->offset = (my_off_t) ullval; + stream->offset += 8; + + /* Reallocate the buffer if needed */ + if (chunk->length > chunk->buflen) { + chunk->data = my_realloc(PSI_NOT_INSTRUMENTED, chunk->data, chunk->length, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + if (chunk->data == NULL) { + msg("xb_stream_read_chunk(): failed to increase buffer " + "to %lu bytes.", (ulong) chunk->length); + goto err; + } + chunk->buflen = chunk->length; + } + + /* Checksum */ + F_READ(tmpbuf, 4); + chunk->checksum = uint4korr(tmpbuf); + chunk->checksum_offset = stream->offset; + + /* Payload */ + if (chunk->length > 0) { + F_READ(chunk->data, chunk->length); + stream->offset += chunk->length; + } + + stream->offset += 4; + + return XB_STREAM_READ_CHUNK; + +err: + return XB_STREAM_READ_ERROR; +} + +int +xb_stream_read_done(xb_rstream_t *stream) +{ + my_free(stream); + + return 0; +} diff --git a/extra/mariabackup/xbstream_write.cc b/extra/mariabackup/xbstream_write.cc new file mode 100644 index 00000000..5801e867 --- /dev/null +++ b/extra/mariabackup/xbstream_write.cc @@ -0,0 +1,293 @@ +/****************************************************** +Copyright (c) 2011-2017 Percona LLC and/or its affiliates. + +The xbstream format writer implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#include <my_global.h> +#include <my_base.h> +#include <zlib.h> +#include "common.h" +#include "xbstream.h" + +/* Group writes smaller than this into a single chunk */ +#define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024) + +struct xb_wstream_struct { + pthread_mutex_t mutex; +}; + +struct xb_wstream_file_struct { + xb_wstream_t *stream; + char *path; + size_t path_len; + char chunk[XB_STREAM_MIN_CHUNK_SIZE]; + char *chunk_ptr; + size_t chunk_free; + my_off_t offset; + void *userdata; + xb_stream_write_callback *write; +}; + +static int xb_stream_flush(xb_wstream_file_t *file); +static int xb_stream_write_chunk(xb_wstream_file_t *file, + const void *buf, size_t len); +static int xb_stream_write_eof(xb_wstream_file_t *file); + +static +ssize_t +xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)), + void *userdata __attribute__((unused)), + const void *buf, size_t len) +{ + if (my_write(my_fileno(stdout), (const uchar *)buf, len, MYF(MY_WME | MY_NABP))) + return -1; + return len; +} + +xb_wstream_t * +xb_stream_write_new(void) +{ + xb_wstream_t *stream; + + stream = (xb_wstream_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(xb_wstream_t), MYF(MY_FAE)); + pthread_mutex_init(&stream->mutex, NULL); + + return stream;; +} + +xb_wstream_file_t * +xb_stream_write_open(xb_wstream_t *stream, const char *path, + MY_STAT *mystat __attribute__((unused)), + void *userdata, + xb_stream_write_callback *onwrite) +{ + xb_wstream_file_t *file; + size_t path_len; + + path_len = strlen(path); + + if (path_len > FN_REFLEN) { + msg("xb_stream_write_open(): file path is too long."); + return NULL; + } + + file = (xb_wstream_file_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(xb_wstream_file_t) + + path_len + 1, MYF(MY_FAE)); + + file->path = (char *) (file + 1); +#ifdef _WIN32 + /* Normalize path on Windows, so we can restore elsewhere.*/ + { + int i; + for (i = 0; ; i++) { + file->path[i] = (path[i] == '\\') ? '/' : path[i]; + if (!path[i]) + break; + } + } +#else + memcpy(file->path, path, path_len + 1); +#endif + file->path_len = path_len; + + file->stream = stream; + file->offset = 0; + file->chunk_ptr = file->chunk; + file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE; + if (onwrite) { +#ifdef _WIN32 + setmode(fileno(stdout), _O_BINARY); +#endif + file->userdata = userdata; + file->write = onwrite; + } else { + file->userdata = NULL; + file->write = xb_stream_default_write_callback; + } + + return file; +} + +int +xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len) +{ + if (len < file->chunk_free) { + memcpy(file->chunk_ptr, buf, len); + file->chunk_ptr += len; + file->chunk_free -= len; + + return 0; + } + + if (xb_stream_flush(file)) + return 1; + + return xb_stream_write_chunk(file, buf, len); +} + +int +xb_stream_write_close(xb_wstream_file_t *file) +{ + if (xb_stream_flush(file) || + xb_stream_write_eof(file)) { + my_free(file); + return 1; + } + + my_free(file); + + return 0; +} + +int +xb_stream_write_done(xb_wstream_t *stream) +{ + pthread_mutex_destroy(&stream->mutex); + + my_free(stream); + + return 0; +} + +static +int +xb_stream_flush(xb_wstream_file_t *file) +{ + if (file->chunk_ptr == file->chunk) { + return 0; + } + + if (xb_stream_write_chunk(file, file->chunk, + file->chunk_ptr - file->chunk)) { + return 1; + } + + file->chunk_ptr = file->chunk; + file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE; + + return 0; +} + +static +int +xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) +{ + /* Chunk magic + flags + chunk type + path_len + path + len + offset + + checksum */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + + FN_REFLEN + 8 + 8 + 4]; + uchar *ptr; + xb_wstream_t *stream = file->stream; + ulong checksum; + + /* Write xbstream header */ + ptr = tmpbuf; + + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_PAYLOAD; /* Chunk type */ + + int4store(ptr, file->path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, file->path, file->path_len); /* Path */ + ptr += file->path_len; + + int8store(ptr, len); /* Payload length */ + ptr += 8; + + checksum = my_checksum(0, buf, len); + + pthread_mutex_lock(&stream->mutex); + + int8store(ptr, file->offset); /* Payload offset */ + ptr += 8; + + int4store(ptr, checksum); + ptr += 4; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + if (file->write(file, file->userdata, tmpbuf, ptr-tmpbuf) == -1) + goto err; + + + if (file->write(file, file->userdata, buf, len) == -1) /* Payload */ + goto err; + + file->offset+= len; + + pthread_mutex_unlock(&stream->mutex); + + return 0; + +err: + + pthread_mutex_unlock(&stream->mutex); + + return 1; +} + +static +int +xb_stream_write_eof(xb_wstream_file_t *file) +{ + /* Chunk magic + flags + chunk type + path_len + path */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + + FN_REFLEN]; + uchar *ptr; + xb_wstream_t *stream = file->stream; + + pthread_mutex_lock(&stream->mutex); + + /* Write xbstream header */ + ptr = tmpbuf; + + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_EOF; /* Chunk type */ + + int4store(ptr, file->path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, file->path, file->path_len); /* Path */ + ptr += file->path_len; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + if (file->write(file, file->userdata, tmpbuf, + (ulonglong) (ptr - tmpbuf)) == -1) + goto err; + + pthread_mutex_unlock(&stream->mutex); + + return 0; +err: + + pthread_mutex_unlock(&stream->mutex); + + return 1; +} diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc new file mode 100644 index 00000000..9e359257 --- /dev/null +++ b/extra/mariabackup/xtrabackup.cc @@ -0,0 +1,7083 @@ +/****************************************************** +MariaBackup: hot backup tool for InnoDB +(c) 2009-2017 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. +(c) 2017, 2022, MariaDB Corporation. +Portions written by Marko Mäkelä. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 51 Franklin +Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +//#define XTRABACKUP_TARGET_IS_PLUGIN + +#include <my_global.h> +#include <my_config.h> +#include <unireg.h> +#include <mysql_version.h> +#include <my_base.h> +#include <my_getopt.h> +#include <mysql_com.h> +#include <my_default.h> +#include <scope.h> +#include <sql_class.h> + +#include <fcntl.h> +#include <string.h> + +#ifdef __linux__ +# include <sys/prctl.h> +# include <sys/resource.h> +#endif + +#ifdef __APPLE__ +# include "libproc.h" +#endif + +#ifdef __FreeBSD__ +# include <sys/sysctl.h> +#endif + + +#include <btr0sea.h> +#include <lock0lock.h> +#include <log0recv.h> +#include <log0crypt.h> +#include <row0mysql.h> +#include <row0quiesce.h> +#include <srv0start.h> +#include "trx0sys.h" +#include <buf0dblwr.h> +#include <buf0flu.h> +#include "ha_innodb.h" + +#include <list> +#include <sstream> +#include <set> +#include <fstream> +#include <mysql.h> + +#define G_PTR uchar* + +#include "common.h" +#include "datasink.h" + +#include "xb_regex.h" +#include "fil_cur.h" +#include "write_filt.h" +#include "xtrabackup.h" +#include "ds_buffer.h" +#include "ds_tmpfile.h" +#include "xbstream.h" +#include "changed_page_bitmap.h" +#include "read_filt.h" +#include "backup_wsrep.h" +#include "innobackupex.h" +#include "backup_mysql.h" +#include "backup_copy.h" +#include "backup_mysql.h" +#include "xb_plugin.h" +#include <sql_plugin.h> +#include <srv0srv.h> +#include <log.h> +#include <derror.h> +#include <thr_timer.h> +#include "backup_debug.h" + +#define MB_CORRUPTED_PAGES_FILE "innodb_corrupted_pages" + +// disable server's systemd notification code +extern "C" { +int sd_notify() { return 0; } +int sd_notifyf() { return 0; } +} + +int sys_var_init(); + +/* === xtrabackup specific options === */ +char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/"; +char *xtrabackup_target_dir= xtrabackup_real_target_dir; +static my_bool xtrabackup_version; +static my_bool verbose; +my_bool xtrabackup_backup; +my_bool xtrabackup_prepare; +my_bool xtrabackup_copy_back; +my_bool xtrabackup_move_back; +my_bool xtrabackup_decrypt_decompress; +my_bool xtrabackup_print_param; +my_bool xtrabackup_mysqld_args; +my_bool xtrabackup_help; + +my_bool xtrabackup_export; + +longlong xtrabackup_use_memory; + +uint opt_protocol; +long xtrabackup_throttle; /* 0:unlimited */ +static lint io_ticket; +static mysql_cond_t wait_throttle; +static mysql_cond_t log_copying_stop; + +char *xtrabackup_incremental; +lsn_t incremental_lsn; +lsn_t incremental_to_lsn; +lsn_t incremental_last_lsn; +xb_page_bitmap *changed_page_bitmap; + +char *xtrabackup_incremental_basedir; /* for --backup */ +char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */ +char *xtrabackup_incremental_dir; /* for --prepare */ + +char xtrabackup_real_incremental_basedir[FN_REFLEN]; +char xtrabackup_real_extra_lsndir[FN_REFLEN]; +char xtrabackup_real_incremental_dir[FN_REFLEN]; + + +char *xtrabackup_tmpdir; + +char *xtrabackup_tables; +char *xtrabackup_tables_file; +char *xtrabackup_tables_exclude; +char *xb_rocksdb_datadir; +my_bool xb_backup_rocksdb = 1; + +typedef std::list<regex_t> regex_list_t; +static regex_list_t regex_include_list; +static regex_list_t regex_exclude_list; + +static hash_table_t tables_include_hash; +static hash_table_t tables_exclude_hash; + +char *xtrabackup_databases = NULL; +char *xtrabackup_databases_file = NULL; +char *xtrabackup_databases_exclude = NULL; +static hash_table_t databases_include_hash; +static hash_table_t databases_exclude_hash; + +static hash_table_t inc_dir_tables_hash; + +struct xb_filter_entry_t{ + char* name; + ibool has_tables; + xb_filter_entry_t *name_hash; +}; + +/** whether log_copying_thread() is active; protected by recv_sys.mutex */ +static bool log_copying_running; + +int xtrabackup_parallel; + +char *xtrabackup_stream_str = NULL; +xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE; +ibool xtrabackup_stream = FALSE; + +const char *xtrabackup_compress_alg = NULL; +uint xtrabackup_compress = FALSE; +uint xtrabackup_compress_threads; +ulonglong xtrabackup_compress_chunk_size = 0; + +/* sleep interval beetween log copy iterations in log copying thread +in milliseconds (default is 1 second) */ +ulint xtrabackup_log_copy_interval = 1000; +static ulong max_buf_pool_modified_pct; + +/* Ignored option (--log) for MySQL option compatibility */ +static char* log_ignored_opt; + + +extern my_bool opt_use_ssl; +extern char *opt_tls_version; +my_bool opt_ssl_verify_server_cert; +my_bool opt_extended_validation; +my_bool opt_encrypted_backup; + +/* === metadata of backup === */ +#define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints" +char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/ +static lsn_t metadata_from_lsn; +lsn_t metadata_to_lsn; +static lsn_t metadata_last_lsn; + +static ds_file_t* dst_log_file; + +static char mysql_data_home_buff[2]; + +const char *defaults_group = "mysqld"; + +/* === static parameters in ha_innodb.cc */ + +#define HA_INNOBASE_ROWS_IN_TABLE 10000 /* to get optimization right */ +#define HA_INNOBASE_RANGE_COUNT 100 + +/* The default values for the following, type long or longlong, start-up +parameters are declared in mysqld.cc: */ + +long innobase_buffer_pool_awe_mem_mb = 0; +long innobase_file_io_threads = 4; +ulong innobase_read_io_threads = 4; +ulong innobase_write_io_threads = 4; + +/** Store the failed read of undo tablespace ids. Protected by +recv_sys.mutex. */ +static std::set<uint32_t> fail_undo_ids; + +longlong innobase_page_size = (1LL << 14); /* 16KB */ +char* innobase_buffer_pool_filename = NULL; + +/* The default values for the following char* start-up parameters +are determined in innobase_init below: */ + +static char* innobase_ignored_opt; +char* innobase_data_home_dir; +char* innobase_data_file_path; + +char *aria_log_dir_path; + +my_bool xtrabackup_incremental_force_scan = FALSE; + +/* + * Ignore corrupt pages (disabled by default; used + * by "innobackupex" as a command line argument). + */ +ulong xtrabackup_innodb_force_recovery = 0; + +/* The flushed lsn which is read from data files */ +lsn_t flushed_lsn= 0; + +ulong xb_open_files_limit= 0; +char *xb_plugin_dir; +char *xb_plugin_load; +my_bool xb_close_files; + + +class Datasink_free_list +{ +protected: + /* + Simple datasink creation tracking... + add datasinks in the reverse order you want them destroyed. + */ +#define XTRABACKUP_MAX_DATASINKS 10 + ds_ctxt_t *m_datasinks_to_destroy[XTRABACKUP_MAX_DATASINKS]; + uint m_actual_datasinks_to_destroy; +public: + Datasink_free_list() + :m_actual_datasinks_to_destroy(0) + { } + + void add_datasink_to_destroy(ds_ctxt_t *ds) + { + xb_ad(m_actual_datasinks_to_destroy < XTRABACKUP_MAX_DATASINKS); + m_datasinks_to_destroy[m_actual_datasinks_to_destroy] = ds; + m_actual_datasinks_to_destroy++; + } + + /* + Destroy datasinks. + Destruction is done in the specific order to not violate their order in the + pipeline so that each datasink is able to flush data down the pipeline. + */ + void destroy() + { + for (uint i= m_actual_datasinks_to_destroy; i > 0; i--) + { + ds_destroy(m_datasinks_to_destroy[i - 1]); + m_datasinks_to_destroy[i - 1] = NULL; + } + } +}; + + +class Backup_datasinks: public Datasink_free_list +{ +public: + ds_ctxt_t *m_data; + ds_ctxt_t *m_meta; + ds_ctxt_t *m_redo; + + Backup_datasinks() + :m_data(NULL), + m_meta(NULL), + m_redo(NULL) + { } + void init(); + void destroy() + { + Datasink_free_list::destroy(); + *this= Backup_datasinks(); + } + bool backup_low(); +}; + + +static bool innobackupex_mode = false; + +/* String buffer used by --print-param to accumulate server options as they are +parsed from the defaults file */ +static std::ostringstream print_param_str; + +/* Set of specified parameters */ +std::set<std::string> param_set; + +static ulonglong global_max_value; + +extern "C" sig_handler handle_fatal_signal(int sig); +extern LOGGER logger; + +my_bool opt_galera_info = FALSE; +my_bool opt_slave_info = FALSE; +my_bool opt_no_lock = FALSE; +my_bool opt_safe_slave_backup = FALSE; +my_bool opt_rsync = FALSE; +my_bool opt_force_non_empty_dirs = FALSE; +my_bool opt_noversioncheck = FALSE; +my_bool opt_no_backup_locks = FALSE; +my_bool opt_decompress = FALSE; +my_bool opt_remove_original; +my_bool opt_log_innodb_page_corruption; + +my_bool opt_lock_ddl_per_table = FALSE; +static my_bool opt_check_privileges; + +extern const char *innodb_checksum_algorithm_names[]; +extern TYPELIB innodb_checksum_algorithm_typelib; +extern const char *innodb_flush_method_names[]; +extern TYPELIB innodb_flush_method_typelib; + +static const char *binlog_info_values[] = {"off", "lockless", "on", "auto", + NullS}; +static TYPELIB binlog_info_typelib = {array_elements(binlog_info_values)-1, "", + binlog_info_values, NULL}; +ulong opt_binlog_info; + +char *opt_incremental_history_name; +char *opt_incremental_history_uuid; + +char *opt_user; +const char *opt_password; +char *opt_host; +char *opt_defaults_group; +char *opt_socket; +uint opt_port; +char *opt_log_bin; + +const char *query_type_names[] = { "ALL", "UPDATE", "SELECT", NullS}; + +TYPELIB query_type_typelib= {array_elements(query_type_names) - 1, "", + query_type_names, NULL}; + +ulong opt_lock_wait_query_type; +ulong opt_kill_long_query_type; + +uint opt_kill_long_queries_timeout = 0; +uint opt_lock_wait_timeout = 0; +uint opt_lock_wait_threshold = 0; +uint opt_debug_sleep_before_unlock = 0; +uint opt_safe_slave_backup_timeout = 0; + +const char *opt_history = NULL; + + +char mariabackup_exe[FN_REFLEN]; +char orig_argv1[FN_REFLEN]; + +pthread_cond_t scanned_lsn_cond; + +/** Store the deferred tablespace name during --backup */ +static std::set<std::string> defer_space_names; + +typedef std::map<space_id_t,std::string> space_id_to_name_t; + +struct ddl_tracker_t { + /** Tablspaces with their ID and name, as they were copied to backup.*/ + space_id_to_name_t tables_in_backup; + /** Drop operations found in redo log. */ + std::set<space_id_t> drops; + /* For DDL operation found in redo log, */ + space_id_to_name_t id_to_name; + /** Deferred tablespaces with their ID and name which was + found in redo log of DDL operations */ + space_id_to_name_t deferred_tables; + + /** Insert the deferred tablespace id with the name */ + void insert_defer_id(space_id_t space_id, std::string name) + { + auto it= defer_space_names.find(name); + if (it != defer_space_names.end()) + { + deferred_tables[space_id]= name; + defer_space_names.erase(it); + } + } + + /** Rename the deferred tablespace with new name */ + void rename_defer(space_id_t space_id, std::string old_name, + std::string new_name) + { + if (deferred_tables.find(space_id) != deferred_tables.end()) + deferred_tables[space_id] = new_name; + auto defer_end= defer_space_names.end(); + auto defer= defer_space_names.find(old_name); + if (defer == defer_end) + defer= defer_space_names.find(new_name); + + if (defer != defer_end) + { + deferred_tables[space_id]= new_name; + defer_space_names.erase(defer); + } + } + + /** Delete the deferred tablespace */ + void delete_defer(space_id_t space_id, std::string name) + { + deferred_tables.erase(space_id); + defer_space_names.erase(name); + } +}; + +static ddl_tracker_t ddl_tracker; + +/** Store the space ids of truncated undo log tablespaces. Protected +by recv_sys.mutex */ +static std::set<uint32_t> undo_trunc_ids; + +/** Stores the space ids of page0 INIT_PAGE redo records. It is +used to indicate whether the given deferred tablespace can +be reconstructed. */ +static std::set<space_id_t> first_page_init_ids; + +// Convert non-null terminated filename to space name +static std::string filename_to_spacename(const void *filename, size_t len); + +CorruptedPages::CorruptedPages() { ut_a(!pthread_mutex_init(&m_mutex, NULL)); } + +CorruptedPages::~CorruptedPages() { ut_a(!pthread_mutex_destroy(&m_mutex)); } + +void CorruptedPages::add_page_no_lock(const char *space_name, + page_id_t page_id, + bool convert_space_name) +{ + space_info_t &space_info = m_spaces[page_id.space()]; + if (space_info.space_name.empty()) + space_info.space_name= convert_space_name + ? filename_to_spacename(space_name, strlen(space_name)) + : space_name; + (void)space_info.pages.insert(page_id.page_no()); +} + +void CorruptedPages::add_page(const char *file_name, page_id_t page_id) +{ + pthread_mutex_lock(&m_mutex); + add_page_no_lock(file_name, page_id, true); + pthread_mutex_unlock(&m_mutex); +} + +bool CorruptedPages::contains(page_id_t page_id) const +{ + bool result = false; + ut_a(!pthread_mutex_lock(&m_mutex)); + container_t::const_iterator space_it= m_spaces.find(page_id.space()); + if (space_it != m_spaces.end()) + result = space_it->second.pages.count(page_id.page_no()); + ut_a(!pthread_mutex_unlock(&m_mutex)); + return result; +} + +void CorruptedPages::drop_space(uint32_t space_id) +{ + ut_a(!pthread_mutex_lock(&m_mutex)); + m_spaces.erase(space_id); + ut_a(!pthread_mutex_unlock(&m_mutex)); +} + +void CorruptedPages::rename_space(uint32_t space_id, + const std::string &new_name) +{ + ut_a(!pthread_mutex_lock(&m_mutex)); + container_t::iterator space_it = m_spaces.find(space_id); + if (space_it != m_spaces.end()) + space_it->second.space_name = new_name; + ut_a(!pthread_mutex_unlock(&m_mutex)); +} + +bool CorruptedPages::print_to_file(ds_ctxt *ds_data, + const char *filename) const +{ + std::ostringstream out; + ut_a(!pthread_mutex_lock(&m_mutex)); + if (!m_spaces.size()) + { + ut_a(!pthread_mutex_unlock(&m_mutex)); + return true; + } + for (container_t::const_iterator space_it= + m_spaces.begin(); + space_it != m_spaces.end(); ++space_it) + { + out << space_it->second.space_name << " " << space_it->first << "\n"; + bool first_page_no= true; + for (std::set<unsigned>::const_iterator page_it= + space_it->second.pages.begin(); + page_it != space_it->second.pages.end(); ++page_it) + if (first_page_no) + { + out << *page_it; + first_page_no= false; + } + else + out << " " << *page_it; + out << "\n"; + } + ut_a(!pthread_mutex_unlock(&m_mutex)); + if (ds_data) + return ds_data->backup_file_print_buf(filename, out.str().c_str(), + static_cast<int>(out.str().size())); + std::ofstream outfile; + outfile.open(filename); + if (!outfile.is_open()) + die("Can't open %s, error number: %d, error message: %s", filename, errno, + strerror(errno)); + outfile << out.str(); + return true; +} + +void CorruptedPages::read_from_file(const char *file_name) +{ + MY_STAT mystat; + if (!my_stat(file_name, &mystat, MYF(0))) + return; + std::ifstream infile; + infile.open(file_name); + if (!infile.is_open()) + die("Can't open %s, error number: %d, error message: %s", file_name, errno, + strerror(errno)); + std::string line; + std::string space_name; + uint32_t space_id; + ulint line_number= 0; + while (std::getline(infile, line)) + { + ++line_number; + std::istringstream iss(line); + if (line_number & 1) { + if (!(iss >> space_name)) + die("Can't parse space name from corrupted pages file at " + "line " ULINTPF, + line_number); + if (!(iss >> space_id)) + die("Can't parse space id from corrupted pages file at line " ULINTPF, + line_number); + } + else + { + std::istringstream iss(line); + unsigned page_no; + while ((iss >> page_no)) + add_page_no_lock(space_name.c_str(), {space_id, page_no}, false); + if (!iss.eof()) + die("Corrupted pages file parse error on line number " ULINTPF, + line_number); + } + } +} + +bool CorruptedPages::empty() const +{ + ut_a(!pthread_mutex_lock(&m_mutex)); + bool result= !m_spaces.size(); + ut_a(!pthread_mutex_unlock(&m_mutex)); + return result; +} + +static void xb_load_single_table_tablespace(const std::string &space_name, + bool set_size, + uint32_t defer_space_id=0); +static void xb_data_files_close(); +static fil_space_t* fil_space_get_by_name(const char* name); + +void CorruptedPages::zero_out_free_pages() +{ + container_t non_free_pages; + byte *zero_page= + static_cast<byte *>(aligned_malloc(srv_page_size, srv_page_size)); + memset(zero_page, 0, srv_page_size); + + ut_a(!pthread_mutex_lock(&m_mutex)); + for (container_t::const_iterator space_it= m_spaces.begin(); + space_it != m_spaces.end(); ++space_it) + { + uint32_t space_id = space_it->first; + const std::string &space_name = space_it->second.space_name; + // There is no need to close tablespaces explixitly as they will be closed + // in innodb_shutdown(). + xb_load_single_table_tablespace(space_name, false); + fil_space_t *space = fil_space_t::get(space_id); + if (!space) + die("Can't find space object for space name %s to check corrupted page", + space_name.c_str()); + for (std::set<unsigned>::const_iterator page_it= + space_it->second.pages.begin(); + page_it != space_it->second.pages.end(); ++page_it) + { + if (fseg_page_is_allocated(space, *page_it)) + { + space_info_t &space_info = non_free_pages[space_id]; + space_info.pages.insert(*page_it); + if (space_info.space_name.empty()) + space_info.space_name = space_name; + msg("Error: corrupted page " UINT32PF + " of tablespace %s can not be fixed", + *page_it, space_name.c_str()); + } + else + { + space->reacquire(); + auto err= space + ->io(IORequest(IORequest::PUNCH_RANGE), + *page_it * srv_page_size, srv_page_size, zero_page) + .err; + if (err != DB_SUCCESS) + die("Can't zero out corrupted page " UINT32PF " of tablespace %s", + *page_it, space_name.c_str()); + msg("Corrupted page " UINT32PF + " of tablespace %s was successfully fixed.", + *page_it, space_name.c_str()); + } + } + space->flush<true>(); + space->release(); + } + m_spaces.swap(non_free_pages); + ut_a(!pthread_mutex_unlock(&m_mutex)); + aligned_free(zero_page); +} + +typedef void (*process_single_tablespace_func_t)(const char *dirname, + const char *filname, + bool is_remote, + bool skip_node_page0, + uint32_t defer_space_id); +static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback); + +/* ======== Datafiles iterator ======== */ +struct datafiles_iter_t { + space_list_t::iterator space = fil_system.space_list.end(); + fil_node_t *node = nullptr; + bool started = false; + std::mutex mutex; +}; + +/* ======== Datafiles iterator ======== */ +static +fil_node_t * +datafiles_iter_next(datafiles_iter_t *it) +{ + fil_node_t *new_node; + + std::lock_guard<std::mutex> _(it->mutex); + + if (it->node == NULL) { + if (it->started) + goto end; + it->started = TRUE; + } else { + it->node = UT_LIST_GET_NEXT(chain, it->node); + if (it->node != NULL) + goto end; + } + + it->space = (it->space == fil_system.space_list.end()) ? + fil_system.space_list.begin() : + std::next(it->space); + + while (it->space != fil_system.space_list.end() && + (it->space->purpose != FIL_TYPE_TABLESPACE || + UT_LIST_GET_LEN(it->space->chain) == 0)) + ++it->space; + if (it->space == fil_system.space_list.end()) + goto end; + + it->node = UT_LIST_GET_FIRST(it->space->chain); + +end: + new_node = it->node; + + return new_node; +} + +#ifndef DBUG_OFF +struct dbug_thread_param_t +{ + MYSQL *con; + const char *query; + int expect_err; + int expect_errno; +}; + + +/* Thread procedure used in dbug_start_query_thread. */ +static void *dbug_execute_in_new_connection(void *arg) +{ + mysql_thread_init(); + dbug_thread_param_t *par= static_cast<dbug_thread_param_t*>(arg); + int err = mysql_query(par->con, par->query); + int err_no = mysql_errno(par->con); + if(par->expect_err != err) + { + msg("FATAL: dbug_execute_in_new_connection : mysql_query '%s' returns %d, instead of expected %d", + par->query, err, par->expect_err); + _exit(1); + } + if (err && par->expect_errno && par->expect_errno != err_no) + { + msg("FATAL: dbug_execute_in_new_connection: mysql_query '%s' returns mysql_errno %d, instead of expected %d", + par->query, err_no, par->expect_errno); + _exit(1); + } + mysql_close(par->con); + mysql_thread_end(); + delete par; + return nullptr; +} + +static pthread_t dbug_alter_thread; + +/* +Execute query from a new connection, in own thread. + +@param query - query to be executed +@param wait_state - if not NULL, wait until query from new connection + reaches this state (value of column State in I_S.PROCESSLIST) +@param expected_err - if 0, query is supposed to finish successfully, + otherwise query should return error. +@param expected_errno - if not 0, and query finished with error, + expected mysql_errno() +*/ +static void dbug_start_query_thread( + const char *query, + const char *wait_state, + int expected_err, + int expected_errno) + +{ + dbug_thread_param_t *par = new dbug_thread_param_t; + par->query = query; + par->expect_err = expected_err; + par->expect_errno = expected_errno; + par->con = xb_mysql_connect(); + + mysql_thread_create(0, &dbug_alter_thread, nullptr, + dbug_execute_in_new_connection, par); + + if (!wait_state) + return; + + char q[256]; + snprintf(q, sizeof(q), + "SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST where ID=%lu" + " AND Command='Query' AND State='%s'", + mysql_thread_id(par->con), wait_state); + for (;;) { + MYSQL_RES *result = xb_mysql_query(mysql_connection,q, true, true); + bool exists = mysql_fetch_row(result) != NULL; + mysql_free_result(result); + if (exists) { + goto end; + } + msg("Waiting for query '%s' on connection %lu to " + " reach state '%s'", query, mysql_thread_id(par->con), + wait_state); + my_sleep(1000); + } +end: + msg("query '%s' on connection %lu reached state '%s'", query, + mysql_thread_id(par->con), wait_state); +} +#endif + +void mdl_lock_all() +{ + mdl_lock_init(); + datafiles_iter_t it; + + while (fil_node_t *node= datafiles_iter_next(&it)) + { + const auto id= node->space->id; + if (const char *name= (fil_is_user_tablespace_id(id) && + node->space->chain.start) + ? node->space->chain.start->name : nullptr) + if (check_if_skip_table(filename_to_spacename(name, + strlen(name)).c_str())) + continue; + mdl_lock_table(id); + } +} + + +// Convert non-null terminated filename to space name +// Note that in 10.6 the filename may be an undo file name +static std::string filename_to_spacename(const void *filename, size_t len) +{ + char f[FN_REFLEN]; + char *p= 0, *table, *db; + DBUG_ASSERT(len < FN_REFLEN); + + strmake(f, (const char*) filename, len); + +#ifdef _WIN32 + for (size_t i = 0; i < len; i++) + { + if (f[i] == '\\') + f[i] = '/'; + } +#endif + + /* Remove extension, if exists */ + if (!(p= strrchr(f, '.'))) + goto err; + *p= 0; + + /* Find table name */ + if (!(table= strrchr(f, '/'))) + goto err; + *table = 0; + + /* Find database name */ + db= strrchr(f, '/'); + *table = '/'; + if (!db) + goto err; + { + std::string s(db+1); + return s; + } + +err: + /* Not a database/table. Return original (converted) name */ + if (p) + *p= '.'; // Restore removed extension + std::string s(f); + return s; +} + +/** Report an operation to create, delete, or rename a file during backup. +@param[in] space_id tablespace identifier +@param[in] type redo log file operation type +@param[in] name file name (not NUL-terminated) +@param[in] len length of name, in bytes +@param[in] new_name new file name (NULL if not rename) +@param[in] new_len length of new_name, in bytes (0 if NULL) */ +static void backup_file_op(uint32_t space_id, int type, + const byte* name, ulint len, + const byte* new_name, ulint new_len) +{ + + ut_ad(name); + ut_ad(len); + ut_ad(!new_name == !new_len); + mysql_mutex_assert_owner(&recv_sys.mutex); + + switch(type) { + case FILE_CREATE: + { + std::string space_name = filename_to_spacename(name, len); + ddl_tracker.id_to_name[space_id] = space_name; + ddl_tracker.delete_defer(space_id, space_name); + msg("DDL tracking : create %u \"%.*s\"", space_id, int(len), name); + } + break; + case FILE_MODIFY: + ddl_tracker.insert_defer_id( + space_id, filename_to_spacename(name, len)); + break; + case FILE_RENAME: + { + std::string new_space_name = filename_to_spacename( + new_name, new_len); + std::string old_space_name = filename_to_spacename( + name, len); + ddl_tracker.id_to_name[space_id] = new_space_name; + ddl_tracker.rename_defer(space_id, old_space_name, + new_space_name); + msg("DDL tracking : rename %u \"%.*s\",\"%.*s\"", + space_id, int(len), name, int(new_len), new_name); + } + break; + case FILE_DELETE: + ddl_tracker.drops.insert(space_id); + ddl_tracker.delete_defer( + space_id, filename_to_spacename(name, len)); + msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name); + break; + default: + ut_ad(0); + break; + } +} + + +/* + This callback is called if DDL operation is detected, + at the end of backup + + Normally, DDL operations are blocked due to FTWRL, + but in rare cases of --no-lock, they are not. + + We will abort backup in this case. +*/ +static void backup_file_op_fail(uint32_t space_id, int type, + const byte* name, ulint len, + const byte* new_name, ulint new_len) +{ + bool fail = false; + switch(type) { + case FILE_CREATE: + msg("DDL tracking : create %u \"%.*s\"", space_id, int(len), name); + fail = !check_if_skip_table( + filename_to_spacename(name, len).c_str()); + break; + case FILE_MODIFY: + break; + case FILE_RENAME: + msg("DDL tracking : rename %u \"%.*s\",\"%.*s\"", + space_id, int(len), name, int(new_len), new_name); + fail = !check_if_skip_table( + filename_to_spacename(name, len).c_str()) + || !check_if_skip_table( + filename_to_spacename(new_name, new_len).c_str()); + break; + case FILE_DELETE: + fail = !check_if_skip_table( + filename_to_spacename(name, len).c_str()); + msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name); + break; + default: + ut_ad(0); + break; + } + + if (fail) { + ut_a(opt_no_lock); + die("DDL operation detected in the late phase of backup." + "Backup is inconsistent. Remove --no-lock option to fix."); + } +} + +static void backup_undo_trunc(uint32_t space_id) +{ + undo_trunc_ids.insert(space_id); +} + +/* Function to store the space id of page0 INIT_PAGE +@param space_id space id which has page0 init page */ +static void backup_first_page_op(space_id_t space_id) +{ + first_page_init_ids.insert(space_id); +} + +/* + Retrieve default data directory, to be used with --copy-back. + + On Windows, default datadir is ..\data, relative to the + directory where mariabackup.exe is located(usually "bin") + + Elsewhere, the compiled-in constant MYSQL_DATADIR is used. +*/ +static char *get_default_datadir() { + static char ddir[] = MYSQL_DATADIR; +#ifdef _WIN32 + static char buf[MAX_PATH]; + DWORD size = (DWORD)sizeof(buf) - 1; + if (GetModuleFileName(NULL, buf, size) <= size) + { + char *p; + if ((p = strrchr(buf, '\\'))) + { + *p = 0; + if ((p = strrchr(buf, '\\'))) + { + strncpy(p + 1, "data", buf + MAX_PATH - p); + return buf; + } + } + } +#endif + return ddir; +} + + +/* ======== Date copying thread context ======== */ + +typedef struct { + datafiles_iter_t *it; + uint num; + uint *count; + pthread_mutex_t* count_mutex; + CorruptedPages *corrupted_pages; + Backup_datasinks *datasinks; +} data_thread_ctxt_t; + +/* ======== for option and variables ======== */ +#include <../../client/client_priv.h> + +enum options_xtrabackup +{ + OPT_XTRA_TARGET_DIR= 1000, /* make sure it is larger + than OPT_MAX_CLIENT_OPTION */ + OPT_XTRA_BACKUP, + OPT_XTRA_PREPARE, + OPT_XTRA_EXPORT, + OPT_XTRA_PRINT_PARAM, + OPT_XTRA_USE_MEMORY, + OPT_XTRA_THROTTLE, + OPT_XTRA_LOG_COPY_INTERVAL, + OPT_XTRA_INCREMENTAL, + OPT_XTRA_INCREMENTAL_BASEDIR, + OPT_XTRA_EXTRA_LSNDIR, + OPT_XTRA_INCREMENTAL_DIR, + OPT_XTRA_TABLES, + OPT_XTRA_TABLES_FILE, + OPT_XTRA_DATABASES, + OPT_XTRA_DATABASES_FILE, + OPT_XTRA_PARALLEL, + OPT_XTRA_EXTENDED_VALIDATION, + OPT_XTRA_ENCRYPTED_BACKUP, + OPT_XTRA_STREAM, + OPT_XTRA_COMPRESS, + OPT_XTRA_COMPRESS_THREADS, + OPT_XTRA_COMPRESS_CHUNK_SIZE, + OPT_LOG, + OPT_INNODB, + OPT_INNODB_DATA_FILE_PATH, + OPT_INNODB_DATA_HOME_DIR, + OPT_INNODB_ADAPTIVE_HASH_INDEX, + OPT_INNODB_DOUBLEWRITE, + OPT_INNODB_FILE_PER_TABLE, + OPT_INNODB_FLUSH_METHOD, + OPT_INNODB_LOG_GROUP_HOME_DIR, + OPT_INNODB_MAX_DIRTY_PAGES_PCT, + OPT_INNODB_MAX_PURGE_LAG, + OPT_INNODB_STATUS_FILE, + OPT_INNODB_AUTOEXTEND_INCREMENT, + OPT_INNODB_BUFFER_POOL_SIZE, + OPT_INNODB_COMMIT_CONCURRENCY, + OPT_INNODB_CONCURRENCY_TICKETS, + OPT_INNODB_FILE_IO_THREADS, + OPT_INNODB_IO_CAPACITY, + OPT_INNODB_READ_IO_THREADS, + OPT_INNODB_WRITE_IO_THREADS, + OPT_INNODB_USE_NATIVE_AIO, + OPT_INNODB_PAGE_SIZE, + OPT_INNODB_BUFFER_POOL_FILENAME, + OPT_INNODB_LOCK_WAIT_TIMEOUT, + OPT_INNODB_LOG_BUFFER_SIZE, +#if defined __linux__ || defined _WIN32 + OPT_INNODB_LOG_FILE_BUFFERING, +#endif + OPT_INNODB_LOG_FILE_SIZE, + OPT_INNODB_OPEN_FILES, + OPT_XTRA_DEBUG_SYNC, + OPT_INNODB_CHECKSUM_ALGORITHM, + OPT_INNODB_UNDO_DIRECTORY, + OPT_INNODB_UNDO_TABLESPACES, + OPT_XTRA_INCREMENTAL_FORCE_SCAN, + OPT_DEFAULTS_GROUP, + OPT_CLOSE_FILES, + OPT_CORE_FILE, + + OPT_COPY_BACK, + OPT_MOVE_BACK, + OPT_GALERA_INFO, + OPT_SLAVE_INFO, + OPT_NO_LOCK, + OPT_SAFE_SLAVE_BACKUP, + OPT_RSYNC, + OPT_FORCE_NON_EMPTY_DIRS, + OPT_NO_VERSION_CHECK, + OPT_NO_BACKUP_LOCKS, + OPT_DECOMPRESS, + OPT_INCREMENTAL_HISTORY_NAME, + OPT_INCREMENTAL_HISTORY_UUID, + OPT_REMOVE_ORIGINAL, + OPT_LOCK_WAIT_QUERY_TYPE, + OPT_KILL_LONG_QUERY_TYPE, + OPT_HISTORY, + OPT_KILL_LONG_QUERIES_TIMEOUT, + OPT_LOCK_WAIT_TIMEOUT, + OPT_LOCK_WAIT_THRESHOLD, + OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + OPT_BINLOG_INFO, + OPT_XB_SECURE_AUTH, + + OPT_XTRA_TABLES_EXCLUDE, + OPT_XTRA_DATABASES_EXCLUDE, + OPT_PROTOCOL, + OPT_INNODB_COMPRESSION_LEVEL, + OPT_LOCK_DDL_PER_TABLE, + OPT_ROCKSDB_DATADIR, + OPT_BACKUP_ROCKSDB, + OPT_XTRA_CHECK_PRIVILEGES, + OPT_XTRA_MYSQLD_ARGS, + OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION, + OPT_INNODB_FORCE_RECOVERY, + OPT_ARIA_LOG_DIR_PATH +}; + +struct my_option xb_client_options[]= { + {"verbose", 'V', "display verbose output", (G_PTR *) &verbose, + (G_PTR *) &verbose, 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0}, + {"version", 'v', "print version information", + (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + {"target-dir", OPT_XTRA_TARGET_DIR, "destination directory", + (G_PTR *) &xtrabackup_target_dir, (G_PTR *) &xtrabackup_target_dir, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"backup", OPT_XTRA_BACKUP, "take backup to target-dir", + (G_PTR *) &xtrabackup_backup, (G_PTR *) &xtrabackup_backup, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + {"prepare", OPT_XTRA_PREPARE, + "prepare a backup for starting mysql server on the backup.", + (G_PTR *) &xtrabackup_prepare, (G_PTR *) &xtrabackup_prepare, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + {"export", OPT_XTRA_EXPORT, + "create files to import to another database when prepare.", + (G_PTR *) &xtrabackup_export, (G_PTR *) &xtrabackup_export, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + {"print-param", OPT_XTRA_PRINT_PARAM, + "print parameter of mysqld needed for copyback.", + (G_PTR *) &xtrabackup_print_param, (G_PTR *) &xtrabackup_print_param, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"use-memory", OPT_XTRA_USE_MEMORY, + "The value is used in place of innodb_buffer_pool_size. " + "This option is only relevant when the --prepare option is specified.", + (G_PTR *) &xtrabackup_use_memory, (G_PTR *) &xtrabackup_use_memory, 0, + GET_LL, REQUIRED_ARG, 100 * 1024 * 1024L, 1024 * 1024L, LONGLONG_MAX, 0, + 1024 * 1024L, 0}, + {"throttle", OPT_XTRA_THROTTLE, + "limit count of IO operations (pairs of read&write) per second to IOS " + "values (for '--backup')", + (G_PTR *) &xtrabackup_throttle, (G_PTR *) &xtrabackup_throttle, 0, + GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0}, + {"log", OPT_LOG, "Ignored option for MySQL option compatibility", + (G_PTR *) &log_ignored_opt, (G_PTR *) &log_ignored_opt, 0, GET_STR, + OPT_ARG, 0, 0, 0, 0, 0, 0}, + {"log-copy-interval", OPT_XTRA_LOG_COPY_INTERVAL, + "time interval between checks done by log copying thread in milliseconds " + "(default is 1 second).", + (G_PTR *) &xtrabackup_log_copy_interval, + (G_PTR *) &xtrabackup_log_copy_interval, 0, GET_LONG, REQUIRED_ARG, 1000, + 0, LONG_MAX, 0, 1, 0}, + {"extra-lsndir", OPT_XTRA_EXTRA_LSNDIR, + "(for --backup): save an extra copy of the xtrabackup_checkpoints file " + "in this directory.", + (G_PTR *) &xtrabackup_extra_lsndir, (G_PTR *) &xtrabackup_extra_lsndir, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"incremental-lsn", OPT_XTRA_INCREMENTAL, + "(for --backup): copy only .ibd pages newer than specified LSN " + "'high:low'. ##ATTENTION##: If a wrong LSN value is specified, it is " + "impossible to diagnose this, causing the backup to be unusable. Be " + "careful!", + (G_PTR *) &xtrabackup_incremental, (G_PTR *) &xtrabackup_incremental, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"incremental-basedir", OPT_XTRA_INCREMENTAL_BASEDIR, + "(for --backup): copy only .ibd pages newer than backup at specified " + "directory.", + (G_PTR *) &xtrabackup_incremental_basedir, + (G_PTR *) &xtrabackup_incremental_basedir, 0, GET_STR, REQUIRED_ARG, 0, 0, + 0, 0, 0, 0}, + {"incremental-dir", OPT_XTRA_INCREMENTAL_DIR, + "(for --prepare): apply .delta files and logfile in the specified " + "directory.", + (G_PTR *) &xtrabackup_incremental_dir, + (G_PTR *) &xtrabackup_incremental_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, + 0, 0, 0}, + {"tables", OPT_XTRA_TABLES, "filtering by regexp for table names.", + (G_PTR *) &xtrabackup_tables, (G_PTR *) &xtrabackup_tables, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tables_file", OPT_XTRA_TABLES_FILE, + "filtering by list of the exact database.table name in the file.", + (G_PTR *) &xtrabackup_tables_file, (G_PTR *) &xtrabackup_tables_file, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.", + (G_PTR *) &xtrabackup_databases, (G_PTR *) &xtrabackup_databases, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"databases_file", OPT_XTRA_DATABASES_FILE, + "filtering by list of databases in the file.", + (G_PTR *) &xtrabackup_databases_file, + (G_PTR *) &xtrabackup_databases_file, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, + 0, 0, 0}, + {"tables-exclude", OPT_XTRA_TABLES_EXCLUDE, + "filtering by regexp for table names. " + "Operates the same way as --tables, but matched names are excluded from " + "backup. " + "Note that this option has a higher priority than --tables.", + (G_PTR *) &xtrabackup_tables_exclude, + (G_PTR *) &xtrabackup_tables_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, + 0, 0, 0}, + {"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE, + "Excluding databases based on name, " + "Operates the same way as --databases, but matched names are excluded " + "from backup. " + "Note that this option has a higher priority than --databases.", + (G_PTR *) &xtrabackup_databases_exclude, + (G_PTR *) &xtrabackup_databases_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0, + 0, 0, 0, 0}, + + {"stream", OPT_XTRA_STREAM, + "Stream all backup files to the standard output " + "in the specified format." + "Supported format is 'mbstream' or 'xbstream'.", + (G_PTR *) &xtrabackup_stream_str, (G_PTR *) &xtrabackup_stream_str, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress", OPT_XTRA_COMPRESS, + "Compress individual backup files using the " + "specified compression algorithm. Currently the only supported algorithm " + "is 'quicklz'. It is also the default algorithm, i.e. the one used when " + "--compress is used without an argument.", + (G_PTR *) &xtrabackup_compress_alg, (G_PTR *) &xtrabackup_compress_alg, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress-threads", OPT_XTRA_COMPRESS_THREADS, + "Number of threads for parallel data compression. The default value is " + "1.", + (G_PTR *) &xtrabackup_compress_threads, + (G_PTR *) &xtrabackup_compress_threads, 0, GET_UINT, REQUIRED_ARG, 1, 1, + UINT_MAX, 0, 0, 0}, + + {"compress-chunk-size", OPT_XTRA_COMPRESS_CHUNK_SIZE, + "Size of working buffer(s) for compression threads in bytes. The default " + "value is 64K.", + (G_PTR *) &xtrabackup_compress_chunk_size, + (G_PTR *) &xtrabackup_compress_chunk_size, 0, GET_ULL, REQUIRED_ARG, + (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"incremental-force-scan", OPT_XTRA_INCREMENTAL_FORCE_SCAN, + "Perform a full-scan incremental backup even in the presence of changed " + "page bitmap data", + (G_PTR *) &xtrabackup_incremental_force_scan, + (G_PTR *) &xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG, 0, 0, + 0, 0, 0, 0}, + + {"close_files", OPT_CLOSE_FILES, + "do not keep files opened. Use at your own " + "risk.", + (G_PTR *) &xb_close_files, (G_PTR *) &xb_close_files, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {"core-file", OPT_CORE_FILE, "Write core on fatal signals", 0, 0, 0, + GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"copy-back", OPT_COPY_BACK, + "Copy all the files in a previously made " + "backup from the backup directory to their original locations.", + (uchar *) &xtrabackup_copy_back, (uchar *) &xtrabackup_copy_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"move-back", OPT_MOVE_BACK, + "Move all the files in a previously made " + "backup from the backup directory to the actual datadir location. " + "Use with caution, as it removes backup files.", + (uchar *) &xtrabackup_move_back, (uchar *) &xtrabackup_move_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"galera-info", OPT_GALERA_INFO, + "This options creates the " + "xtrabackup_galera_info file which contains the local node state at " + "the time of the backup. Option should be used when performing the " + "backup of MariaDB Galera Cluster. Has no effect when backup locks " + "are used to create the backup.", + (uchar *) &opt_galera_info, (uchar *) &opt_galera_info, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"slave-info", OPT_SLAVE_INFO, + "This option is useful when backing " + "up a replication slave server. It prints the binary log position " + "and name of the master server. It also writes this information to " + "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. " + "A new slave for this master can be set up by starting a slave server " + "on this backup and issuing a \"CHANGE MASTER\" command with the " + "binary log position saved in the \"xtrabackup_slave_info\" file.", + (uchar *) &opt_slave_info, (uchar *) &opt_slave_info, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {"no-lock", OPT_NO_LOCK, + "Use this option to disable table lock " + "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + "tables are InnoDB and you DO NOT CARE about the binary log " + "position of the backup. This option shouldn't be used if there " + "are any DDL statements being executed or if any updates are " + "happening on non-InnoDB tables (this includes the system MyISAM " + "tables in the mysql database), otherwise it could lead to an " + "inconsistent backup. If you are considering to use --no-lock " + "because your backups are failing to acquire the lock, this could " + "be because of incoming replication events preventing the lock " + "from succeeding. Please try using --safe-slave-backup to " + "momentarily stop the replication slave thread, this may help " + "the backup to succeed and you then don't need to resort to " + "using this option.", + (uchar *) &opt_no_lock, (uchar *) &opt_no_lock, 0, GET_BOOL, NO_ARG, 0, 0, + 0, 0, 0, 0}, + + {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP, + "Stop slave SQL thread " + "and wait to start backup until Slave_open_temp_tables in " + "\"SHOW STATUS\" is zero. If there are no open temporary tables, " + "the backup will take place, otherwise the SQL thread will be " + "started and stopped until there are no open temporary tables. " + "The backup will fail if Slave_open_temp_tables does not become " + "zero after --safe-slave-backup-timeout seconds. The slave SQL " + "thread will be restarted when the backup finishes.", + (uchar *) &opt_safe_slave_backup, (uchar *) &opt_safe_slave_backup, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rsync", OPT_RSYNC, + "Uses the rsync utility to optimize local file " + "transfers. When this option is specified, " XB_TOOL_NAME " uses rsync " + "to copy all non-InnoDB files instead of spawning a separate cp for " + "each file, which can be much faster for servers with a large number " + "of databases or tables. This option cannot be used together with " + "--stream.", + (uchar *) &opt_rsync, (uchar *) &opt_rsync, 0, GET_BOOL, NO_ARG, 0, 0, 0, + 0, 0, 0}, + + {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, + "This " + "option, when specified, makes --copy-back or --move-back transfer " + "files to non-empty directories. Note that no existing files will be " + "overwritten. If --copy-back or --move-back has to copy a file from " + "the backup directory which already exists in the destination " + "directory, it will still fail with an error.", + (uchar *) &opt_force_non_empty_dirs, (uchar *) &opt_force_non_empty_dirs, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-version-check", OPT_NO_VERSION_CHECK, + "This option disables the " + "version check which is enabled by the --version-check option.", + (uchar *) &opt_noversioncheck, (uchar *) &opt_noversioncheck, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, + "This option controls if " + "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " + "on the backup stage. The option has no effect when backup locks are " + "not supported by the server. This option is enabled by default, " + "disable with --no-backup-locks.", + (uchar *) &opt_no_backup_locks, (uchar *) &opt_no_backup_locks, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"decompress", OPT_DECOMPRESS, + "Decompresses all files with the .qp " + "extension in a backup previously made with the --compress option.", + (uchar *) &opt_decompress, (uchar *) &opt_decompress, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {"user", 'u', + "This option specifies the MySQL username used " + "when connecting to the server, if that's not the current user. " + "The option accepts a string argument. See mysql --help for details.", + (uchar *) &opt_user, (uchar *) &opt_user, 0, GET_STR, REQUIRED_ARG, 0, 0, + 0, 0, 0, 0}, + + {"host", 'H', + "This option specifies the host to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + (uchar *) &opt_host, (uchar *) &opt_host, 0, GET_STR, REQUIRED_ARG, 0, 0, + 0, 0, 0, 0}, + + {"port", 'P', + "This option specifies the port to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + &opt_port, &opt_port, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"password", 'p', + "This option specifies the password to use " + "when connecting to the database. It accepts a string argument. " + "See mysql --help for details.", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"protocol", OPT_PROTOCOL, + "The protocol to use for connection (tcp, socket, pipe, memory).", 0, 0, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"socket", 'S', + "This option specifies the socket to use when " + "connecting to the local database server with a UNIX domain socket. " + "The option accepts a string argument. See mysql --help for details.", + (uchar *) &opt_socket, (uchar *) &opt_socket, 0, GET_STR, REQUIRED_ARG, 0, + 0, 0, 0, 0, 0}, + + {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME, + "This option specifies the name of the backup series stored in the " + XB_HISTORY_TABLE " history record to base an " + "incremental backup on. Xtrabackup will search the history table " + "looking for the most recent (highest innodb_to_lsn), successful " + "backup in the series and take the to_lsn value to use as the " + "starting lsn for the incremental backup. This will be mutually " + "exclusive with --incremental-history-uuid, --incremental-basedir " + "and --incremental-lsn. If no valid lsn can be found (no series by " + "that name, no successful backups by that name), an error will be returned." + " It is used with the --incremental option.", + (uchar *) &opt_incremental_history_name, + (uchar *) &opt_incremental_history_name, 0, GET_STR, REQUIRED_ARG, 0, 0, + 0, 0, 0, 0}, + + {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID, + "This option specifies the UUID of the specific history record " + "stored in the " XB_HISTORY_TABLE " table to base an " + "incremental backup on. --incremental-history-name, " + "--incremental-basedir and --incremental-lsn. If no valid lsn can be " + "found (no success record with that uuid), an error will be returned." + " It is used with the --incremental option.", + (uchar *) &opt_incremental_history_uuid, + (uchar *) &opt_incremental_history_uuid, 0, GET_STR, REQUIRED_ARG, 0, 0, + 0, 0, 0, 0}, + + {"remove-original", OPT_REMOVE_ORIGINAL, + "Remove .qp files after decompression.", (uchar *) &opt_remove_original, + (uchar *) &opt_remove_original, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, + "This option specifies which types of queries are allowed to complete " + "before " XB_TOOL_NAME " will issue the global lock. Default is all.", + (uchar *) &opt_lock_wait_query_type, (uchar *) &opt_lock_wait_query_type, + &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, + 0}, + + {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, + "This option specifies which types of queries should be killed to " + "unblock the global lock. Default is \"all\".", + (uchar *) &opt_kill_long_query_type, (uchar *) &opt_kill_long_query_type, + &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0, + 0, 0}, + + {"history", OPT_HISTORY, + "This option enables the tracking of backup history in the " + XB_HISTORY_TABLE " table. An optional history " + "series name may be specified that will be placed with the history " + "record for the current backup being taken.", + NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, + "This option specifies the number of seconds " XB_TOOL_NAME " waits " + "between starting FLUSH TABLES WITH READ LOCK and killing those " + "queries that block it. Default is 0 seconds, which means " + XB_TOOL_NAME " will not attempt to kill any queries.", + (uchar *) &opt_kill_long_queries_timeout, + (uchar *) &opt_kill_long_queries_timeout, 0, GET_UINT, REQUIRED_ARG, 0, 0, + 0, 0, 0, 0}, + + {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that " XB_TOOL_NAME " should wait " + "for queries that would block FTWRL before running it. If there are " + "still such queries when the timeout expires, " XB_TOOL_NAME " terminates " + "with an error. Default is 0, in which case " XB_TOOL_NAME " does not " + "wait for queries to complete and starts FTWRL immediately.", + (uchar *) &opt_lock_wait_timeout, (uchar *) &opt_lock_wait_timeout, 0, + GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, + "This option specifies the query run time threshold which is used by " + XB_TOOL_NAME " to detect long-running queries with a non-zero value " + "of --ftwrl-wait-timeout. FTWRL is not started until such " + "long-running queries exist. This option has no effect if " + "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", + (uchar *) &opt_lock_wait_threshold, (uchar *) &opt_lock_wait_threshold, 0, + GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, + + + {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + "How many seconds --safe-slave-backup should wait for " + "Slave_open_temp_tables to become zero. (default 300)", + (uchar *) &opt_safe_slave_backup_timeout, + (uchar *) &opt_safe_slave_backup_timeout, 0, GET_UINT, REQUIRED_ARG, 300, + 0, 0, 0, 0, 0}, + + {"binlog-info", OPT_BINLOG_INFO, + "This option controls how backup should retrieve server's binary log " + "coordinates corresponding to the backup. Possible values are OFF, ON, " + "LOCKLESS and AUTO.", + &opt_binlog_info, &opt_binlog_info, &binlog_info_typelib, GET_ENUM, + OPT_ARG, BINLOG_INFO_AUTO, 0, 0, 0, 0, 0}, + + {"secure-auth", OPT_XB_SECURE_AUTH, + "Refuse client connecting to server if it" + " uses old (pre-4.1.1) protocol.", + &opt_secure_auth, &opt_secure_auth, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, + 0}, + + {"log-innodb-page-corruption", OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION, + "Continue backup if innodb corrupted pages are found. The pages are " + "logged in " MB_CORRUPTED_PAGES_FILE + " and backup is finished with error. " + "--prepare will try to fix corrupted pages. If " MB_CORRUPTED_PAGES_FILE + " exists after --prepare in base backup directory, backup still contains " + "corrupted pages and can not be considered as consistent.", + &opt_log_innodb_page_corruption, &opt_log_innodb_page_corruption, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + +#define MYSQL_CLIENT +#include "sslopt-longopts.h" +#undef MYSQL_CLIENT + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}}; + +uint xb_client_options_count = array_elements(xb_client_options); + +#ifndef DBUG_OFF +/** Parameters to DBUG */ +static const char *dbug_option; +#endif + +#ifdef HAVE_URING +extern const char *io_uring_may_be_unsafe; +bool innodb_use_native_aio_default(); +#endif + +struct my_option xb_server_options[] = +{ + {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home, + (G_PTR*) &mysql_data_home, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tmpdir", 't', + "Path for temporary files. Several paths may be specified, separated by a " +#if defined(_WIN32) + "semicolon (;)" +#else + "colon (:)" +#endif + ", in this case they are used in a round-robin fashion.", + (G_PTR*) &opt_mysql_tmpdir, + (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"parallel", OPT_XTRA_PARALLEL, + "Number of threads to use for parallel datafiles transfer. " + "The default value is 1.", + (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT, + REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, + + {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION, + "Enable extended validation for Innodb data pages during backup phase. " + "Will slow down backup considerably, in case encryption is used. " + "May fail if tables are created during the backup.", + (G_PTR*)&opt_extended_validation, + (G_PTR*)&opt_extended_validation, + 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0}, + + {"encrypted_backup", OPT_XTRA_ENCRYPTED_BACKUP, + "In --backup, assume that nonzero key_version implies that the page" + " is encrypted. Use --backup --skip-encrypted-backup to allow" + " copying unencrypted that were originally created before MySQL 5.1.48.", + (G_PTR*)&opt_encrypted_backup, + (G_PTR*)&opt_encrypted_backup, + 0, GET_BOOL, NO_ARG, TRUE, 0, 0, 0, 0, 0}, + + {"log", OPT_LOG, "Ignored option for MySQL option compatibility", + (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"log_bin", OPT_LOG, "Base name for the log sequence", + &opt_log_bin, &opt_log_bin, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility", + (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, +#ifdef BTR_CUR_HASH_ADAPT + {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX, + "Enable InnoDB adaptive hash index (disabled by default).", + &btr_search_enabled, + &btr_search_enabled, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, +#endif /* BTR_CUR_HASH_ADAPT */ + {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT, + "Data file autoextend increment in megabytes", + (G_PTR*) &sys_tablespace_auto_extend_increment, + (G_PTR*) &sys_tablespace_auto_extend_increment, + 0, GET_UINT, REQUIRED_ARG, 8, 1, 1000, 0, 1, 0}, + {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH, + "Path to individual files and their sizes.", &innobase_data_file_path, + &innobase_data_file_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR, + "The common part for InnoDB table spaces.", &innobase_data_home_dir, + &innobase_data_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE, + "Enable InnoDB doublewrite buffer during --prepare.", + (G_PTR*) &srv_use_doublewrite_buf, + (G_PTR*) &srv_use_doublewrite_buf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY, + "Number of IOPs the server can do. Tunes the background IO rate", + (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity, + 0, GET_ULONG, OPT_ARG, 200, 100, ~0UL, 0, 0, 0}, + {"innodb_file_io_threads", OPT_INNODB_FILE_IO_THREADS, + "Number of file I/O threads in InnoDB.", (G_PTR*) &innobase_file_io_threads, + (G_PTR*) &innobase_file_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 4, 64, 0, + 1, 0}, + {"innodb_read_io_threads", OPT_INNODB_READ_IO_THREADS, + "Number of background read I/O threads in InnoDB.", (G_PTR*) &innobase_read_io_threads, + (G_PTR*) &innobase_read_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0, + 1, 0}, + {"innodb_write_io_threads", OPT_INNODB_WRITE_IO_THREADS, + "Number of background write I/O threads in InnoDB.", (G_PTR*) &innobase_write_io_threads, + (G_PTR*) &innobase_write_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0, + 1, 0}, + {"innodb_file_per_table", OPT_INNODB_FILE_PER_TABLE, + "Stores each InnoDB table to an .ibd file in the database dir.", + (G_PTR*) &srv_file_per_table, + (G_PTR*) &srv_file_per_table, 0, GET_BOOL, NO_ARG, + FALSE, 0, 0, 0, 0, 0}, + + {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD, + "With which method to flush data.", + &srv_file_flush_method, &srv_file_flush_method, + &innodb_flush_method_typelib, GET_ENUM, REQUIRED_ARG, + IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_O_DIRECT), 0, 0, 0, 0, 0}, + + {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE, + "Redo log buffer size in bytes.", + (G_PTR*) &log_sys.buf_size, (G_PTR*) &log_sys.buf_size, 0, + IF_WIN(GET_ULL,GET_ULONG), REQUIRED_ARG, 2U << 20, + 2U << 20, SIZE_T_MAX, 0, 4096, 0}, +#if defined __linux__ || defined _WIN32 + {"innodb_log_file_buffering", OPT_INNODB_LOG_FILE_BUFFERING, + "Whether the file system cache for ib_logfile0 is enabled during --backup", + (G_PTR*) &log_sys.log_buffered, + (G_PTR*) &log_sys.log_buffered, 0, GET_BOOL, NO_ARG, + TRUE, 0, 0, 0, 0, 0}, +#endif + {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE, + "Ignored for mysqld option compatibility", + (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0, + GET_ULL, REQUIRED_ARG, 96 << 20, 4 << 20, + std::numeric_limits<ulonglong>::max(), 0, 4096, 0}, + {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR, + "Path to InnoDB log files.", &srv_log_group_home_dir, + &srv_log_group_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT, + "Percentage of dirty pages allowed in bufferpool.", (G_PTR*) &srv_max_buf_pool_modified_pct, + (G_PTR*) &srv_max_buf_pool_modified_pct, 0, GET_ULONG, REQUIRED_ARG, 90, 0, 100, 0, 0, 0}, + {"innodb_use_native_aio", OPT_INNODB_USE_NATIVE_AIO, + "Use native AIO if supported on this platform.", + (G_PTR*) &srv_use_native_aio, + (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG, +#ifdef HAVE_URING + innodb_use_native_aio_default(), +#else + TRUE, +#endif + 0, 0, 0, 0, 0}, + {"innodb_page_size", OPT_INNODB_PAGE_SIZE, + "The universal page size of the database.", + (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0, + /* Use GET_LL to support numeric suffixes in 5.6 */ + GET_LL, REQUIRED_ARG, + (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0}, + {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME, + "Ignored for mysqld option compatibility", + (G_PTR*) &innobase_buffer_pool_filename, + (G_PTR*) &innobase_buffer_pool_filename, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + +#ifndef DBUG_OFF /* unfortunately "debug" collides with existing options */ + {"dbug", '#', "Built in DBUG debugger.", + &dbug_option, &dbug_option, 0, GET_STR, OPT_ARG, + 0, 0, 0, 0, 0, 0}, +#endif + + {"innodb_checksum_algorithm", OPT_INNODB_CHECKSUM_ALGORITHM, + "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, " + "FULL_CRC32, STRICT_FULL_CRC32]", &srv_checksum_algorithm, + &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, + REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_CRC32, 0, 0, 0, 0, 0}, + + {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY, + "Directory where undo tablespace files live, this path can be absolute.", + &srv_undo_dir, &srv_undo_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, + 0}, + + {"innodb_undo_tablespaces", OPT_INNODB_UNDO_TABLESPACES, + "Number of undo tablespaces to use.", + (G_PTR*)&srv_undo_tablespaces, (G_PTR*)&srv_undo_tablespaces, + 0, GET_UINT, REQUIRED_ARG, 0, 0, 126, 0, 1, 0}, + + {"innodb_compression_level", OPT_INNODB_COMPRESSION_LEVEL, + "Compression level used for zlib compression.", + (G_PTR*)&page_zip_level, (G_PTR*)&page_zip_level, + 0, GET_UINT, REQUIRED_ARG, 6, 0, 9, 0, 0, 0}, + + {"defaults_group", OPT_DEFAULTS_GROUP, "defaults group in config file (default \"mysqld\").", + (G_PTR*) &defaults_group, (G_PTR*) &defaults_group, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"plugin-dir", OPT_PLUGIN_DIR, + "Server plugin directory. Used to load plugins during 'prepare' phase." + "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", + &xb_plugin_dir, &xb_plugin_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + {"aria_log_dir_path", OPT_ARIA_LOG_DIR_PATH, + "Path to individual files and their sizes.", + &aria_log_dir_path, &aria_log_dir_path, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " + "descriptors to reserve with setrlimit().", + (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, + REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0}, + + {"lock-ddl-per-table", OPT_LOCK_DDL_PER_TABLE, "Lock DDL for each table " + "before backup starts to copy it and until the backup is completed.", + (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory." + "This option is only used with --copy-back or --move-back option", + &xb_rocksdb_datadir, &xb_rocksdb_datadir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + {"rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed." + "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data", + &xb_backup_rocksdb, &xb_backup_rocksdb, + 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, + + {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user " + "privileges fro the backup user", + &opt_check_privileges, &opt_check_privileges, + 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, + + {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY, + "(for --prepare): Crash recovery mode (ignores " + "page corruption; for emergencies only).", + (G_PTR*)&srv_force_recovery, + (G_PTR*)&srv_force_recovery, + 0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0}, + + {"mysqld-args", OPT_XTRA_MYSQLD_ARGS, + "All arguments that follow this argument are considered as server " + "options, and if some of them are not supported by mariabackup, they " + "will be ignored.", + (G_PTR *) &xtrabackup_mysqld_args, (G_PTR *) &xtrabackup_mysqld_args, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"help", '?', + "Display this help and exit.", + (G_PTR *) &xtrabackup_help, (G_PTR *) &xtrabackup_help, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +uint xb_server_options_count = array_elements(xb_server_options); + + +static std::set<std::string> tables_for_export; + +static void append_export_table(const char *dbname, const char *tablename, + bool is_remote, bool skip_node_page0, + uint32_t defer_space_id) +{ + if(dbname && tablename && !is_remote) + { + char buf[3*FN_REFLEN]; + snprintf(buf,sizeof(buf),"%s/%s",dbname, tablename); + // trim .ibd + char *p=strrchr(buf, '.'); + if (p) *p=0; + + std::string name=ut_get_name(0, buf); + /* Strip partition name comment from table name, if any */ + if (ends_with(name.c_str(), "*/")) + { + size_t pos= name.rfind("/*"); + if (pos != std::string::npos) + name.resize(pos); + } + tables_for_export.insert(name); + } +} + + +#define BOOTSTRAP_FILENAME "mariabackup_prepare_for_export.sql" + +static int create_bootstrap_file() +{ + FILE *f= fopen(BOOTSTRAP_FILENAME,"wb"); + if(!f) + return -1; + + fputs("SET NAMES UTF8;\n",f); + enumerate_ibd_files(append_export_table); + for (std::set<std::string>::iterator it = tables_for_export.begin(); + it != tables_for_export.end(); it++) + { + const char *tab = it->c_str(); + fprintf(f, + "BEGIN NOT ATOMIC " + "DECLARE CONTINUE HANDLER FOR NOT FOUND,SQLEXCEPTION BEGIN END;" + "FLUSH TABLES %s FOR EXPORT;" + "END;\n" + "UNLOCK TABLES;\n", + tab); + } + fclose(f); + return 0; +} + +static int prepare_export() +{ + int err= -1; + + char cmdline[2*FN_REFLEN]; + FILE *outf; + + if (create_bootstrap_file()) + return -1; + + // Process defaults-file , it can have some --lc-language stuff, + // which is* unfortunately* still necessary to get mysqld up + if (strncmp(orig_argv1,"--defaults-file=", 16) == 0) + { + snprintf(cmdline, sizeof cmdline, + IF_WIN("\"","") "\"%s\" --mysqld \"%s\"" + " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." + " --innodb --innodb-fast-shutdown=0 --loose-partition" + " --innodb-buffer-pool-size=%llu" + " --console --skip-log-error --skip-log-bin --bootstrap %s< " + BOOTSTRAP_FILENAME IF_WIN("\"",""), + mariabackup_exe, + orig_argv1, (my_defaults_group_suffix?my_defaults_group_suffix:""), + xtrabackup_use_memory, + (srv_force_recovery ? "--innodb-force-recovery=1 " : "")); + } + else + { + snprintf(cmdline, sizeof cmdline, + IF_WIN("\"","") "\"%s\" --mysqld" + " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." + " --innodb --innodb-fast-shutdown=0 --loose-partition" + " --innodb-buffer-pool-size=%llu" + " --console --log-error= --skip-log-bin --bootstrap %s< " + BOOTSTRAP_FILENAME IF_WIN("\"",""), + mariabackup_exe, + (my_defaults_group_suffix?my_defaults_group_suffix:""), + xtrabackup_use_memory, + (srv_force_recovery ? "--innodb-force-recovery=1 " : "")); + } + + msg("Prepare export : executing %s\n", cmdline); + fflush(stderr); + + outf= popen(cmdline,"r"); + if (!outf) + goto end; + + char outline[FN_REFLEN]; + while (fgets(outline, FN_REFLEN - 1, outf)) + fprintf(stderr,"%s",outline); + + err = pclose(outf); +end: + unlink(BOOTSTRAP_FILENAME); + return err; +} + +static const char *xb_client_default_groups[]= { + "client", "client-server", "client-mariadb", 0, 0, 0}; + +static const char *backup_default_groups[]= { + "xtrabackup", "mariabackup", "mariadb-backup", 0, 0, 0}; + +static void print_version(void) +{ + fprintf(stderr, "%s based on MariaDB server %s %s (%s)\n", + my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE); +} + +static void concatenate_default_groups(std::vector<const char*> &backup_load_groups, const char **default_groups) +{ + for ( ; *default_groups ; default_groups++) + backup_load_groups.push_back(*default_groups); +} + +static void usage(void) +{ + puts("Open source backup tool for InnoDB and XtraDB\n\ +\n\ +Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\ +Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\ +\n\ +This program is free software; you can redistribute it and/or\n\ +modify it under the terms of the GNU General Public License\n\ +as published by the Free Software Foundation version 2\n\ +of the License.\n\ +\n\ +This program is distributed in the hope that it will be useful,\n\ +but WITHOUT ANY WARRANTY; without even the implied warranty of\n\ +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\ +GNU General Public License for more details.\n\ +\n\ +You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n"); + + printf("Usage: %s [--defaults-file=#] [--backup | --prepare | --copy-back | --move-back] [OPTIONS]\n",my_progname); + std::vector<const char*> backup_load_default_groups; + concatenate_default_groups(backup_load_default_groups, backup_default_groups); + concatenate_default_groups(backup_load_default_groups, load_default_groups); + backup_load_default_groups.push_back(nullptr); + print_defaults("my", &backup_load_default_groups[0]); + my_print_help(xb_client_options); + my_print_help(xb_server_options); + my_print_variables(xb_server_options); + my_print_variables(xb_client_options); +} + +#define ADD_PRINT_PARAM_OPT(value) \ + { \ + print_param_str << opt->name << "=" << value << "\n"; \ + param_set.insert(opt->name); \ + } + +/************************************************************************ +Check if parameter is set in defaults file or via command line argument +@return true if parameter is set. */ +bool +check_if_param_set(const char *param) +{ + return param_set.find(param) != param_set.end(); +} + +my_bool +xb_get_one_option(const struct my_option *opt, + const char *argument, const char *) +{ + switch(opt->id) { + case 'h': + strmake(mysql_real_data_home,argument, FN_REFLEN - 1); + mysql_data_home= mysql_real_data_home; + + ADD_PRINT_PARAM_OPT(mysql_real_data_home); + break; + + case 't': + + ADD_PRINT_PARAM_OPT(opt_mysql_tmpdir); + break; + + case OPT_INNODB_DATA_HOME_DIR: + + ADD_PRINT_PARAM_OPT(innobase_data_home_dir); + break; + + case OPT_INNODB_DATA_FILE_PATH: + + ADD_PRINT_PARAM_OPT(innobase_data_file_path); + break; + + case OPT_INNODB_LOG_GROUP_HOME_DIR: + + ADD_PRINT_PARAM_OPT(srv_log_group_home_dir); + break; + + case OPT_INNODB_FLUSH_METHOD: +#ifdef _WIN32 + /* From: storage/innobase/handler/ha_innodb.cc:innodb_init_params */ + switch (srv_file_flush_method) { + case SRV_ALL_O_DIRECT_FSYNC + 1 /* "async_unbuffered"="unbuffered" */: + srv_file_flush_method= SRV_ALL_O_DIRECT_FSYNC; + break; + case SRV_ALL_O_DIRECT_FSYNC + 2 /* "normal"="fsync" */: + srv_file_flush_method= SRV_FSYNC; + break; + } +#endif + ut_a(srv_file_flush_method + <= IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_O_DIRECT_NO_FSYNC)); + ADD_PRINT_PARAM_OPT(innodb_flush_method_names[srv_file_flush_method]); + break; + + case OPT_INNODB_PAGE_SIZE: + + ADD_PRINT_PARAM_OPT(innobase_page_size); + break; + + case OPT_INNODB_UNDO_DIRECTORY: + + ADD_PRINT_PARAM_OPT(srv_undo_dir); + break; + + case OPT_INNODB_UNDO_TABLESPACES: + + ADD_PRINT_PARAM_OPT(srv_undo_tablespaces); + break; + + case OPT_INNODB_CHECKSUM_ALGORITHM: + + ut_a(srv_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32); + + ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]); + break; + + case OPT_INNODB_COMPRESSION_LEVEL: + ADD_PRINT_PARAM_OPT(page_zip_level); + break; + + case OPT_INNODB_BUFFER_POOL_FILENAME: + + ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename); + break; + + case OPT_INNODB_FORCE_RECOVERY: + + if (srv_force_recovery) { + ADD_PRINT_PARAM_OPT(srv_force_recovery); + } + break; + + case OPT_ARIA_LOG_DIR_PATH: + ADD_PRINT_PARAM_OPT(aria_log_dir_path); + break; + + case OPT_XTRA_TARGET_DIR: + strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1); + xtrabackup_target_dir= xtrabackup_real_target_dir; + break; + case OPT_XTRA_STREAM: + if (!strcasecmp(argument, "mbstream") || + !strcasecmp(argument, "xbstream")) + xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM; + else + { + msg("Invalid --stream argument: %s", argument); + return 1; + } + xtrabackup_stream = TRUE; + break; + case OPT_XTRA_COMPRESS: + if (argument == NULL) + xtrabackup_compress_alg = "quicklz"; + else if (strcasecmp(argument, "quicklz")) + { + msg("Invalid --compress argument: %s", argument); + return 1; + } + xtrabackup_compress = TRUE; + break; + case OPT_DECOMPRESS: + opt_decompress = TRUE; + xtrabackup_decrypt_decompress = true; + break; + case (int) OPT_CORE_FILE: + test_flags |= TEST_CORE_ON_SIGNAL; + break; + case OPT_HISTORY: + if (argument) { + opt_history = argument; + } else { + opt_history = ""; + } + break; + case 'p': + opt_password = argument; + break; + case OPT_PROTOCOL: + if (argument) + { + if ((opt_protocol= find_type_with_warning(argument, &sql_protocol_typelib, + opt->name)) <= 0) + { + sf_leaking_memory= 1; /* no memory leak reports here */ + exit(1); + } + } + break; +#define MYSQL_CLIENT +#include "sslopt-case.h" +#undef MYSQL_CLIENT + + case '?': + usage(); + exit(EXIT_SUCCESS); + break; + case 'v': + print_version(); + exit(EXIT_SUCCESS); + break; + default: + break; + } + return 0; +} + +static bool innodb_init_param() +{ + srv_is_being_started = TRUE; + /* === some variables from mysqld === */ + memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list)); + + if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) { + msg("init_tmpdir() failed"); + return true; + } + xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list); + /* dummy for initialize all_charsets[] */ + get_charset_name(0); + + srv_page_size = 0; + srv_page_size_shift = 0; +#ifdef BTR_CUR_HASH_ADAPT + btr_ahi_parts = 1; +#endif /* BTR_CUR_HASH_ADAPT */ + + if (innobase_page_size != (1LL << 14)) { + size_t n_shift = get_bit_shift(size_t(innobase_page_size)); + + if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) { + srv_page_size_shift = uint32_t(n_shift); + srv_page_size = 1U << n_shift; + msg("InnoDB: The universal page size of the " + "database is set to %lu.", srv_page_size); + } else { + msg("invalid value of " + "innobase_page_size: %lld", innobase_page_size); + goto error; + } + } else { + srv_page_size_shift = 14; + srv_page_size = 1U << 14; + } + + /* Check that values don't overflow on 32-bit systems. */ + if (sizeof(ulint) == 4) { + if (xtrabackup_use_memory > UINT_MAX32) { + msg("mariabackup: use-memory can't be over 4GB" + " on 32-bit systems"); + } + } + + static char default_path[2] = { FN_CURLIB, 0 }; + fil_path_to_mysql_datadir = default_path; + + /* Set InnoDB initialization parameters according to the values + read from MySQL .cnf file */ + + if (xtrabackup_backup) { + msg("mariabackup: using the following InnoDB configuration:"); + } else { + msg("mariabackup: using the following InnoDB configuration " + "for recovery:"); + } + + /*--------------- Data files -------------------------*/ + + /* The default dir for data files is the datadir of MySQL */ + + srv_data_home = (xtrabackup_backup && innobase_data_home_dir + ? innobase_data_home_dir : default_path); + msg("innodb_data_home_dir = %s", srv_data_home); + + /* Set default InnoDB data file size to 10 MB and let it be + auto-extending. Thus users can use InnoDB in >= 4.0 without having + to specify any startup options. */ + + if (!innobase_data_file_path) { + innobase_data_file_path = (char*) "ibdata1:10M:autoextend"; + } + msg("innodb_data_file_path = %s", + innobase_data_file_path); + + srv_sys_space.set_space_id(TRX_SYS_SPACE); + srv_sys_space.set_path(srv_data_home); + switch (srv_checksum_algorithm) { + case SRV_CHECKSUM_ALGORITHM_FULL_CRC32: + case SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32: + srv_sys_space.set_flags(FSP_FLAGS_FCRC32_MASK_MARKER + | FSP_FLAGS_FCRC32_PAGE_SSIZE()); + break; + default: + srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE()); + } + + if (!srv_sys_space.parse_params(innobase_data_file_path, true)) { + goto error; + } + + srv_sys_space.normalize_size(); + srv_lock_table_size = 5 * (srv_buf_pool_size >> srv_page_size_shift); + + /* -------------- Log files ---------------------------*/ + + /* The default dir for log files is the datadir of MySQL */ + + if (!(xtrabackup_backup && srv_log_group_home_dir)) { + srv_log_group_home_dir = default_path; + } + if (xtrabackup_prepare && xtrabackup_incremental_dir) { + srv_log_group_home_dir = xtrabackup_incremental_dir; + } + msg("innodb_log_group_home_dir = %s", + srv_log_group_home_dir); + + if (strchr(srv_log_group_home_dir, ';')) { + msg("syntax error in innodb_log_group_home_dir, "); + goto error; + } + + srv_adaptive_flushing = FALSE; + + /* We set srv_pool_size here in units of 1 kB. InnoDB internally + changes the value so that it becomes the number of database pages. */ + + srv_buf_pool_size = (ulint) xtrabackup_use_memory; + srv_buf_pool_chunk_unit = srv_buf_pool_size; + + srv_n_read_io_threads = (uint) innobase_read_io_threads; + srv_n_write_io_threads = (uint) innobase_write_io_threads; + + srv_max_n_open_files = ULINT_UNDEFINED - 5; + + srv_print_verbose_log = verbose ? 2 : 1; + + /* Store the default charset-collation number of this MySQL + installation */ + + /* We cannot treat characterset here for now!! */ + data_mysql_default_charset_coll = (ulint)default_charset_info->number; + + ut_ad(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number); + +#ifdef _WIN32 + srv_use_native_aio = TRUE; + +#elif defined(LINUX_NATIVE_AIO) + + if (srv_use_native_aio) { + msg("InnoDB: Using Linux native AIO"); + } +#elif defined(HAVE_URING) + if (!srv_use_native_aio) { + } else if (io_uring_may_be_unsafe) { + msg("InnoDB: Using liburing on this kernel %s may cause hangs;" + " see https://jira.mariadb.org/browse/MDEV-26674", + io_uring_may_be_unsafe); + } else { + msg("InnoDB: Using liburing"); + } +#else + /* Currently native AIO is supported only on windows and linux + and that also when the support is compiled in. In all other + cases, we ignore the setting of innodb_use_native_aio. */ + srv_use_native_aio = FALSE; + +#endif + + /* Assign the default value to srv_undo_dir if it's not specified, as + my_getopt does not support default values for string options. We also + ignore the option and override innodb_undo_directory on --prepare, + because separate undo tablespaces are copied to the root backup + directory. */ + + if (!srv_undo_dir || !xtrabackup_backup) { + srv_undo_dir = (char*) "."; + } + + compile_time_assert(SRV_FORCE_IGNORE_CORRUPT == 1); + + /* + * This option can be read both from the command line, and the + * defaults file. The assignment should account for both cases, + * and for "--innobackupex". Since the command line argument is + * parsed after the defaults file, it takes precedence. + */ + if (xtrabackup_innodb_force_recovery) { + srv_force_recovery = xtrabackup_innodb_force_recovery; + } + + if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { + if (!xtrabackup_prepare) { + msg("mariabackup: The option \"innodb_force_recovery\"" + " should only be used with \"%s\".", + (innobackupex_mode ? "--apply-log" : "--prepare")); + goto error; + } else { + msg("innodb_force_recovery = %lu", srv_force_recovery); + } + } + +#ifdef _WIN32 + srv_use_native_aio = TRUE; +#endif + return false; + +error: + msg("mariabackup: innodb_init_param(): Error occurred.\n"); + return true; +} + +static byte log_hdr_buf[log_t::START_OFFSET + SIZE_OF_FILE_CHECKPOINT]; + +/** Initialize an InnoDB log file header in log_hdr_buf[] */ +static void log_hdr_init() +{ + memset(log_hdr_buf, 0, sizeof log_hdr_buf); + mach_write_to_4(LOG_HEADER_FORMAT + log_hdr_buf, log_t::FORMAT_10_8); + mach_write_to_8(LOG_HEADER_START_LSN + log_hdr_buf, + log_sys.next_checkpoint_lsn); + snprintf(reinterpret_cast<char*>(LOG_HEADER_CREATOR + log_hdr_buf), + 16, "Backup %u.%u.%u", + MYSQL_VERSION_ID / 10000, MYSQL_VERSION_ID / 100 % 100, + MYSQL_VERSION_ID % 100); + if (log_sys.is_encrypted()) + log_crypt_write_header(log_hdr_buf + LOG_HEADER_CREATOR_END); + mach_write_to_4(508 + log_hdr_buf, my_crc32c(0, log_hdr_buf, 508)); + mach_write_to_8(log_hdr_buf + 0x1000, log_sys.next_checkpoint_lsn); + mach_write_to_8(log_hdr_buf + 0x1008, recv_sys.lsn); + mach_write_to_4(log_hdr_buf + 0x103c, + my_crc32c(0, log_hdr_buf + 0x1000, 60)); +} + +static bool innodb_init() +{ + bool create_new_db= false; + + srv_max_io_capacity= srv_io_capacity >= SRV_MAX_IO_CAPACITY_LIMIT / 2 + ? SRV_MAX_IO_CAPACITY_LIMIT : std::max(2 * srv_io_capacity, 2000UL); + + /* Check if the data files exist or not. */ + dberr_t err= srv_sys_space.check_file_spec(&create_new_db, 5U << 20); + + if (create_new_db) + { + msg("mariadb-backup: InnoDB files do not exist"); + return true; + } + + if (err == DB_SUCCESS) + err= srv_start(false); + + if (err != DB_SUCCESS) + { + msg("mariadb-backup: srv_start() returned %d (%s).", err, ut_strerr(err)); + return true; + } + + ut_ad(srv_force_recovery <= SRV_FORCE_IGNORE_CORRUPT); + ut_ad(recv_no_log_write); + buf_flush_sync(); + recv_sys.debug_free(); + ut_ad(!os_aio_pending_reads()); + ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); + ut_ad(!buf_pool.get_oldest_modification(0)); + ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex)); + /* os_aio_pending_writes() may hold here if some write_io_callback() + did not release the slot yet. However, the page write itself must + have completed, because the buf_pool.flush_list is empty. In debug + builds, we wait for this to happen, hoping to get a hung process if + this assumption does not hold. */ + ut_d(os_aio_wait_until_no_pending_writes(false)); + log_sys.close_file(); + + if (xtrabackup_incremental) + /* Reset the ib_logfile0 in --target-dir, not --incremental-dir. */ + srv_log_group_home_dir= xtrabackup_target_dir; + + bool ret; + const std::string ib_logfile0{get_log_file_path()}; + os_file_delete_if_exists_func(ib_logfile0.c_str(), nullptr); + os_file_t file= os_file_create_func(ib_logfile0.c_str(), + OS_FILE_CREATE, OS_FILE_NORMAL, + OS_DATA_FILE_NO_O_DIRECT, false, &ret); + if (!ret) + { + invalid_log: + msg("mariadb-backup: Cannot create %s", ib_logfile0.c_str()); + return true; + } + + recv_sys.lsn= log_sys.next_checkpoint_lsn= + log_sys.get_lsn() - SIZE_OF_FILE_CHECKPOINT; + log_sys.set_latest_format(false); // not encrypted + log_hdr_init(); + byte *b= &log_hdr_buf[log_t::START_OFFSET]; + b[0]= FILE_CHECKPOINT | 10; + mach_write_to_8(b + 3, recv_sys.lsn); + b[11]= 1; + mach_write_to_4(b + 12, my_crc32c(0, b, 11)); + static_assert(12 + 4 == SIZE_OF_FILE_CHECKPOINT, "compatibility"); + +#ifdef _WIN32 + DWORD len; + ret= WriteFile(file, log_hdr_buf, sizeof log_hdr_buf, + &len, nullptr) && len == sizeof log_hdr_buf; +#else + ret= sizeof log_hdr_buf == write(file, log_hdr_buf, sizeof log_hdr_buf); +#endif + if (!os_file_close_func(file) || !ret) + goto invalid_log; + return false; +} + +/* ================= common ================= */ + +/*********************************************************************** +Read backup meta info. +@return TRUE on success, FALSE on failure. */ +static +my_bool +xtrabackup_read_metadata(char *filename) +{ + FILE *fp; + my_bool r = TRUE; + + fp = fopen(filename,"r"); + if(!fp) { + msg("Error: cannot open %s", filename); + return(FALSE); + } + + if (fscanf(fp, "backup_type = %29s\n", metadata_type) + != 1) { + r = FALSE; + goto end; + } + /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file + format. */ + if (fscanf(fp, "from_lsn = " UINT64PF "\n", &metadata_from_lsn) + != 1) { + r = FALSE; + goto end; + } + if (fscanf(fp, "to_lsn = " UINT64PF "\n", &metadata_to_lsn) + != 1) { + r = FALSE; + goto end; + } + if (fscanf(fp, "last_lsn = " UINT64PF "\n", &metadata_last_lsn) + != 1) { + metadata_last_lsn = 0; + } + /* Optional fields */ + +end: + fclose(fp); + + return(r); +} + +/*********************************************************************** +Print backup meta info to a specified buffer. */ +static +void +xtrabackup_print_metadata(char *buf, size_t buf_len) +{ + /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file + format. */ + snprintf(buf, buf_len, + "backup_type = %s\n" + "from_lsn = " UINT64PF "\n" + "to_lsn = " UINT64PF "\n" + "last_lsn = " UINT64PF "\n", + metadata_type, + metadata_from_lsn, + metadata_to_lsn, + metadata_last_lsn); +} + +/*********************************************************************** +Stream backup meta info to a specified datasink. +@return TRUE on success, FALSE on failure. */ +static +my_bool +xtrabackup_stream_metadata(ds_ctxt_t *ds_ctxt) +{ + char buf[1024]; + size_t len; + ds_file_t *stream; + MY_STAT mystat; + my_bool rc = TRUE; + + xtrabackup_print_metadata(buf, sizeof(buf)); + + len = strlen(buf); + + mystat.st_size = len; + mystat.st_mtime = my_time(0); + + stream = ds_open(ds_ctxt, XTRABACKUP_METADATA_FILENAME, &mystat); + if (stream == NULL) { + msg("Error: cannot open output stream for %s", XTRABACKUP_METADATA_FILENAME); + return(FALSE); + } + + if (ds_write(stream, buf, len)) { + rc = FALSE; + } + + if (ds_close(stream)) { + rc = FALSE; + } + + return(rc); +} + +/*********************************************************************** +Write backup meta info to a specified file. +@return TRUE on success, FALSE on failure. */ +static +my_bool +xtrabackup_write_metadata(const char *filepath) +{ + char buf[1024]; + size_t len; + FILE *fp; + + xtrabackup_print_metadata(buf, sizeof(buf)); + + len = strlen(buf); + + fp = fopen(filepath, "w"); + if(!fp) { + msg("Error: cannot open %s", filepath); + return(FALSE); + } + if (fwrite(buf, len, 1, fp) < 1) { + fclose(fp); + return(FALSE); + } + + fclose(fp); + + return(TRUE); +} + +/*********************************************************************** +Read meta info for an incremental delta. +@return TRUE on success, FALSE on failure. */ +static my_bool +xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) +{ + FILE* fp; + char key[51]; + char value[51]; + my_bool r = TRUE; + + /* set defaults */ + ulint page_size = ULINT_UNDEFINED, zip_size = 0; + info->space_id = UINT32_MAX; + + fp = fopen(filepath, "r"); + if (!fp) { + /* Meta files for incremental deltas are optional */ + return(TRUE); + } + + while (!feof(fp)) { + if (fscanf(fp, "%50s = %50s\n", key, value) == 2) { + if (strcmp(key, "page_size") == 0) { + page_size = strtoul(value, NULL, 10); + } else if (strcmp(key, "zip_size") == 0) { + zip_size = strtoul(value, NULL, 10); + } else if (strcmp(key, "space_id") == 0) { + info->space_id = static_cast<uint32_t> + (strtoul(value, NULL, 10)); + } + } + } + + fclose(fp); + + if (page_size == ULINT_UNDEFINED) { + msg("page_size is required in %s", filepath); + r = FALSE; + } else { + info->page_size = page_size; + info->zip_size = zip_size; + } + + if (info->space_id == UINT32_MAX) { + msg("mariabackup: Warning: This backup was taken with XtraBackup 2.0.1 " + "or earlier, some DDL operations between full and incremental " + "backups may be handled incorrectly"); + } + + return(r); +} + +/*********************************************************************** +Write meta info for an incremental delta. +@return TRUE on success, FALSE on failure. */ +my_bool +xb_write_delta_metadata(ds_ctxt *ds_meta, + const char *filename, const xb_delta_info_t *info) +{ + ds_file_t *f; + char buf[64]; + my_bool ret; + size_t len; + MY_STAT mystat; + + snprintf(buf, sizeof(buf), + "page_size = " ULINTPF "\n" + "zip_size = " ULINTPF " \n" + "space_id = %u\n", + info->page_size, + info->zip_size, + info->space_id); + len = strlen(buf); + + mystat.st_size = len; + mystat.st_mtime = my_time(0); + + f = ds_open(ds_meta, filename, &mystat); + if (f == NULL) { + msg("Error: Can't open output stream for %s",filename); + return(FALSE); + } + + ret = (ds_write(f, buf, len) == 0); + + if (ds_close(f)) { + ret = FALSE; + } + + return(ret); +} + +/* ================= backup ================= */ +void xtrabackup_io_throttling() +{ + if (!xtrabackup_backup || !xtrabackup_throttle) + return; + + mysql_mutex_lock(&recv_sys.mutex); + if (io_ticket-- < 0) + mysql_cond_wait(&wait_throttle, &recv_sys.mutex); + mysql_mutex_unlock(&recv_sys.mutex); +} + +static +my_bool regex_list_check_match( + const regex_list_t& list, + const char* name) +{ + regmatch_t tables_regmatch[1]; + for (regex_list_t::const_iterator i = list.begin(), end = list.end(); + i != end; ++i) { + const regex_t& regex = *i; + int regres = regexec(®ex, name, 1, tables_regmatch, 0); + + if (regres != REG_NOMATCH) { + return(TRUE); + } + } + return(FALSE); +} + +static +my_bool +find_filter_in_hashtable( + const char* name, + hash_table_t* table, + xb_filter_entry_t** result +) +{ + xb_filter_entry_t* found = NULL; + const ulint fold = my_crc32c(0, name, strlen(name)); + HASH_SEARCH(name_hash, table, fold, + xb_filter_entry_t*, + found, (void) 0, + !strcmp(found->name, name)); + + if (found && result) { + *result = found; + } + return (found != NULL); +} + +/************************************************************************ +Checks if a given table name matches any of specifications given in +regex_list or tables_hash. + +@return TRUE on match or both regex_list and tables_hash are empty.*/ +static my_bool +check_if_table_matches_filters(const char *name, + const regex_list_t& regex_list, + hash_table_t* tables_hash) +{ + if (regex_list.empty() && !tables_hash->array) { + return(FALSE); + } + + if (regex_list_check_match(regex_list, name)) { + return(TRUE); + } + + return tables_hash->array && + find_filter_in_hashtable(name, tables_hash, NULL); +} + +enum skip_database_check_result { + DATABASE_SKIP, + DATABASE_SKIP_SOME_TABLES, + DATABASE_DONT_SKIP, + DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED, +}; + +/************************************************************************ +Checks if a database specified by name should be skipped from backup based on +the --databases, --databases_file or --databases_exclude options. + +@return TRUE if entire database should be skipped, + FALSE otherwise. +*/ +static +skip_database_check_result +check_if_skip_database( + const char* name /*!< in: path to the database */ +) +{ + /* There are some filters for databases, check them */ + xb_filter_entry_t* database = NULL; + + if (databases_exclude_hash.array && + find_filter_in_hashtable(name, &databases_exclude_hash, + &database) && + (!database->has_tables || !databases_include_hash.array)) { + /* Database is found and there are no tables specified, + skip entire db. */ + return DATABASE_SKIP; + } + + if (databases_include_hash.array) { + if (!find_filter_in_hashtable(name, &databases_include_hash, + &database)) { + /* Database isn't found, skip the database */ + return DATABASE_SKIP; + } else if (database->has_tables) { + return DATABASE_SKIP_SOME_TABLES; + } else { + return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED; + } + } + + return DATABASE_DONT_SKIP; +} + +/************************************************************************ +Checks if a database specified by path should be skipped from backup based on +the --databases, --databases_file or --databases_exclude options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_database_by_path( + const char* path /*!< in: path to the db directory. */ +) +{ + if (!databases_include_hash.array && !databases_exclude_hash.array) { + return(FALSE); + } + + const char* db_name = strrchr(path, '/'); +#ifdef _WIN32 + if (const char* last = strrchr(path, '\\')) { + if (!db_name || last > db_name) { + db_name = last; + } + } +#endif + + if (db_name == NULL) { + db_name = path; + } else { + ++db_name; + } + + return check_if_skip_database(db_name) == DATABASE_SKIP; +} + +/************************************************************************ +Checks if a table specified as a name in the form "database/name" (InnoDB 5.6) +or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on +the --tables or --tables-file options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_table( +/******************/ + const char* name) /*!< in: path to the table */ +{ + char buf[FN_REFLEN]; + const char *dbname, *tbname; + const char *ptr; + char *eptr; + + + dbname = NULL; + tbname = name; + for (;;) { + ptr= strchr(tbname, '/'); +#ifdef _WIN32 + if (!ptr) { + ptr= strchr(tbname,'\\'); + } +#endif + if (!ptr) { + break; + } + dbname = tbname; + tbname = ptr + 1; + } + + if (strncmp(tbname, tmp_file_prefix, tmp_file_prefix_length) == 0) { + return TRUE; + } + + if (regex_exclude_list.empty() && + regex_include_list.empty() && + !tables_include_hash.array && + !tables_exclude_hash.array && + !databases_include_hash.array && + !databases_exclude_hash.array) { + return(FALSE); + } + + if (dbname == NULL) { + return(FALSE); + } + + strncpy(buf, dbname, FN_REFLEN - 1); + buf[FN_REFLEN - 1] = '\0'; + buf[tbname - 1 - dbname] = '\0'; + + const skip_database_check_result skip_database = + check_if_skip_database(buf); + if (skip_database == DATABASE_SKIP) { + return (TRUE); + } + + buf[tbname - 1 - dbname] = '.'; + + /* Check if there's a suffix in the table name. If so, truncate it. We + rely on the fact that a dot cannot be a part of a table name (it is + encoded by the server with the @NNNN syntax). */ + if ((eptr = strchr(&buf[tbname - dbname], '.')) != NULL) { + + *eptr = '\0'; + } + + /* For partitioned tables first try to match against the regexp + without truncating the #P#... suffix so we can backup individual + partitions with regexps like '^test[.]t#P#p5' */ + if (check_if_table_matches_filters(buf, regex_exclude_list, + &tables_exclude_hash)) { + return(TRUE); + } + if (check_if_table_matches_filters(buf, regex_include_list, + &tables_include_hash)) { + return(FALSE); + } + if ((eptr = strstr(buf, "#P#")) != NULL) { + *eptr = 0; + + if (check_if_table_matches_filters(buf, regex_exclude_list, + &tables_exclude_hash)) { + return (TRUE); + } + if (check_if_table_matches_filters(buf, regex_include_list, + &tables_include_hash)) { + return(FALSE); + } + } + + if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) { + /* Database is in include-list, and qualified name wasn't + found in any of exclusion filters.*/ + return (FALSE); + } + + if (skip_database == DATABASE_SKIP_SOME_TABLES || + !regex_include_list.empty() || + tables_include_hash.array) { + + /* Include lists are present, but qualified name + failed to match any.*/ + return(TRUE); + } + + return(FALSE); +} + +const char* +xb_get_copy_action(const char *dflt) +{ + const char *action; + + if (xtrabackup_stream) { + if (xtrabackup_compress) { + action = "Compressing and streaming"; + } else { + action = "Streaming"; + } + } else { + if (xtrabackup_compress) { + action = "Compressing"; + } else { + action = dflt; + } + } + + return(action); +} + + +/** Copy innodb data file to the specified destination. + +@param[in] node file node of a tablespace +@param[in] thread_n thread id, used in the text of diagnostic messages +@param[in] dest_name destination file name +@param[in] write_filter write filter to copy data, can be pass-through filter +for full backup, pages filter for incremental backup, etc. + +@return FALSE on success and TRUE on error */ +static my_bool xtrabackup_copy_datafile(ds_ctxt *ds_data, + ds_ctxt *ds_meta, + fil_node_t *node, uint thread_n, + const char *dest_name, + const xb_write_filt_t &write_filter, + CorruptedPages &corrupted_pages) +{ + char dst_name[FN_REFLEN]; + ds_file_t *dstfile = NULL; + xb_fil_cur_t cursor; + xb_fil_cur_result_t res; + xb_write_filt_ctxt_t write_filt_ctxt; + const char *action; + xb_read_filt_t *read_filter; + my_bool rc = FALSE; + + if (fil_is_user_tablespace_id(node->space->id) + && check_if_skip_table(filename_to_spacename(node->name, + strlen(node->name)). + c_str())) { + msg(thread_n, "Skipping %s.", node->name); + return(FALSE); + } + + memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t)); + + bool was_dropped; + mysql_mutex_lock(&recv_sys.mutex); + was_dropped = (ddl_tracker.drops.find(node->space->id) != ddl_tracker.drops.end()); + mysql_mutex_unlock(&recv_sys.mutex); + if (was_dropped) { + if (node->is_open()) { + mysql_mutex_lock(&fil_system.mutex); + node->close(); + mysql_mutex_unlock(&fil_system.mutex); + } + goto skip; + } + + if (!changed_page_bitmap) { + read_filter = &rf_pass_through; + } + else { + read_filter = &rf_bitmap; + } + + res = xb_fil_cur_open(&cursor, read_filter, node, thread_n, ULLONG_MAX); + if (res == XB_FIL_CUR_SKIP) { + goto skip; + } else if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + strncpy(dst_name, dest_name ? dest_name : cursor.rel_path, + sizeof dst_name - 1); + dst_name[sizeof dst_name - 1] = '\0'; + + ut_a(write_filter.process != NULL); + + if (write_filter.init != NULL && + !write_filter.init(ds_meta, &write_filt_ctxt, dst_name, &cursor, + opt_log_innodb_page_corruption ? &corrupted_pages : NULL)) { + msg (thread_n, "mariabackup: error: failed to initialize page write filter."); + goto error; + } + + dstfile = ds_open(ds_data, dst_name, &cursor.statinfo); + if (dstfile == NULL) { + msg(thread_n,"mariabackup: error: can't open the destination stream for %s", dst_name); + goto error; + } + + action = xb_get_copy_action(); + + if (xtrabackup_stream) { + msg(thread_n, "%s %s", action, node->name); + } else { + msg(thread_n, "%s %s to %s", action, node->name, + dstfile->path); + } + + /* The main copy loop */ + while (1) { + res = xb_fil_cur_read(&cursor, corrupted_pages); + if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + if (res == XB_FIL_CUR_EOF) { + break; + } + + if (!write_filter.process(&write_filt_ctxt, dstfile)) { + goto error; + } + + if (res == XB_FIL_CUR_SKIP) { + mysql_mutex_lock(&recv_sys.mutex); + fail_undo_ids.insert( + static_cast<uint32_t>(cursor.space_id)); + mysql_mutex_unlock(&recv_sys.mutex); + break; + } + } + + if (write_filter.finalize + && !write_filter.finalize(&write_filt_ctxt, dstfile)) { + goto error; + } else { + const fil_space_t::name_type name = node->space->name(); + + mysql_mutex_lock(&recv_sys.mutex); + ddl_tracker.tables_in_backup.emplace(node->space->id, + std::string(name.data(), + name.size())); + mysql_mutex_unlock(&recv_sys.mutex); + } + + /* close */ + msg(thread_n," ...done"); + xb_fil_cur_close(&cursor); + if (ds_close(dstfile)) { + rc = TRUE; + } + if (write_filter.deinit) { + write_filter.deinit(&write_filt_ctxt); + } + return(rc); + +error: + xb_fil_cur_close(&cursor); + if (dstfile != NULL) { + ds_close(dstfile); + } + if (write_filter.deinit) { + write_filter.deinit(&write_filt_ctxt);; + } + msg(thread_n, "mariabackup: xtrabackup_copy_datafile() failed."); + return(TRUE); /*ERROR*/ + +skip: + + if (dstfile != NULL) { + ds_close(dstfile); + } + if (write_filter.deinit) { + write_filter.deinit(&write_filt_ctxt); + } + msg(thread_n,"Warning: We assume the table was dropped during xtrabackup execution and ignore the tablespace %s", node->name); + return(FALSE); +} + +/** Copy redo log until the current end of the log is reached +@return whether the operation failed */ +static bool xtrabackup_copy_logfile() +{ + mysql_mutex_assert_owner(&recv_sys.mutex); + DBUG_EXECUTE_IF("log_checksum_mismatch", return false;); + + ut_a(dst_log_file); + ut_ad(recv_sys.is_initialised()); + const size_t sequence_offset{log_sys.is_encrypted() ? 8U + 5U : 5U}; + const size_t block_size_1{log_sys.get_block_size() - 1}; + + ut_ad(!log_sys.is_pmem()); + + { + recv_sys.offset= size_t(recv_sys.lsn - log_sys.get_first_lsn()) & + block_size_1; + recv_sys.len= 0; + } + + for (unsigned retry_count{0};;) + { + recv_sys_t::parse_mtr_result r; + size_t start_offset{recv_sys.offset}; + + { + { + auto source_offset= + log_sys.calc_lsn_offset(recv_sys.lsn + recv_sys.len - + recv_sys.offset); + source_offset&= ~block_size_1; + size_t size{log_sys.buf_size - recv_sys.len}; + if (UNIV_UNLIKELY(source_offset + size > log_sys.file_size)) + { + const size_t first{size_t(log_sys.file_size - source_offset)}; + ut_ad(first <= log_sys.buf_size); + log_sys.log.read(source_offset, {log_sys.buf, first}); + size-= first; + if (log_sys.START_OFFSET + size > source_offset) + size= size_t(source_offset - log_sys.START_OFFSET); + if (size) + log_sys.log.read(log_sys.START_OFFSET, + {log_sys.buf + first, size}); + size+= first; + } + else + log_sys.log.read(source_offset, {log_sys.buf, size}); + recv_sys.len= size; + } + + if (log_sys.buf[recv_sys.offset] <= 1) + break; + + if (recv_sys.parse_mtr<false>(false) == recv_sys_t::OK) + { + do + { + /* Set the sequence bit (the backed-up log will not wrap around) */ + byte *seq= &log_sys.buf[recv_sys.offset - sequence_offset]; + ut_ad(*seq == log_sys.get_sequence_bit(recv_sys.lsn - + sequence_offset)); + *seq= 1; + } + while ((r= recv_sys.parse_mtr<false>(false)) == recv_sys_t::OK); + + if (ds_write(dst_log_file, log_sys.buf + start_offset, + recv_sys.offset - start_offset)) + { + msg("Error: write to ib_logfile0 failed"); + return true; + } + else + { + const auto ofs= recv_sys.offset & ~block_size_1; + memmove_aligned<64>(log_sys.buf, log_sys.buf + ofs, + recv_sys.len - ofs); + recv_sys.len-= ofs; + recv_sys.offset&= block_size_1; + } + + pthread_cond_broadcast(&scanned_lsn_cond); + + if (r == recv_sys_t::GOT_EOF) + break; + + if (recv_sys.offset < log_sys.get_block_size()) + break; + + if (xtrabackup_throttle && io_ticket-- < 0) + mysql_cond_wait(&wait_throttle, &recv_sys.mutex); + + retry_count= 0; + continue; + } + else + { + recv_sys.len= recv_sys.offset & ~block_size_1; + if (retry_count == 100) + break; + + mysql_mutex_unlock(&recv_sys.mutex); + if (!retry_count++) + msg("Retrying read of log at LSN=" LSN_PF, recv_sys.lsn); + my_sleep(1000); + } + } + mysql_mutex_lock(&recv_sys.mutex); + } + + if (verbose) + msg(">> log scanned up to (" LSN_PF ")", recv_sys.lsn); + return false; +} + +/** +Wait until redo log copying thread processes given lsn +*/ +void backup_wait_for_lsn(lsn_t lsn) +{ + mysql_mutex_lock(&recv_sys.mutex); + for (lsn_t last_lsn{recv_sys.lsn}; last_lsn < lsn; ) + { + timespec abstime; + set_timespec(abstime, 5); + if (my_cond_timedwait(&scanned_lsn_cond, &recv_sys.mutex.m_mutex, + &abstime) && + last_lsn == recv_sys.lsn) + die("Was only able to copy log from " LSN_PF " to " LSN_PF + ", not " LSN_PF "; try increasing innodb_log_file_size", + log_sys.next_checkpoint_lsn, last_lsn, lsn); + last_lsn= recv_sys.lsn; + } + mysql_mutex_unlock(&recv_sys.mutex); +} + +extern lsn_t server_lsn_after_lock; + +static void log_copying_thread() +{ + my_thread_init(); + mysql_mutex_lock(&recv_sys.mutex); + while (!xtrabackup_copy_logfile() && + (!metadata_to_lsn || metadata_to_lsn > recv_sys.lsn)) + { + timespec abstime; + set_timespec_nsec(abstime, 1000000ULL * xtrabackup_log_copy_interval); + mysql_cond_timedwait(&log_copying_stop, &recv_sys.mutex, &abstime); + } + log_copying_running= false; + mysql_mutex_unlock(&recv_sys.mutex); + my_thread_end(); +} + +/** whether io_watching_thread() is active; protected by recv_sys.mutex */ +static bool have_io_watching_thread; + +/* io throttle watching (rough) */ +static void io_watching_thread() +{ + my_thread_init(); + /* currently, for --backup only */ + ut_ad(xtrabackup_backup); + + mysql_mutex_lock(&recv_sys.mutex); + ut_ad(have_io_watching_thread); + + while (log_copying_running && !metadata_to_lsn) + { + timespec abstime; + set_timespec(abstime, 1); + mysql_cond_timedwait(&log_copying_stop, &recv_sys.mutex, &abstime); + io_ticket= xtrabackup_throttle; + mysql_cond_broadcast(&wait_throttle); + } + + /* stop io throttle */ + xtrabackup_throttle= 0; + have_io_watching_thread= false; + mysql_cond_broadcast(&wait_throttle); + mysql_mutex_unlock(&recv_sys.mutex); + my_thread_end(); +} + +#ifndef DBUG_OFF +char *dbug_mariabackup_get_val(const char *event, + const fil_space_t::name_type key) +{ + char envvar[FN_REFLEN]; + strncpy(envvar, event, sizeof envvar - 1); + envvar[(sizeof envvar) - 1] = '\0'; + + if (key.size() && key.size() + strlen(envvar) < (sizeof envvar) - 2) + { + strcat(envvar, "_"); + strncat(envvar, key.data(), key.size()); + if (char *slash= strchr(envvar, '/')) + *slash= '_'; + } + + char *val = getenv(envvar); + return val && *val ? val : nullptr; +} + +/* +In debug mode, execute SQL statement that was passed via environment. +To use this facility, you need to + +1. Add code DBUG_EXECUTE_MARIABACKUP_EVENT("my_event_name", key);); + to the code. key is usually a table name +2. Set environment variable my_event_name_$key SQL statement you want to execute + when event occurs, in DBUG_EXECUTE_IF from above. + In mtr , you can set environment via 'let' statement (do not use $ as the first char + for the variable) +3. start mariabackup with --dbug=+d,debug_mariabackup_events +*/ +void dbug_mariabackup_event(const char *event, + const fil_space_t::name_type key) +{ + char *sql = dbug_mariabackup_get_val(event, key); + if (sql && *sql) { + msg("dbug_mariabackup_event : executing '%s'", sql); + xb_mysql_query(mysql_connection, sql, false, true); + } +} +#endif // DBUG_OFF + +/** Datafiles copying thread.*/ +static void data_copy_thread_func(data_thread_ctxt_t *ctxt) /* thread context */ +{ + uint num = ctxt->num; + fil_node_t* node; + ut_ad(ctxt->corrupted_pages); + + /* + Initialize mysys thread-specific memory so we can + use mysys functions in this thread. + */ + my_thread_init(); + + while ((node = datafiles_iter_next(ctxt->it)) != NULL) { + DBUG_MARIABACKUP_EVENT("before_copy", node->space->name()); + DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", + node->space->name(), + backup_wait_for_lsn(get_current_lsn(mysql_connection));); + /* copy the datafile */ + if (xtrabackup_copy_datafile(ctxt->datasinks->m_data, + ctxt->datasinks->m_meta, node, num, NULL, + xtrabackup_incremental ? wf_incremental : wf_write_through, + *ctxt->corrupted_pages)) + die("failed to copy datafile."); + + DBUG_MARIABACKUP_EVENT("after_copy", node->space->name()); + } + + pthread_mutex_lock(ctxt->count_mutex); + (*ctxt->count)--; + pthread_mutex_unlock(ctxt->count_mutex); + + my_thread_end(); +} + +/************************************************************************ +Initialize the appropriate datasink(s). Both local backups and streaming in the +'xbstream' format allow parallel writes so we can write directly. + +Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks +for the data stream (and don't allow parallel data copying) and for metainfo +files (including ib_logfile0). The second datasink writes to temporary +files first, and then streams them in a serialized way when closed. */ +void Backup_datasinks::init() +{ + /* Start building out the pipelines from the terminus back */ + if (xtrabackup_stream) { + /* All streaming goes to stdout */ + m_data = m_meta = m_redo = ds_create(xtrabackup_target_dir, + DS_TYPE_STDOUT); + } else { + /* Local filesystem */ + m_data = m_meta = m_redo = ds_create(xtrabackup_target_dir, + DS_TYPE_LOCAL); + } + + /* Track it for destruction */ + add_datasink_to_destroy(m_data); + + /* Stream formatting */ + if (xtrabackup_stream) { + ds_ctxt_t *ds; + + ut_a(xtrabackup_stream_fmt == XB_STREAM_FMT_XBSTREAM); + ds = ds_create(xtrabackup_target_dir, DS_TYPE_XBSTREAM); + + add_datasink_to_destroy(ds); + + ds_set_pipe(ds, m_data); + m_data = ds; + + + m_redo = m_meta = m_data; + } + + /* Compression for m_data and m_redo */ + if (xtrabackup_compress) { + ds_ctxt_t *ds; + + /* Use a 1 MB buffer for compressed output stream */ + ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); + ds_buffer_set_size(ds, 1024 * 1024); + add_datasink_to_destroy(ds); + ds_set_pipe(ds, m_data); + if (m_data != m_redo) { + m_data = ds; + ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); + ds_buffer_set_size(ds, 1024 * 1024); + add_datasink_to_destroy(ds); + ds_set_pipe(ds, m_redo); + m_redo = ds; + } else { + m_redo = m_data = ds; + } + + ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS); + add_datasink_to_destroy(ds); + ds_set_pipe(ds, m_data); + if (m_data != m_redo) { + m_data = ds; + ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS); + add_datasink_to_destroy(ds); + ds_set_pipe(ds, m_redo); + m_redo = ds; + } else { + m_redo = m_data = ds; + } + } +} + +#define SRV_MAX_N_PENDING_SYNC_IOS 100 + +/** Initialize the tablespace cache subsystem. */ +static +void +xb_fil_io_init() +{ + fil_system.create(srv_file_per_table ? 50000 : 5000); + fil_system.freeze_space_list = 1; + fil_system.space_id_reuse_warned = true; +} + +/** Load tablespace. + +@param[in] dirname directory name of the tablespace to open +@param[in] filname file name of the tablespece to open +@param[in] is_remote true if tablespace file is .isl +@param[in] skip_node_page0 true if we don't need to read node page 0. Otherwise +node page0 will be read, and it's size and free pages limit +will be set from page 0, what is neccessary for checking and fixing corrupted +pages. +@param[in] defer_space_id use the space id to create space object +when there is deferred tablespace +*/ +static void xb_load_single_table_tablespace(const char *dirname, + const char *filname, + bool is_remote, + bool skip_node_page0, + uint32_t defer_space_id) +{ + ut_ad(srv_operation == SRV_OPERATION_BACKUP + || srv_operation == SRV_OPERATION_RESTORE_DELTA + || srv_operation == SRV_OPERATION_RESTORE + || srv_operation == SRV_OPERATION_BACKUP_NO_DEFER); + /* Ignore .isl files on XtraBackup recovery. All tablespaces must be + local. */ + if (is_remote && srv_operation == SRV_OPERATION_RESTORE_DELTA) { + return; + } + if (check_if_skip_table(filname)) { + return; + } + + /* The name ends in .ibd or .isl; + try opening the file */ + char* name; + size_t dirlen = dirname == NULL ? 0 : strlen(dirname); + size_t namelen = strlen(filname); + ulint pathlen = dirname == NULL ? namelen + 1: dirlen + namelen + 2; + dberr_t err; + fil_space_t *space; + bool defer = false; + + name = static_cast<char*>(ut_malloc_nokey(pathlen)); + + if (dirname != NULL) { + snprintf(name, pathlen, "%s/%s", dirname, filname); + name[pathlen - 5] = 0; + } else { + snprintf(name, pathlen, "%s", filname); + name[pathlen - 5] = 0; + } + + const fil_space_t::name_type n{name, pathlen - 5}; + Datafile *file; + + if (is_remote) { + RemoteDatafile* rf = new RemoteDatafile(); + if (!rf->open_link_file(n)) { + die("Can't open datafile %s", name); + } + file = rf; + } else { + file = new Datafile(); + file->make_filepath(".", n, IBD); + } + + if (file->open_read_only(true) != DB_SUCCESS) { + die("Can't open datafile %s", name); + } + + for (int i = 0; i < 10; i++) { + file->m_defer = false; + err = file->validate_first_page(); + + if (file->m_defer) { + if (defer_space_id) { + defer = true; + file->set_space_id(defer_space_id); + file->set_flags(FSP_FLAGS_PAGE_SSIZE()); + err = DB_SUCCESS; + break; + } + } else if (err != DB_CORRUPTION) { + break; + } + + my_sleep(1000); + } + + if (!defer && file->m_defer) { + const char *file_path = file->filepath(); + defer_space_names.insert( + filename_to_spacename( + file_path, strlen(file_path))); + delete file; + ut_free(name); + return; + } + + bool is_empty_file = file->exists() && file->is_empty_file(); + + if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) { + mysql_mutex_lock(&fil_system.mutex); + space = fil_space_t::create( + file->space_id(), file->flags(), + FIL_TYPE_TABLESPACE, nullptr/* TODO: crypt_data */, + FIL_ENCRYPTION_DEFAULT, + file->handle() != OS_FILE_CLOSED); + ut_ad(space); + fil_node_t* node= space->add( + file->filepath(), + skip_node_page0 ? file->detach() : pfs_os_file_t(), + 0, false, false); + node->deferred= defer; + if (!space->read_page0()) + err = DB_CANNOT_OPEN_FILE; + mysql_mutex_unlock(&fil_system.mutex); + + if (srv_operation == SRV_OPERATION_RESTORE_DELTA + || xb_close_files) { + space->close(); + } + } + + delete file; + + if (err != DB_SUCCESS && xtrabackup_backup && !is_empty_file) { + die("Failed to validate first page of the file %s, error %d",name, (int)err); + } + + ut_free(name); +} + +static void xb_load_single_table_tablespace(const std::string &space_name, + bool skip_node_page0, + uint32_t defer_space_id) +{ + std::string name(space_name); + bool is_remote= access((name + ".ibd").c_str(), R_OK) != 0; + const char *extension= is_remote ? ".isl" : ".ibd"; + name.append(extension); + char buf[FN_REFLEN]; + strncpy(buf, name.c_str(), sizeof buf - 1); + buf[sizeof buf - 1]= '\0'; + const char *dbname= buf; + char *p= strchr(buf, '/'); + if (!p) + die("Unexpected tablespace %s filename %s", space_name.c_str(), + name.c_str()); + *p= 0; + const char *tablename= p + 1; + xb_load_single_table_tablespace(dbname, tablename, is_remote, + skip_node_page0, defer_space_id); +} + +#ifdef _WIN32 +/** +The os_file_opendir() function opens a directory stream corresponding to the +directory named by the dirname argument. The directory stream is positioned +at the first entry. In both Unix and Windows we automatically skip the '.' +and '..' items at the start of the directory listing. +@param[in] dirname directory name; it must not contain a trailing + '\' or '/' +@return directory stream, NULL if error */ +os_file_dir_t os_file_opendir(const char *dirname) +{ + char path[OS_FILE_MAX_PATH + 3]; + + ut_a(strlen(dirname) < OS_FILE_MAX_PATH); + + strcpy(path, dirname); + strcpy(path + strlen(path), "\\*"); + + /* Note that in Windows opening the 'directory stream' also retrieves + the first entry in the directory. Since it is '.', that is no problem, + as we will skip over the '.' and '..' entries anyway. */ + + LPWIN32_FIND_DATA lpFindFileData= static_cast<LPWIN32_FIND_DATA> + (ut_malloc_nokey(sizeof(WIN32_FIND_DATA))); + os_file_dir_t dir= FindFirstFile((LPCTSTR) path, lpFindFileData); + ut_free(lpFindFileData); + + return dir; +} +#endif + +/** This function returns information of the next file in the directory. We jump +over the '.' and '..' entries in the directory. +@param[in] dirname directory name or path +@param[in] dir directory stream +@param[out] info buffer where the info is returned +@return 0 if ok, -1 if error, 1 if at the end of the directory */ +int +os_file_readdir_next_file( + const char* dirname, + os_file_dir_t dir, + os_file_stat_t* info) +{ +#ifdef _WIN32 + BOOL ret; + int status; + WIN32_FIND_DATA find_data; + +next_file: + ret = FindNextFile(dir, &find_data); + + if (ret > 0) { + + const char* name; + + name = static_cast<const char*>(find_data.cFileName); + + ut_a(strlen(name) < OS_FILE_MAX_PATH); + + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) { + + goto next_file; + } + + strcpy(info->name, name); + + info->size = find_data.nFileSizeHigh; + info->size <<= 32; + info->size |= find_data.nFileSizeLow; + + if (find_data.dwFileAttributes + & FILE_ATTRIBUTE_REPARSE_POINT) { + + /* TODO: test Windows symlinks */ + /* TODO: MySQL has apparently its own symlink + implementation in Windows, dbname.sym can + redirect a database directory: + REFMAN "windows-symbolic-links.html" */ + + info->type = OS_FILE_TYPE_LINK; + + } else if (find_data.dwFileAttributes + & FILE_ATTRIBUTE_DIRECTORY) { + + info->type = OS_FILE_TYPE_DIR; + + } else { + + /* It is probably safest to assume that all other + file types are normal. Better to check them rather + than blindly skip them. */ + + info->type = OS_FILE_TYPE_FILE; + } + + status = 0; + + } else { + DWORD err = GetLastError(); + if (err == ERROR_NO_MORE_FILES) { + status = 1; + } else { + msg("FindNextFile in %s returned %lu", dirname, err); + status = -1; + } + } + + return(status); +#else + struct dirent* ent; + char* full_path; + int ret; + struct stat statinfo; + +next_file: + + ent = readdir(dir); + + if (ent == NULL) { + + return(1); + } + + ut_a(strlen(ent->d_name) < OS_FILE_MAX_PATH); + + if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) { + + goto next_file; + } + + strcpy(info->name, ent->d_name); + + full_path = static_cast<char*>( + ut_malloc_nokey(strlen(dirname) + strlen(ent->d_name) + 10)); + if (!full_path) { + return -1; + } + + sprintf(full_path, "%s/%s", dirname, ent->d_name); + + ret = stat(full_path, &statinfo); + + if (ret) { + + if (errno == ENOENT) { + /* readdir() returned a file that does not exist, + it must have been deleted in the meantime. Do what + would have happened if the file was deleted before + readdir() - ignore and go to the next entry. + If this is the last entry then info->name will still + contain the name of the deleted file when this + function returns, but this is not an issue since the + caller shouldn't be looking at info when end of + directory is returned. */ + + ut_free(full_path); + + goto next_file; + } + + msg("stat %s: Got error %d", full_path, errno); + + ut_free(full_path); + + return(-1); + } + + info->size = statinfo.st_size; + + if (S_ISDIR(statinfo.st_mode)) { + info->type = OS_FILE_TYPE_DIR; + } else if (S_ISLNK(statinfo.st_mode)) { + info->type = OS_FILE_TYPE_LINK; + } else if (S_ISREG(statinfo.st_mode)) { + info->type = OS_FILE_TYPE_FILE; + } else { + info->type = OS_FILE_TYPE_UNKNOWN; + } + + ut_free(full_path); + return(0); +#endif +} + +/***********************************************************************//** +A fault-tolerant function that tries to read the next file name in the +directory. We retry 100 times if os_file_readdir_next_file() returns -1. The +idea is to read as much good data as we can and jump over bad data. +@return 0 if ok, -1 if error even after the retries, 1 if at the end +of the directory */ +int +fil_file_readdir_next_file( +/*=======================*/ + dberr_t* err, /*!< out: this is set to DB_ERROR if an error + was encountered, otherwise not changed */ + const char* dirname,/*!< in: directory name or path */ + os_file_dir_t dir, /*!< in: directory stream */ + os_file_stat_t* info) /*!< in/out: buffer where the + info is returned */ +{ + for (ulint i = 0; i < 100; i++) { + int ret = os_file_readdir_next_file(dirname, dir, info); + + if (ret != -1) { + + return(ret); + } + + ib::error() << "os_file_readdir_next_file() returned -1 in" + " directory " << dirname + << ", crash recovery may have failed" + " for some .ibd files!"; + + *err = DB_ERROR; + } + + return(-1); +} + +/** Scan the database directories under the MySQL datadir, looking for +.ibd files and determining the space id in each of them. +@return DB_SUCCESS or error number */ + +static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback) +{ + int ret; + char* dbpath = NULL; + ulint dbpath_len = 100; + os_file_dir_t dir; + os_file_dir_t dbdir; + os_file_stat_t dbinfo; + os_file_stat_t fileinfo; + dberr_t err = DB_SUCCESS; + size_t len; + + /* The datadir of MySQL is always the default directory of mysqld */ + + dir = os_file_opendir(fil_path_to_mysql_datadir); + + if (UNIV_UNLIKELY(dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr))) { + msg("cannot open dir %s", fil_path_to_mysql_datadir); + return(DB_ERROR); + } + + dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len)); + + /* Scan all directories under the datadir. They are the database + directories of MySQL. */ + + ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir, + &dbinfo); + while (ret == 0) { + + /* General tablespaces are always at the first level of the + data home dir */ + if (dbinfo.type != OS_FILE_TYPE_FILE) { + const bool is_isl = ends_with(dbinfo.name, ".isl"); + if (is_isl || ends_with(dbinfo.name,".ibd")) { + (*callback)(nullptr, dbinfo.name, is_isl, + false, 0); + } + } + + if (dbinfo.type == OS_FILE_TYPE_FILE + || dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + goto next_datadir_item; + } + + /* We found a symlink or a directory; try opening it to see + if a symlink is a directory */ + + len = strlen(fil_path_to_mysql_datadir) + + strlen (dbinfo.name) + 2; + if (len > dbpath_len) { + dbpath_len = len; + + if (dbpath) { + ut_free(dbpath); + } + + dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len)); + } + snprintf(dbpath, dbpath_len, + "%s/%s", fil_path_to_mysql_datadir, dbinfo.name); + + if (check_if_skip_database_by_path(dbpath)) { + fprintf(stderr, "Skipping db: %s\n", dbpath); + goto next_datadir_item; + } + + dbdir = os_file_opendir(dbpath); + + if (UNIV_UNLIKELY(dbdir != IF_WIN(INVALID_HANDLE_VALUE,NULL))){ + /* We found a database directory; loop through it, + looking for possible .ibd files in it */ + + for (ret = fil_file_readdir_next_file(&err, dbpath, + dbdir, + &fileinfo); + ret == 0; + ret = fil_file_readdir_next_file(&err, dbpath, + dbdir, + &fileinfo)) { + if (fileinfo.type == OS_FILE_TYPE_DIR) { + continue; + } + + /* We found a symlink or a file */ + if (strlen(fileinfo.name) > 4) { + bool is_isl= false; + if (ends_with(fileinfo.name, ".ibd") || ((is_isl = ends_with(fileinfo.name, ".isl")))) + (*callback)(dbinfo.name, fileinfo.name, is_isl, false, 0); + } + } + + if (os_file_closedir_failed(dbdir)) { + fprintf(stderr, "InnoDB: Warning: could not" + " close database directory %s\n", + dbpath); + + err = DB_ERROR; + } + + } else { + msg("Can't open dir %s", dbpath); + err = DB_ERROR; + break; + + } + +next_datadir_item: + ret = fil_file_readdir_next_file(&err, + fil_path_to_mysql_datadir, + dir, &dbinfo); + } + + ut_free(dbpath); + + if (os_file_closedir_failed(dir)) { + fprintf(stderr, + "InnoDB: Error: could not close MariaDB datadir\n"); + return(DB_ERROR); + } + + return(err); +} + +/** Close all undo tablespaces while applying incremental delta */ +static void xb_close_undo_tablespaces() +{ + if (srv_undo_space_id_start == 0) + return; + for (uint32_t space_id= srv_undo_space_id_start; + space_id < srv_undo_space_id_start + srv_undo_tablespaces_open; + space_id++) + { + fil_space_t *space= fil_space_get(space_id); + ut_ad(space); + space->close(); + } +} + +/**************************************************************************** +Populates the tablespace memory cache by scanning for and opening data files. +@returns DB_SUCCESS or error code.*/ +static +dberr_t +xb_load_tablespaces() +{ + bool create_new_db; + dberr_t err; + ulint sum_of_new_sizes; + + ut_ad(srv_operation == SRV_OPERATION_BACKUP + || srv_operation == SRV_OPERATION_RESTORE_DELTA); + + err = srv_sys_space.check_file_spec(&create_new_db, 0); + + /* create_new_db must not be true. */ + if (err != DB_SUCCESS || create_new_db) { + msg("Could not find data files at the specified datadir"); + return(DB_ERROR); + } + + for (int i= 0; i < 10; i++) { + err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes); + if (err == DB_PAGE_CORRUPTED || err == DB_CORRUPTION) { + my_sleep(1000); + } + else + break; + } + + if (err != DB_SUCCESS) { + msg("Could not open data files.\n"); + return(err); + } + + /* Add separate undo tablespaces to fil_system */ + err = srv_undo_tablespaces_init(false, nullptr); + + if (err != DB_SUCCESS) { + return(err); + } + + /* It is important to call xb_load_single_table_tablespaces() after + srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() * + relies on srv_undo_tablespaces_open to be properly initialized */ + + msg("mariabackup: Generating a list of tablespaces"); + + err = enumerate_ibd_files(xb_load_single_table_tablespace); + if (err != DB_SUCCESS) { + return(err); + } + + if (srv_operation == SRV_OPERATION_RESTORE_DELTA) { + xb_close_undo_tablespaces(); + } + + DBUG_MARIABACKUP_EVENT("after_load_tablespaces", {}); + return(DB_SUCCESS); +} + +/** Destroy the tablespace memory cache. */ +static void xb_data_files_close() +{ + fil_space_t::close_all(); + buf_dblwr.close(); +} + +/*********************************************************************** +Allocate and initialize the entry for databases and tables filtering +hash tables. If memory allocation is not successful, terminate program. +@return pointer to the created entry. */ +static +xb_filter_entry_t * +xb_new_filter_entry( +/*================*/ + const char* name) /*!< in: name of table/database */ +{ + xb_filter_entry_t *entry; + ulint namelen = strlen(name); + + ut_a(namelen <= NAME_LEN * 2 + 1); + + entry = static_cast<xb_filter_entry_t *> + (malloc(sizeof(xb_filter_entry_t) + namelen + 1)); + memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1); + entry->name = ((char*)entry) + sizeof(xb_filter_entry_t); + strcpy(entry->name, name); + entry->has_tables = FALSE; + + return entry; +} + +/*********************************************************************** +Add entry to hash table. If hash table is NULL, allocate and initialize +new hash table */ +static +xb_filter_entry_t* +xb_add_filter( + const char* name, /*!< in: name of table/database */ + hash_table_t* hash) /*!< in/out: hash to insert into */ +{ + xb_filter_entry_t* entry = xb_new_filter_entry(name); + + if (UNIV_UNLIKELY(!hash->array)) { + hash->create(1000); + } + const ulint fold = my_crc32c(0, entry->name, strlen(entry->name)); + HASH_INSERT(xb_filter_entry_t, name_hash, hash, fold, entry); + return entry; +} + +/*********************************************************************** +Validate name of table or database. If name is invalid, program will +be finished with error code */ +static +void +xb_validate_name( +/*=============*/ + const char* name, /*!< in: name */ + size_t len) /*!< in: length of name */ +{ + const char* p; + + /* perform only basic validation. validate length and + path symbols */ + if (len > NAME_LEN) { + die("name `%s` is too long.", name); + } + p = strpbrk(name, "/\\~"); + if (p && (uint) (p - name) < NAME_LEN) { + die("name `%s` is not valid.", name); + } +} + +/*********************************************************************** +Register new filter entry which can be either database +or table name. */ +static +void +xb_register_filter_entry( +/*=====================*/ + const char* name, /*!< in: name */ + hash_table_t* databases_hash, + hash_table_t* tables_hash + ) +{ + const char* p; + size_t namelen; + xb_filter_entry_t* db_entry = NULL; + + namelen = strlen(name); + if ((p = strchr(name, '.')) != NULL) { + char dbname[NAME_LEN + 1]; + + xb_validate_name(name, p - name); + xb_validate_name(p + 1, namelen - (p - name)); + + strncpy(dbname, name, p - name); + dbname[p - name] = 0; + + if (databases_hash && databases_hash->array) { + const ulint fold = my_crc32c(0, dbname, p - name); + HASH_SEARCH(name_hash, databases_hash, + fold, + xb_filter_entry_t*, + db_entry, (void) 0, + !strcmp(db_entry->name, dbname)); + } + if (!db_entry) { + db_entry = xb_add_filter(dbname, databases_hash); + } + db_entry->has_tables = TRUE; + xb_add_filter(name, tables_hash); + } else { + xb_validate_name(name, namelen); + + xb_add_filter(name, databases_hash); + } +} + +static +void +xb_register_include_filter_entry( + const char* name +) +{ + xb_register_filter_entry(name, &databases_include_hash, + &tables_include_hash); +} + +static +void +xb_register_exclude_filter_entry( + const char* name +) +{ + xb_register_filter_entry(name, &databases_exclude_hash, + &tables_exclude_hash); +} + +void register_ignore_db_dirs_filter(const char *name) +{ + xb_add_filter(name, &databases_exclude_hash); +} + +/*********************************************************************** +Register new table for the filter. */ +static +void +xb_register_table( +/*==============*/ + const char* name) /*!< in: name of table */ +{ + if (strchr(name, '.') == NULL) { + die("`%s` is not fully qualified name.", name); + } + + xb_register_include_filter_entry(name); +} + +static +void +xb_add_regex_to_list( + const char* regex, /*!< in: regex */ + const char* error_context, /*!< in: context to error message */ + regex_list_t* list) /*! in: list to put new regex to */ +{ + char errbuf[100]; + int ret; + + regex_t compiled_regex; + ret = regcomp(&compiled_regex, regex, REG_EXTENDED); + + if (ret != 0) { + regerror(ret, &compiled_regex, errbuf, sizeof(errbuf)); + msg("mariabackup: error: %s regcomp(%s): %s", + error_context, regex, errbuf); + exit(EXIT_FAILURE); + } + + list->push_back(compiled_regex); +} + +/*********************************************************************** +Register new regex for the include filter. */ +static +void +xb_register_include_regex( +/*==============*/ + const char* regex) /*!< in: regex */ +{ + xb_add_regex_to_list(regex, "tables", ®ex_include_list); +} + +/*********************************************************************** +Register new regex for the exclude filter. */ +static +void +xb_register_exclude_regex( +/*==============*/ + const char* regex) /*!< in: regex */ +{ + xb_add_regex_to_list(regex, "tables-exclude", ®ex_exclude_list); +} + +typedef void (*insert_entry_func_t)(const char*); + +/* Scan string and load filter entries from it. +@param[in] list string representing a list +@param[in] delimiters delimiters of entries +@param[in] ins callback to add entry */ +void xb_load_list_string(char *list, const char *delimiters, + insert_entry_func_t ins) +{ + char *p; + char *saveptr; + + p= strtok_r(list, delimiters, &saveptr); + while (p) + { + + ins(p); + + p= strtok_r(NULL, delimiters, &saveptr); + } +} + +/*********************************************************************** +Scan file and load filter entries from it. */ +static +void +xb_load_list_file( +/*==============*/ + const char* filename, /*!< in: name of file */ + insert_entry_func_t ins) /*!< in: callback to add entry */ +{ + char name_buf[NAME_LEN*2+2]; + FILE* fp; + + /* read and store the filenames */ + fp = fopen(filename, "r"); + if (!fp) { + die("Can't open %s", + filename); + } + while (fgets(name_buf, sizeof(name_buf), fp) != NULL) { + char* p = strchr(name_buf, '\n'); + if (p) { + *p = '\0'; + } else { + die("`%s...` name is too long", name_buf); + } + + ins(name_buf); + } + + fclose(fp); +} + + +static +void +xb_filters_init() +{ + if (xtrabackup_databases) { + xb_load_list_string(xtrabackup_databases, " \t", + xb_register_include_filter_entry); + } + + if (xtrabackup_databases_file) { + xb_load_list_file(xtrabackup_databases_file, + xb_register_include_filter_entry); + } + + if (xtrabackup_databases_exclude) { + xb_load_list_string(xtrabackup_databases_exclude, " \t", + xb_register_exclude_filter_entry); + } + + if (xtrabackup_tables) { + xb_load_list_string(xtrabackup_tables, ",", + xb_register_include_regex); + } + + if (xtrabackup_tables_file) { + xb_load_list_file(xtrabackup_tables_file, xb_register_table); + } + + if (xtrabackup_tables_exclude) { + xb_load_list_string(xtrabackup_tables_exclude, ",", + xb_register_exclude_regex); + } +} + +static +void +xb_filter_hash_free(hash_table_t* hash) +{ + ulint i; + + /* free the hash elements */ + for (i = 0; i < hash->n_cells; i++) { + xb_filter_entry_t* table; + + table = static_cast<xb_filter_entry_t *> + (HASH_GET_FIRST(hash, i)); + + while (table) { + xb_filter_entry_t* prev_table = table; + + table = static_cast<xb_filter_entry_t *> + (HASH_GET_NEXT(name_hash, prev_table)); + const ulint fold = my_crc32c(0, prev_table->name, + strlen(prev_table->name)); + HASH_DELETE(xb_filter_entry_t, name_hash, hash, + fold, prev_table); + free(prev_table); + } + } + + hash->free(); +} + +static void xb_regex_list_free(regex_list_t* list) +{ + while (list->size() > 0) { + xb_regfree(&list->front()); + list->pop_front(); + } +} + +/************************************************************************ +Destroy table filters for partial backup. */ +static +void +xb_filters_free() +{ + xb_regex_list_free(®ex_include_list); + xb_regex_list_free(®ex_exclude_list); + + if (tables_include_hash.array) { + xb_filter_hash_free(&tables_include_hash); + } + + if (tables_exclude_hash.array) { + xb_filter_hash_free(&tables_exclude_hash); + } + + if (databases_include_hash.array) { + xb_filter_hash_free(&databases_include_hash); + } + + if (databases_exclude_hash.array) { + xb_filter_hash_free(&databases_exclude_hash); + } +} + +#ifdef RLIMIT_NOFILE +/** +Set the open files limit. Based on set_max_open_files(). +@param max_file_limit requested open files limit +@return the resulting open files limit. May be less or more than the requested +value. */ +static ulong xb_set_max_open_files(rlim_t max_file_limit) +{ + struct rlimit rlimit; + rlim_t old_cur; + + if (getrlimit(RLIMIT_NOFILE, &rlimit)) { + + goto end; + } + + old_cur = rlimit.rlim_cur; + + if (rlimit.rlim_cur == RLIM_INFINITY) { + + rlimit.rlim_cur = max_file_limit; + } + + if (rlimit.rlim_cur >= max_file_limit) { + + max_file_limit = rlimit.rlim_cur; + goto end; + } + + rlimit.rlim_cur = rlimit.rlim_max = max_file_limit; + + if (setrlimit(RLIMIT_NOFILE, &rlimit)) { + /* Use original value */ + max_file_limit = static_cast<ulong>(old_cur); + } else { + + rlimit.rlim_cur = 0; /* Safety if next call fails */ + + (void) getrlimit(RLIMIT_NOFILE, &rlimit); + + if (rlimit.rlim_cur) { + + /* If call didn't fail */ + max_file_limit = rlimit.rlim_cur; + } + } + +end: + return static_cast<ulong>(max_file_limit); +} +#else +# define xb_set_max_open_files(x) 0UL +#endif + +static void stop_backup_threads() +{ + mysql_cond_broadcast(&log_copying_stop); + + if (log_copying_running || have_io_watching_thread) + { + mysql_mutex_unlock(&recv_sys.mutex); + fputs("mariabackup: Stopping log copying thread", stderr); + fflush(stderr); + mysql_mutex_lock(&recv_sys.mutex); + while (log_copying_running || have_io_watching_thread) + { + mysql_cond_broadcast(&log_copying_stop); + mysql_mutex_unlock(&recv_sys.mutex); + putc('.', stderr); + fflush(stderr); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + mysql_mutex_lock(&recv_sys.mutex); + } + putc('\n', stderr); + } + + mysql_cond_destroy(&log_copying_stop); +} + +/** Implement the core of --backup +@return whether the operation succeeded */ +bool Backup_datasinks::backup_low() +{ + mysql_mutex_lock(&recv_sys.mutex); + ut_ad(!metadata_to_lsn); + + /* read the latest checkpoint lsn */ + { + const lsn_t lsn = recv_sys.lsn; + if (recv_sys.find_checkpoint() == DB_SUCCESS + && log_sys.is_latest()) { + metadata_to_lsn = log_sys.next_checkpoint_lsn; + msg("mariabackup: The latest check point" + " (for incremental): '" LSN_PF "'", + metadata_to_lsn); + } else { + msg("Error: recv_sys.find_checkpoint() failed."); + } + + recv_sys.lsn = lsn; + stop_backup_threads(); + } + + if (metadata_to_lsn && xtrabackup_copy_logfile()) { + mysql_mutex_unlock(&recv_sys.mutex); + ds_close(dst_log_file); + dst_log_file = NULL; + return false; + } + + mysql_mutex_unlock(&recv_sys.mutex); + + if (ds_close(dst_log_file) || !metadata_to_lsn) { + dst_log_file = NULL; + return false; + } + + dst_log_file = NULL; + + std::vector<uint32_t> failed_ids; + std::set_difference( + fail_undo_ids.begin(), fail_undo_ids.end(), + undo_trunc_ids.begin(), undo_trunc_ids.end(), + std::inserter(failed_ids, failed_ids.begin())); + + for (uint32_t id : failed_ids) { + msg("mariabackup: Failed to read undo log " + "tablespace space id %d and there is no undo " + "tablespace truncation redo record.", + id); + } + + if (failed_ids.size() > 0) { + return false; + } + + if (!xtrabackup_incremental) { + safe_strcpy(metadata_type, sizeof(metadata_type), + "full-backuped"); + metadata_from_lsn = 0; + } else { + safe_strcpy(metadata_type, sizeof(metadata_type), + "incremental"); + metadata_from_lsn = incremental_lsn; + } + metadata_last_lsn = recv_sys.lsn; + + if (!xtrabackup_stream_metadata(m_meta)) { + msg("Error: failed to stream metadata."); + return false; + } + if (xtrabackup_extra_lsndir) { + char filename[FN_REFLEN]; + + sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, + XTRABACKUP_METADATA_FILENAME); + if (!xtrabackup_write_metadata(filename)) { + msg("Error: failed to write metadata " + "to '%s'.", filename); + return false; + } + sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, + XTRABACKUP_INFO); + if (!write_xtrabackup_info(m_data, + mysql_connection, filename, false, false)) { + msg("Error: failed to write info " + "to '%s'.", filename); + return false; + } + } + + return true; +} + +/** Implement --backup +@return whether the operation succeeded */ +static bool xtrabackup_backup_func() +{ + MY_STAT stat_info; + uint i; + uint count; + pthread_mutex_t count_mutex; + CorruptedPages corrupted_pages; + data_thread_ctxt_t *data_threads; + Backup_datasinks backup_datasinks; + pthread_cond_init(&scanned_lsn_cond, NULL); + +#ifdef USE_POSIX_FADVISE + msg("uses posix_fadvise()."); +#endif + + /* cd to datadir */ + + if (my_setwd(mysql_real_data_home,MYF(MY_WME))) + { + msg("my_setwd() failed , %s", mysql_real_data_home); + return(false); + } + msg("cd to %s", mysql_real_data_home); + xb_plugin_backup_init(mysql_connection); + msg("open files limit requested %lu, set to %lu", + xb_open_files_limit, + xb_set_max_open_files(xb_open_files_limit)); + + mysql_data_home= mysql_data_home_buff; + mysql_data_home[0]=FN_CURLIB; // all paths are relative from here + mysql_data_home[1]=0; + + srv_n_purge_threads = 1; + srv_read_only_mode = TRUE; + + srv_operation = SRV_OPERATION_BACKUP; + log_file_op = backup_file_op; + undo_space_trunc = backup_undo_trunc; + first_page_init = backup_first_page_op; + metadata_to_lsn = 0; + + /* initialize components */ + if(innodb_init_param()) { +fail: + if (log_copying_running) { + mysql_mutex_lock(&recv_sys.mutex); + metadata_to_lsn = 1; + stop_backup_threads(); + mysql_mutex_unlock(&recv_sys.mutex); + } + + log_file_op = NULL; + undo_space_trunc = NULL; + first_page_init = NULL; + if (dst_log_file) { + ds_close(dst_log_file); + dst_log_file = NULL; + } + if (fil_system.is_initialised()) { + innodb_shutdown(); + } + return(false); + } + + if (srv_buf_pool_size >= 1000 * 1024 * 1024) { + /* Here we still have srv_pool_size counted + in kilobytes (in 4.0 this was in bytes) + srv_boot() converts the value to + pages; if buffer pool is less than 1000 MB, + assume fewer threads. */ + srv_max_n_threads = 50000; + + } else if (srv_buf_pool_size >= 8 * 1024 * 1024) { + + srv_max_n_threads = 10000; + } else { + srv_max_n_threads = 1000; /* saves several MB of memory, + especially in 64-bit + computers */ + } + srv_thread_pool_init(); + /* Reset the system variables in the recovery module. */ + trx_pool_init(); + recv_sys.create(); + + xb_filters_init(); + + xb_fil_io_init(); + + if (os_aio_init()) { + msg("Error: cannot initialize AIO subsystem"); + goto fail; + } + + if (!log_sys.create()) { + goto fail; + } + /* get current checkpoint_lsn */ + { + mysql_mutex_lock(&recv_sys.mutex); + + dberr_t err = recv_sys.find_checkpoint(); + + if (err != DB_SUCCESS) { + msg("Error: cannot read redo log header"); + } else if (!log_sys.is_latest()) { + msg("Error: cannot process redo log before " + "MariaDB 10.8"); + err = DB_ERROR; + } else { + recv_needed_recovery = true; + } + mysql_mutex_unlock(&recv_sys.mutex); + + if (err != DB_SUCCESS) { + goto fail; + } + } + + /* create extra LSN dir if it does not exist. */ + if (xtrabackup_extra_lsndir + &&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0)) + && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) { + msg("Error: cannot mkdir %d: %s\n", + my_errno, xtrabackup_extra_lsndir); + goto fail; + } + + /* create target dir if not exist */ + if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0)) + && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){ + msg("Error: cannot mkdir %d: %s\n", + my_errno, xtrabackup_target_dir); + goto fail; + } + + backup_datasinks.init(); + + if (!select_history()) { + goto fail; + } + + /* open the log file */ + memset(&stat_info, 0, sizeof(MY_STAT)); + dst_log_file = ds_open(backup_datasinks.m_redo, LOG_FILE_NAME, &stat_info); + if (dst_log_file == NULL) { + msg("Error: failed to open the target stream for '%s'.", + LOG_FILE_NAME); + goto fail; + } + + /* label it */ + recv_sys.file_checkpoint = log_sys.next_checkpoint_lsn; + log_hdr_init(); + /* Write log header*/ + if (ds_write(dst_log_file, log_hdr_buf, 12288)) { + msg("error: write to logfile failed"); + goto fail; + } + log_copying_running = true; + + mysql_cond_init(0, &log_copying_stop, nullptr); + + /* start io throttle */ + if (xtrabackup_throttle) { + io_ticket = xtrabackup_throttle; + have_io_watching_thread = true; + mysql_cond_init(0, &wait_throttle, nullptr); + std::thread(io_watching_thread).detach(); + } + + /* Populate fil_system with tablespaces to copy */ + if (dberr_t err = xb_load_tablespaces()) { + msg("merror: xb_load_tablespaces() failed with" + " error %s.", ut_strerr(err)); + log_copying_running = false; + goto fail; + } + + /* copy log file by current position */ + + mysql_mutex_lock(&recv_sys.mutex); + recv_sys.lsn = log_sys.next_checkpoint_lsn; + + const bool log_copy_failed = xtrabackup_copy_logfile(); + + mysql_mutex_unlock(&recv_sys.mutex); + + if (log_copy_failed) { + log_copying_running = false; + goto fail; + } + + DBUG_MARIABACKUP_EVENT("before_innodb_log_copy_thread_started", {}); + + std::thread(log_copying_thread).detach(); + + /* FLUSH CHANGED_PAGE_BITMAPS call */ + if (!flush_changed_page_bitmaps()) { + goto fail; + } + + ut_a(xtrabackup_parallel > 0); + + if (xtrabackup_parallel > 1) { + msg("mariabackup: Starting %u threads for parallel data " + "files transfer", xtrabackup_parallel); + } + + if (opt_lock_ddl_per_table) { + mdl_lock_all(); + + DBUG_EXECUTE_IF("check_mdl_lock_works", + dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int", + "Waiting for table metadata lock", 0, 0);); + } + + datafiles_iter_t it; + + /* Create data copying threads */ + data_threads = (data_thread_ctxt_t *) + malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); + count = xtrabackup_parallel; + pthread_mutex_init(&count_mutex, NULL); + + for (i = 0; i < (uint) xtrabackup_parallel; i++) { + data_threads[i].it = ⁢ + data_threads[i].num = i+1; + data_threads[i].count = &count; + data_threads[i].count_mutex = &count_mutex; + data_threads[i].corrupted_pages = &corrupted_pages; + data_threads[i].datasinks= &backup_datasinks; + std::thread(data_copy_thread_func, data_threads + i).detach(); + } + + /* Wait for threads to exit */ + while (1) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + pthread_mutex_lock(&count_mutex); + bool stop = count == 0; + pthread_mutex_unlock(&count_mutex); + if (stop) { + break; + } + } + + pthread_mutex_destroy(&count_mutex); + free(data_threads); + + DBUG_ASSERT(backup_datasinks.m_data); + DBUG_ASSERT(backup_datasinks.m_meta); + bool ok = backup_start(backup_datasinks.m_data, + backup_datasinks.m_meta, corrupted_pages); + + if (ok) { + ok = backup_datasinks.backup_low(); + + backup_release(); + + DBUG_EXECUTE_IF("check_mdl_lock_works", + pthread_join(dbug_alter_thread, nullptr);); + + if (ok) { + backup_finish(backup_datasinks.m_data); + } + } + + if (opt_log_innodb_page_corruption) + ok = corrupted_pages.print_to_file(backup_datasinks.m_data, + MB_CORRUPTED_PAGES_FILE); + + if (!ok) { + goto fail; + } + + if (changed_page_bitmap) { + xb_page_bitmap_deinit(changed_page_bitmap); + } + backup_datasinks.destroy(); + + msg("Redo log (from LSN " LSN_PF " to " LSN_PF ") was copied.", + log_sys.next_checkpoint_lsn, recv_sys.lsn); + xb_filters_free(); + + xb_data_files_close(); + + /* Make sure that the latest checkpoint was included */ + if (metadata_to_lsn > recv_sys.lsn) { + msg("Error: failed to copy enough redo log (" + "LSN=" LSN_PF "; checkpoint LSN=" LSN_PF ").", + recv_sys.lsn, metadata_to_lsn); + goto fail; + } + + innodb_shutdown(); + log_file_op = NULL; + undo_space_trunc = NULL; + first_page_init = NULL; + pthread_cond_destroy(&scanned_lsn_cond); + if (!corrupted_pages.empty()) { + ut_ad(opt_log_innodb_page_corruption); + msg("Error: corrupted innodb pages are found and logged to " + MB_CORRUPTED_PAGES_FILE " file"); + } + return(true); +} + + +/** +This function handles DDL changes at the end of backup, under protection of +FTWRL. This ensures consistent backup in presence of DDL. + +- New tables, that were created during backup, are now copied into backup. + Also, tablespaces with optimized (no redo loggin DDL) are re-copied into + backup. This tablespaces will get the extension ".new" in the backup + +- Tables that were renamed during backup, are marked as renamed + For these, file <old_name>.ren will be created. + The content of the file is the new tablespace name. + +- Tables that were deleted during backup, are marked as deleted + For these , an empty file <name>.del will be created + + It is the responsibility of the prepare phase to deal with .new, .ren, and .del + files. +*/ +void CorruptedPages::backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta) +{ + std::set<std::string> dropped_tables; + std::map<std::string, std::string> renamed_tables; + space_id_to_name_t new_tables; + + /* Disable further DDL on backed up tables (only needed for --no-lock).*/ + mysql_mutex_lock(&recv_sys.mutex); + log_file_op = backup_file_op_fail; + mysql_mutex_unlock(&recv_sys.mutex); + + DBUG_MARIABACKUP_EVENT("backup_fix_ddl", {}); + + for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin(); + iter != ddl_tracker.tables_in_backup.end(); + iter++) { + + const std::string name = iter->second; + uint32_t id = iter->first; + + if (ddl_tracker.drops.find(id) != ddl_tracker.drops.end()) { + dropped_tables.insert(name); + drop_space(id); + continue; + } + + if (ddl_tracker.id_to_name.find(id) == ddl_tracker.id_to_name.end()) { + continue; + } + + /* tablespace was affected by DDL. */ + const std::string new_name = ddl_tracker.id_to_name[id]; + if (new_name != name) { + renamed_tables[name] = new_name; + if (opt_log_innodb_page_corruption) + rename_space(id, new_name); + } + } + + /* Find tables that were created during backup (and not removed).*/ + for(space_id_to_name_t::iterator iter = ddl_tracker.id_to_name.begin(); + iter != ddl_tracker.id_to_name.end(); + iter++) { + + uint32_t id = iter->first; + std::string name = iter->second; + + if (ddl_tracker.tables_in_backup.find(id) != ddl_tracker.tables_in_backup.end()) { + /* already processed above */ + continue; + } + + if (ddl_tracker.drops.find(id) == ddl_tracker.drops.end() + && ddl_tracker.deferred_tables.find(id) + == ddl_tracker.deferred_tables.end()) { + dropped_tables.erase(name); + new_tables[id] = name; + if (opt_log_innodb_page_corruption) + drop_space(id); + } + } + + // Mark tablespaces for rename + for (std::map<std::string, std::string>::iterator iter = renamed_tables.begin(); + iter != renamed_tables.end(); ++iter) { + const std::string old_name = iter->first; + std::string new_name = iter->second; + DBUG_ASSERT(ds_data); + ds_data->backup_file_printf((old_name + ".ren").c_str(), "%s", new_name.c_str()); + } + + // Mark tablespaces for drop + for (std::set<std::string>::iterator iter = dropped_tables.begin(); + iter != dropped_tables.end(); + iter++) { + const std::string name(*iter); + ds_data->backup_file_printf((name + ".del").c_str(), "%s", ""); + } + + // Load and copy new tables. + // Close all datanodes first, reload only new tables. + std::vector<fil_node_t *> all_nodes; + datafiles_iter_t it; + while (fil_node_t *node = datafiles_iter_next(&it)) { + all_nodes.push_back(node); + } + for (size_t i = 0; i < all_nodes.size(); i++) { + fil_node_t *n = all_nodes[i]; + if (n->space->id == 0) + continue; + if (n->is_open()) { + mysql_mutex_lock(&fil_system.mutex); + n->close(); + mysql_mutex_unlock(&fil_system.mutex); + } + fil_space_free(n->space->id, false); + } + + DBUG_EXECUTE_IF("check_mdl_lock_works", DBUG_ASSERT(new_tables.size() == 0);); + + srv_operation = SRV_OPERATION_BACKUP_NO_DEFER; + + /* Mariabackup detected the FILE_MODIFY or FILE_RENAME + for the deferred tablespace. So it needs to read the + tablespace again if innodb doesn't have page0 initialization + redo log for it */ + for (space_id_to_name_t::iterator iter = + ddl_tracker.deferred_tables.begin(); + iter != ddl_tracker.deferred_tables.end(); + iter++) { + if (check_if_skip_table(iter->second.c_str())) { + continue; + } + + if (first_page_init_ids.find(iter->first) + != first_page_init_ids.end()) { + new_tables[iter->first] = iter->second.c_str(); + continue; + } + + xb_load_single_table_tablespace(iter->second, false); + } + + /* Mariabackup doesn't detect any FILE_OP for the deferred + tablespace. There is a possiblity that page0 could've + been corrupted persistently in the disk */ + for (auto space_name: defer_space_names) { + if (!check_if_skip_table(space_name.c_str())) { + xb_load_single_table_tablespace( + space_name, false); + } + } + + srv_operation = SRV_OPERATION_BACKUP; + + for (const auto &t : new_tables) { + if (!check_if_skip_table(t.second.c_str())) { + xb_load_single_table_tablespace(t.second, false, + t.first); + } + } + + datafiles_iter_t it2; + + while (fil_node_t *node = datafiles_iter_next(&it2)) { + if (!fil_is_user_tablespace_id(node->space->id)) + continue; + std::string dest_name= filename_to_spacename( + node->name, strlen(node->name)); + dest_name.append(".new"); + + xtrabackup_copy_datafile(ds_data, ds_meta, + node, 0, dest_name.c_str(), + wf_write_through, *this); + } +} + +/* ================= prepare ================= */ + +/*********************************************************************** +Generates path to the meta file path from a given path to an incremental .delta +by replacing trailing ".delta" with ".meta", or returns error if 'delta_path' +does not end with the ".delta" character sequence. +@return TRUE on success, FALSE on error. */ +static +ibool +get_meta_path( + const char *delta_path, /* in: path to a .delta file */ + char *meta_path) /* out: path to the corresponding .meta + file */ +{ + size_t len = strlen(delta_path); + + if (len <= 6 || strcmp(delta_path + len - 6, ".delta")) { + return FALSE; + } + memcpy(meta_path, delta_path, len - 6); + strcpy(meta_path + len - 6, XB_DELTA_INFO_SUFFIX); + + return TRUE; +} + +/****************************************************************//** +Create a new tablespace on disk and return the handle to its opened +file. Code adopted from fil_create_new_single_table_tablespace with +the main difference that only disk file is created without updating +the InnoDB in-memory dictionary data structures. + +@return true on success, false on error. */ +static +bool +xb_space_create_file( +/*==================*/ + const char* path, /*!<in: path to tablespace */ + uint32_t space_id, /*!<in: space id */ + uint32_t flags, /*!<in: tablespace flags */ + pfs_os_file_t* file) /*!<out: file handle */ +{ + bool ret; + + *file = os_file_create_simple_no_error_handling( + 0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, false, &ret); + if (!ret) { + msg("Can't create file %s", path); + return ret; + } + + ret = os_file_set_size(path, *file, + FIL_IBD_FILE_INITIAL_SIZE + << srv_page_size_shift); + if (!ret) { + msg("mariabackup: cannot set size for file %s", path); + os_file_close(*file); + os_file_delete(0, path); + return ret; + } + + return TRUE; +} + +static fil_space_t* fil_space_get_by_name(const char* name) +{ + mysql_mutex_assert_owner(&fil_system.mutex); + for (fil_space_t &space : fil_system.space_list) + if (space.chain.start) + if (const char *str= strstr(space.chain.start->name, name)) + if (!strcmp(str + strlen(name), ".ibd") && + (str == space.chain.start->name || + IF_WIN(str[-1] == '\\' ||,) str[-1] == '/')) + return &space; + return nullptr; +} + +/*********************************************************************** +Searches for matching tablespace file for given .delta file and space_id +in given directory. When matching tablespace found, renames it to match the +name of .delta file. If there was a tablespace with matching name and +mismatching ID, renames it to xtrabackup_tmp_#ID.ibd. If there was no +matching file, creates a new tablespace. +@return file handle of matched or created file */ +static +pfs_os_file_t +xb_delta_open_matching_space( + const char* dbname, /* in: path to destination database dir */ + const char* name, /* in: name of delta file (without .delta) */ + const xb_delta_info_t& info, + char* real_name, /* out: full path of destination file */ + size_t real_name_len, /* out: buffer size for real_name */ + bool* success) /* out: indicates error. true = success */ +{ + char dest_dir[FN_REFLEN]; + char dest_space_name[FN_REFLEN]; + fil_space_t* fil_space; + pfs_os_file_t file; + xb_filter_entry_t* table; + + ut_a(dbname != NULL || + !fil_is_user_tablespace_id(info.space_id) || + info.space_id == UINT32_MAX); + + *success = false; + + if (dbname) { + snprintf(dest_dir, FN_REFLEN, "%s/%s", + xtrabackup_target_dir, dbname); + snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name); + } else { + snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir); + snprintf(dest_space_name, FN_REFLEN, "%s", name); + } + + snprintf(real_name, real_name_len, + "%s/%s", + xtrabackup_target_dir, dest_space_name); + /* Truncate ".ibd" */ + dest_space_name[strlen(dest_space_name) - 4] = '\0'; + + /* Create the database directory if it doesn't exist yet */ + if (!os_file_create_directory(dest_dir, FALSE)) { + msg("mariabackup: error: cannot create dir %s", dest_dir); + return file; + } + + if (!info.space_id && fil_system.sys_space) { + fil_node_t *node + = UT_LIST_GET_FIRST(fil_system.sys_space->chain); + for (; node; node = UT_LIST_GET_NEXT(chain, node)) { + if (!strcmp(node->name, real_name)) { + break; + } + } + if (node && node->handle != OS_FILE_CLOSED) { + *success = true; + return node->handle; + } + msg("mariabackup: Cannot find file %s\n", real_name); + return OS_FILE_CLOSED; + } + + mysql_mutex_lock(&recv_sys.mutex); + if (!fil_is_user_tablespace_id(info.space_id)) { +found: + /* open the file and return its handle */ + + file = os_file_create_simple_no_error_handling( + 0, real_name, + OS_FILE_OPEN, OS_FILE_READ_WRITE, false, success); + + if (!*success) { + msg("mariabackup: Cannot open file %s\n", real_name); + } +exit: + mysql_mutex_unlock(&recv_sys.mutex); + return file; + } + + const size_t len = strlen(dest_space_name); + /* remember space name for further reference */ + table = static_cast<xb_filter_entry_t *> + (malloc(sizeof(xb_filter_entry_t) + + len + 1)); + + table->name = ((char*)table) + sizeof(xb_filter_entry_t); + memcpy(table->name, dest_space_name, len + 1); + const ulint fold = my_crc32c(0, dest_space_name, len); + HASH_INSERT(xb_filter_entry_t, name_hash, &inc_dir_tables_hash, + fold, table); + + mysql_mutex_lock(&fil_system.mutex); + fil_space = fil_space_get_by_name(dest_space_name); + mysql_mutex_unlock(&fil_system.mutex); + + if (fil_space != NULL) { + if (fil_space->id == info.space_id + || info.space_id == UINT32_MAX) { + /* we found matching space */ + goto found; + } else { + + char tmpname[FN_REFLEN]; + + snprintf(tmpname, FN_REFLEN, "%s/xtrabackup_tmp_#%u", + dbname, fil_space->id); + + msg("mariabackup: Renaming %s to %s.ibd", + fil_space->chain.start->name, tmpname); + + if (fil_space->rename(tmpname, false) != DB_SUCCESS) { + msg("mariabackup: Cannot rename %s to %s", + fil_space->chain.start->name, tmpname); + goto exit; + } + } + } + + if (info.space_id == UINT32_MAX) + { + die("Can't handle DDL operation on tablespace " + "%s\n", dest_space_name); + } + mysql_mutex_lock(&fil_system.mutex); + fil_space = fil_space_get_by_id(info.space_id); + mysql_mutex_unlock(&fil_system.mutex); + if (fil_space != NULL) { + char tmpname[FN_REFLEN]; + + snprintf(tmpname, sizeof tmpname, "%s.ibd", dest_space_name); + + msg("mariabackup: Renaming %s to %s", + fil_space->chain.start->name, tmpname); + + if (fil_space->rename(tmpname, false) != DB_SUCCESS) { + msg("mariabackup: Cannot rename %s to %s", + fil_space->chain.start->name, tmpname); + goto exit; + } + + goto found; + } + + /* No matching space found. create the new one. */ + const uint32_t flags = info.zip_size + ? get_bit_shift(info.page_size + >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1)) + << FSP_FLAGS_POS_ZIP_SSIZE + | FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS + | (srv_page_size == UNIV_PAGE_SIZE_ORIG + ? 0 + : get_bit_shift(srv_page_size + >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1)) + << FSP_FLAGS_POS_PAGE_SSIZE) + : FSP_FLAGS_PAGE_SSIZE(); + ut_ad(fil_space_t::zip_size(flags) == info.zip_size); + ut_ad(fil_space_t::physical_size(flags) == info.page_size); + + mysql_mutex_lock(&fil_system.mutex); + fil_space_t* space = fil_space_t::create(info.space_id, flags, + FIL_TYPE_TABLESPACE, 0, + FIL_ENCRYPTION_DEFAULT, true); + mysql_mutex_unlock(&fil_system.mutex); + if (space) { + *success = xb_space_create_file(real_name, info.space_id, + flags, &file); + } else { + msg("Can't create tablespace %s\n", dest_space_name); + } + + goto exit; +} + +/************************************************************************ +Applies a given .delta file to the corresponding data file. +@return TRUE on success */ +static +ibool +xtrabackup_apply_delta( + const char* dirname, /* in: dir name of incremental */ + const char* dbname, /* in: database name (ibdata: NULL) */ + const char* filename, /* in: file name (not a path), + including the .delta extension */ + void* /*data*/) +{ + pfs_os_file_t src_file; + pfs_os_file_t dst_file; + char src_path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + char meta_path[FN_REFLEN]; + char space_name[FN_REFLEN]; + bool success; + + ibool last_buffer = FALSE; + ulint page_in_buffer; + ulint incremental_buffers = 0; + + xb_delta_info_t info(srv_page_size, 0, SRV_TMP_SPACE_ID); + ulint page_size; + ulint page_size_shift; + byte* incremental_buffer = NULL; + + size_t offset; + + ut_a(xtrabackup_incremental); + + if (dbname) { + snprintf(src_path, sizeof(src_path), "%s/%s/%s", + dirname, dbname, filename); + snprintf(dst_path, sizeof(dst_path), "%s/%s/%s", + xtrabackup_real_target_dir, dbname, filename); + } else { + snprintf(src_path, sizeof(src_path), "%s/%s", + dirname, filename); + snprintf(dst_path, sizeof(dst_path), "%s/%s", + xtrabackup_real_target_dir, filename); + } + dst_path[strlen(dst_path) - 6] = '\0'; + + strncpy(space_name, filename, FN_REFLEN - 1); + space_name[FN_REFLEN - 1] = '\0'; + space_name[strlen(space_name) - 6] = 0; + + if (!get_meta_path(src_path, meta_path)) { + goto error; + } + + if (!xb_read_delta_metadata(meta_path, &info)) { + goto error; + } + + page_size = info.page_size; + page_size_shift = get_bit_shift(page_size); + msg("page size for %s is %zu bytes", + src_path, page_size); + if (page_size_shift < 10 || + page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) { + msg("error: invalid value of page_size " + "(%zu bytes) read from %s", page_size, meta_path); + goto error; + } + + src_file = os_file_create_simple_no_error_handling( + 0, src_path, + OS_FILE_OPEN, OS_FILE_READ_WRITE, false, &success); + if (!success) { + os_file_get_last_error(TRUE); + msg("error: can't open %s", src_path); + goto error; + } + + posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL); + + dst_file = xb_delta_open_matching_space( + dbname, space_name, info, + dst_path, sizeof(dst_path), &success); + if (!success) { + msg("error: can't open %s", dst_path); + goto error; + } + + posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED); + + /* allocate buffer for incremental backup (4096 pages) */ + incremental_buffer = static_cast<byte *> + (aligned_malloc(page_size / 4 * page_size, page_size)); + + msg("Applying %s to %s...", src_path, dst_path); + + while (!last_buffer) { + ulint cluster_header; + + /* read to buffer */ + /* first block of block cluster */ + offset = ((incremental_buffers * (page_size / 4)) + << page_size_shift); + if (os_file_read(IORequestRead, src_file, + incremental_buffer, offset, page_size, + nullptr) + != DB_SUCCESS) { + goto error; + } + + cluster_header = mach_read_from_4(incremental_buffer); + switch(cluster_header) { + case 0x78747261UL: /*"xtra"*/ + break; + case 0x58545241UL: /*"XTRA"*/ + last_buffer = TRUE; + break; + default: + msg("error: %s seems not " + ".delta file.", src_path); + goto error; + } + + /* FIXME: If the .delta modifies FSP_SIZE on page 0, + extend the file to that size. */ + + for (page_in_buffer = 1; page_in_buffer < page_size / 4; + page_in_buffer++) { + if (mach_read_from_4(incremental_buffer + page_in_buffer * 4) + == 0xFFFFFFFFUL) + break; + } + + ut_a(last_buffer || page_in_buffer == page_size / 4); + + /* read whole of the cluster */ + if (os_file_read(IORequestRead, src_file, + incremental_buffer, + offset, page_in_buffer * page_size, nullptr) + != DB_SUCCESS) { + goto error; + } + + posix_fadvise(src_file, offset, page_in_buffer * page_size, + POSIX_FADV_DONTNEED); + + for (page_in_buffer = 1; page_in_buffer < page_size / 4; + page_in_buffer++) { + ulint offset_on_page; + + offset_on_page = mach_read_from_4(incremental_buffer + page_in_buffer * 4); + + if (offset_on_page == 0xFFFFFFFFUL) + break; + + uchar *buf = incremental_buffer + page_in_buffer * page_size; + const os_offset_t off = os_offset_t(offset_on_page)*page_size; + + if (off == 0) { + /* Read tablespace size from page 0, + and extend the file to specified size.*/ + os_offset_t n_pages = mach_read_from_4( + buf + FSP_HEADER_OFFSET + FSP_SIZE); + if (mach_read_from_4(buf + + FIL_PAGE_SPACE_ID)) { + if (!os_file_set_size( + dst_path, dst_file, + n_pages * page_size)) + goto error; + } else if (fil_space_t* space + = fil_system.sys_space) { + /* The system tablespace can + consist of multiple files. The + first one has full tablespace + size in page 0, but only the last + file should be extended. */ + fil_node_t* n = UT_LIST_GET_FIRST( + space->chain); + bool fail = !strcmp(n->name, dst_path) + && !fil_space_extend( + space, uint32_t(n_pages)); + if (fail) goto error; + } + } + + if (os_file_write(IORequestWrite, + dst_path, dst_file, buf, off, + page_size) != DB_SUCCESS) { + goto error; + } + } + + /* Free file system buffer cache after the batch was written. */ +#ifdef __linux__ + os_file_flush_func(dst_file); +#endif + posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED); + + + incremental_buffers++; + } + + aligned_free(incremental_buffer); + if (src_file != OS_FILE_CLOSED) { + os_file_close(src_file); + os_file_delete(0,src_path); + } + if (dst_file != OS_FILE_CLOSED && info.space_id) + os_file_close(dst_file); + return TRUE; + +error: + aligned_free(incremental_buffer); + if (src_file != OS_FILE_CLOSED) + os_file_close(src_file); + if (dst_file != OS_FILE_CLOSED && info.space_id) + os_file_close(dst_file); + msg("Error: xtrabackup_apply_delta(): " + "failed to apply %s to %s.\n", src_path, dst_path); + return FALSE; +} + + +std::string change_extension(std::string filename, std::string new_ext) { + DBUG_ASSERT(new_ext.size() == 3); + std::string new_name(filename); + new_name.resize(new_name.size() - new_ext.size()); + new_name.append(new_ext); + return new_name; +} + + +static void rename_file(const char *from,const char *to) { + msg("Renaming %s to %s\n", from, to); + if (my_rename(from, to, MY_WME)) { + die("Can't rename %s to %s errno %d", from, to, errno); + } +} + +static void rename_file(const std::string& from, const std::string &to) { + rename_file(from.c_str(), to.c_str()); +} +/************************************************************************ +Callback to handle datadir entry. Function of this type will be called +for each entry which matches the mask by xb_process_datadir. +@return should return TRUE on success */ +typedef ibool (*handle_datadir_entry_func_t)( +/*=========================================*/ + const char* data_home_dir, /*!<in: path to datadir */ + const char* db_name, /*!<in: database name */ + const char* file_name, /*!<in: file name with suffix */ + void* arg); /*!<in: caller-provided data */ + +/** Rename, and replace destination file, if exists */ +static void rename_force(const char *from, const char *to) { + if (access(to, R_OK) == 0) { + msg("Removing %s", to); + if (my_delete(to, MYF(MY_WME))) { + msg("Can't remove %s, errno %d", to, errno); + exit(EXIT_FAILURE); + } + } + rename_file(from,to); +} + + +/** During prepare phase, rename ".new" files, that were created in +backup_fix_ddl() and backup_optimized_ddl_op(), to ".ibd". In the case of +incremental backup, i.e. of arg argument is set, move ".new" files to +destination directory and rename them to ".ibd", remove existing ".ibd.delta" +and ".idb.meta" files in incremental directory to avoid applying delta to +".ibd" file. + +@param[in] data_home_dir path to datadir +@param[in] db_name database name +@param[in] file_name file name with suffix +@param[in] arg destination path, used in incremental backup to notify, that +*.new file must be moved to destibation directory + +@return true */ +static ibool prepare_handle_new_files(const char *data_home_dir, + const char *db_name, + const char *file_name, void *arg) +{ + const char *dest_dir = static_cast<const char *>(arg); + std::string src_path = std::string(data_home_dir) + '/' + std::string(db_name) + '/'; + /* Copy "*.new" files from incremental to base dir for incremental backup */ + std::string dest_path= + dest_dir ? std::string(dest_dir) + '/' + std::string(db_name) + + '/' : src_path; + + /* + A CREATE DATABASE could have happened during the base mariabackup run. + In case if the current table file (e.g. `t1.new`) is from such + a new database, the database directory may not exist yet in + the base backup directory. Let's make sure to check if the directory + exists (and create if needed). + */ + if (!directory_exists(dest_path.c_str(), true/*create if not exists*/)) + return FALSE; + src_path+= file_name; + dest_path+= file_name; + + size_t index = dest_path.find(".new"); + DBUG_ASSERT(index != std::string::npos); + dest_path.replace(index, strlen(".ibd"), ".ibd"); + rename_force(src_path.c_str(),dest_path.c_str()); + + if (dest_dir) { + /* remove delta and meta files to avoid delta applying for new file */ + index = src_path.find(".new"); + DBUG_ASSERT(index != std::string::npos); + src_path.replace(index, std::string::npos, ".ibd.delta"); + if (access(src_path.c_str(), R_OK) == 0) { + msg("Removing %s", src_path.c_str()); + if (my_delete(src_path.c_str(), MYF(MY_WME))) + die("Can't remove %s, errno %d", src_path.c_str(), errno); + } + src_path.replace(index, std::string::npos, ".ibd.meta"); + if (access(src_path.c_str(), R_OK) == 0) { + msg("Removing %s", src_path.c_str()); + if (my_delete(src_path.c_str(), MYF(MY_WME))) + die("Can't remove %s, errno %d", src_path.c_str(), errno); + } + + /* add table name to the container to avoid it's deletion at the end of + prepare */ + std::string table_name = std::string(db_name) + '/' + + std::string(file_name, file_name + strlen(file_name) - strlen(".new")); + xb_filter_entry_t *table = static_cast<xb_filter_entry_t *> + (malloc(sizeof(xb_filter_entry_t) + table_name.size() + 1)); + table->name = ((char*)table) + sizeof(xb_filter_entry_t); + strcpy(table->name, table_name.c_str()); + const ulint fold = my_crc32c(0, table->name, + table_name.size()); + HASH_INSERT(xb_filter_entry_t, name_hash, &inc_dir_tables_hash, + fold, table); + } + + return TRUE; +} + +/************************************************************************ +Callback to handle datadir entry. Deletes entry if it has no matching +fil_space in fil_system directory. +@return FALSE if delete attempt was unsuccessful */ +static +ibool +rm_if_not_found( + const char* data_home_dir, /*!<in: path to datadir */ + const char* db_name, /*!<in: database name */ + const char* file_name, /*!<in: file name with suffix */ + void* arg __attribute__((unused))) +{ + char name[FN_REFLEN]; + xb_filter_entry_t* table; + + snprintf(name, FN_REFLEN, "%s/%s", db_name, file_name); + /* Truncate ".ibd" */ + const size_t len = strlen(name) - 4; + name[len] = '\0'; + const ulint fold = my_crc32c(0, name, len); + + HASH_SEARCH(name_hash, &inc_dir_tables_hash, fold, + xb_filter_entry_t*, + table, (void) 0, + !strcmp(table->name, name)); + + if (!table) { + snprintf(name, FN_REFLEN, "%s/%s/%s", data_home_dir, + db_name, file_name); + return os_file_delete(0, name); + } + + return(TRUE); +} + +/** Function enumerates files in datadir (provided by path) which are matched +by provided suffix. For each entry callback is called. + +@param[in] path datadir path +@param[in] suffix suffix to match against +@param[in] func callback +@param[in] func_arg arguments for the above callback + +@return FALSE if callback for some entry returned FALSE */ +static ibool xb_process_datadir(const char *path, const char *suffix, + handle_datadir_entry_func_t func, + void *func_arg = NULL) +{ + ulint ret; + char dbpath[OS_FILE_MAX_PATH+2]; + os_file_dir_t dir; + os_file_dir_t dbdir; + os_file_stat_t dbinfo; + os_file_stat_t fileinfo; + ulint suffix_len; + dberr_t err = DB_SUCCESS; + static char current_dir[2]; + + current_dir[0] = FN_CURLIB; + current_dir[1] = 0; + srv_data_home = current_dir; + + suffix_len = strlen(suffix); + + /* datafile */ + dbdir = os_file_opendir(path); + if (UNIV_UNLIKELY(dbdir != IF_WIN(INVALID_HANDLE_VALUE, nullptr))) { + ret = fil_file_readdir_next_file(&err, path, dbdir, &fileinfo); + while (ret == 0) { + if (fileinfo.type == OS_FILE_TYPE_DIR) { + goto next_file_item_1; + } + + if (strlen(fileinfo.name) > suffix_len + && 0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - suffix_len, + suffix)) { + if (!func( + path, NULL, + fileinfo.name, func_arg)) + { + os_file_closedir(dbdir); + return(FALSE); + } + } +next_file_item_1: + ret = fil_file_readdir_next_file(&err, + path, dbdir, + &fileinfo); + } + + os_file_closedir(dbdir); + } else { + msg("Can't open dir %s", path); + } + + /* single table tablespaces */ + dir = os_file_opendir(path); + + if (UNIV_UNLIKELY(dbdir == IF_WIN(INVALID_HANDLE_VALUE, nullptr))) { + msg("Can't open dir %s", path); + return TRUE; + } + + ret = fil_file_readdir_next_file(&err, path, dir, &dbinfo); + while (ret == 0) { + if (dbinfo.type == OS_FILE_TYPE_FILE + || dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + goto next_datadir_item; + } + + snprintf(dbpath, sizeof(dbpath), "%.*s/%.*s", + OS_FILE_MAX_PATH/2-1, + path, + OS_FILE_MAX_PATH/2-1, + dbinfo.name); + + dbdir = os_file_opendir(dbpath); + + if (dbdir != IF_WIN(INVALID_HANDLE_VALUE, nullptr)) { + ret = fil_file_readdir_next_file(&err, dbpath, dbdir, + &fileinfo); + while (ret == 0) { + + if (fileinfo.type == OS_FILE_TYPE_DIR) { + + goto next_file_item_2; + } + + if (strlen(fileinfo.name) > suffix_len + && 0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - + suffix_len, + suffix)) { + /* The name ends in suffix; process + the file */ + if (!func( + path, + dbinfo.name, + fileinfo.name, func_arg)) + { + os_file_closedir(dbdir); + os_file_closedir(dir); + return(FALSE); + } + } +next_file_item_2: + ret = fil_file_readdir_next_file(&err, + dbpath, dbdir, + &fileinfo); + } + + os_file_closedir(dbdir); + } +next_datadir_item: + ret = fil_file_readdir_next_file(&err, + path, + dir, &dbinfo); + } + + os_file_closedir(dir); + + return(TRUE); +} + +/************************************************************************ +Applies all .delta files from incremental_dir to the full backup. +@return TRUE on success. */ +static +ibool +xtrabackup_apply_deltas() +{ + return xb_process_datadir(xtrabackup_incremental_dir, ".delta", + xtrabackup_apply_delta); +} + + +static +void +innodb_free_param() +{ + srv_sys_space.shutdown(); + free_tmpdir(&mysql_tmpdir_list); +} + + +/** Check if file exists*/ +static bool file_exists(std::string name) +{ + return access(name.c_str(), R_OK) == 0 ; +} + +/** Read file content into STL string */ +static std::string read_file_as_string(const std::string file) { + char content[FN_REFLEN]; + FILE *f = fopen(file.c_str(), "r"); + if (!f) { + msg("Can not open %s", file.c_str()); + } + size_t len = fread(content, 1, FN_REFLEN, f); + fclose(f); + return std::string(content, len); +} + +/** Delete file- Provide verbose diagnostics and exit, if operation fails. */ +static void delete_file(const std::string& file, bool if_exists = false) { + if (if_exists && !file_exists(file)) + return; + if (my_delete(file.c_str(), MYF(MY_WME))) { + die("Can't remove %s, errno %d", file.c_str(), errno); + } +} + +/** +Rename tablespace during prepare. +Backup in its end phase may generate some .ren files, recording +tablespaces that should be renamed in --prepare. +*/ +static void rename_table_in_prepare(const std::string &datadir, const std::string& from , const std::string& to, + const char *extension=0) { + if (!extension) { + static const char *extensions_nonincremental[] = { ".ibd", 0 }; + static const char *extensions_incremental[] = { ".ibd.delta", ".ibd.meta", 0 }; + const char **extensions = xtrabackup_incremental_dir ? + extensions_incremental : extensions_nonincremental; + for (size_t i = 0; extensions[i]; i++) { + rename_table_in_prepare(datadir, from, to, extensions[i]); + } + return; + } + std::string src = std::string(datadir) + "/" + from + extension; + std::string dest = std::string(datadir) + "/" + to + extension; + std::string ren2, tmp; + if (file_exists(dest)) { + ren2= std::string(datadir) + "/" + to + ".ren"; + if (!file_exists(ren2)) { + die("ERROR : File %s was not found, but expected during rename processing\n", ren2.c_str()); + } + tmp = to + "#"; + rename_table_in_prepare(datadir, to, tmp); + } + rename_file(src, dest); + if (ren2.size()) { + // Make sure the temp. renamed file is processed. + std::string to2 = read_file_as_string(ren2); + rename_table_in_prepare(datadir, tmp, to2); + delete_file(ren2); + } +} + +static ibool prepare_handle_ren_files(const char *datadir, const char *db, const char *filename, void *) { + + std::string ren_file = std::string(datadir) + "/" + db + "/" + filename; + if (!file_exists(ren_file)) + return TRUE; + + std::string to = read_file_as_string(ren_file); + std::string source_space_name = std::string(db) + "/" + filename; + source_space_name.resize(source_space_name.size() - 4); // remove extension + + rename_table_in_prepare(datadir, source_space_name.c_str(), to.c_str()); + delete_file(ren_file); + return TRUE; +} + +/* Remove tablespaces during backup, based on */ +static ibool prepare_handle_del_files(const char *datadir, const char *db, const char *filename, void *) { + std::string del_file = std::string(datadir) + "/" + db + "/" + filename; + std::string path(del_file); + path.resize(path.size() - 4); // remove extension; + if (xtrabackup_incremental) { + delete_file(path + ".ibd.delta", true); + delete_file(path + ".ibd.meta", true); + } + else { + delete_file(path + ".ibd", true); + } + delete_file(del_file); + return TRUE; +} + +/** Implement --prepare +@return whether the operation succeeded */ +static bool xtrabackup_prepare_func(char** argv) +{ + CorruptedPages corrupted_pages; + char metadata_path[FN_REFLEN]; + + /* cd to target-dir */ + + if (my_setwd(xtrabackup_real_target_dir,MYF(MY_WME))) + { + msg("can't my_setwd %s", xtrabackup_real_target_dir); + return(false); + } + msg("cd to %s", xtrabackup_real_target_dir); + + fil_path_to_mysql_datadir = "."; + + ut_ad(xtrabackup_incremental == xtrabackup_incremental_dir); + if (xtrabackup_incremental) + inc_dir_tables_hash.create(1000); + + msg("open files limit requested %u, set to %lu", + (uint) xb_open_files_limit, + xb_set_max_open_files(xb_open_files_limit)); + + /* Fix DDL for prepare. Process .del,.ren, and .new files. + The order in which files are processed, is important + (see MDEV-18185, MDEV-18201) + */ + xb_process_datadir(xtrabackup_incremental_dir ? xtrabackup_incremental_dir : ".", + ".del", prepare_handle_del_files); + xb_process_datadir(xtrabackup_incremental_dir? xtrabackup_incremental_dir:".", + ".ren", prepare_handle_ren_files); + if (xtrabackup_incremental_dir) { + xb_process_datadir(xtrabackup_incremental_dir, ".new.meta", prepare_handle_new_files); + xb_process_datadir(xtrabackup_incremental_dir, ".new.delta", prepare_handle_new_files); + xb_process_datadir(xtrabackup_incremental_dir, ".new", + prepare_handle_new_files, (void *)"."); + } + else { + xb_process_datadir(".", ".new", prepare_handle_new_files); + } + + int argc; for (argc = 0; argv[argc]; argc++) {} + xb_plugin_prepare_init(argc, argv, xtrabackup_incremental_dir); + + xtrabackup_target_dir= mysql_data_home_buff; + xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here + xtrabackup_target_dir[1]=0; + const lsn_t target_lsn = xtrabackup_incremental + ? incremental_to_lsn : metadata_to_lsn; + + /* + read metadata of target + */ + sprintf(metadata_path, "%s/%s", xtrabackup_target_dir, + XTRABACKUP_METADATA_FILENAME); + + if (!xtrabackup_read_metadata(metadata_path)) { + msg("Error: failed to read metadata from '%s'\n", + metadata_path); + return(false); + } + + if (!strcmp(metadata_type, "full-backuped")) { + if (xtrabackup_incremental) { + msg("error: applying incremental backup " + "needs a prepared target."); + return(false); + } + msg("This target seems to be not prepared yet."); + } else if (!strcmp(metadata_type, "log-applied")) { + msg("This target seems to be already prepared."); + } else { + msg("This target does not have correct metadata."); + return(false); + } + + bool ok = !xtrabackup_incremental + || metadata_to_lsn == incremental_lsn; + if (!ok) { + msg("error: This incremental backup seems " + "not to be proper for the target. Check 'to_lsn' of the target and " + "'from_lsn' of the incremental."); + return(false); + } + + srv_max_n_threads = 1000; + srv_n_purge_threads = 1; + + xb_filters_init(); + + srv_log_group_home_dir = NULL; + + if (xtrabackup_incremental) { + srv_operation = SRV_OPERATION_RESTORE_DELTA; + + if (innodb_init_param()) { +error: + ok = false; + goto cleanup; + } + + recv_sys.create(); + if (!log_sys.create()) { + goto error; + } + recv_sys.recovery_on = true; + + xb_fil_io_init(); + if (dberr_t err = xb_load_tablespaces()) { + msg("mariabackup: error: xb_data_files_init() failed " + "with error %s\n", ut_strerr(err)); + goto error; + } + + ok = fil_system.sys_space->open(false) + && xtrabackup_apply_deltas(); + + xb_data_files_close(); + + if (ok) { + /* Cleanup datadir from tablespaces deleted + between full and incremental backups */ + + xb_process_datadir("./", ".ibd", rm_if_not_found); + } + + xb_filter_hash_free(&inc_dir_tables_hash); + + fil_system.close(); + innodb_free_param(); + log_sys.close(); + if (!ok) goto cleanup; + } + + srv_operation = xtrabackup_export + ? SRV_OPERATION_RESTORE_EXPORT : SRV_OPERATION_RESTORE; + + if (innodb_init_param()) { + goto error; + } + + fil_system.freeze_space_list = 0; + + msg("Starting InnoDB instance for recovery."); + + msg("mariabackup: Using %lld bytes for buffer pool " + "(set by --use-memory parameter)", xtrabackup_use_memory); + + srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct; + + if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) { + srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct; + } + + if (innodb_init()) { + goto error; + } + + ut_ad(!fil_system.freeze_space_list); + + corrupted_pages.read_from_file(MB_CORRUPTED_PAGES_FILE); + if (xtrabackup_incremental) + { + char inc_filename[FN_REFLEN]; + sprintf(inc_filename, "%s/%s", xtrabackup_incremental_dir, + MB_CORRUPTED_PAGES_FILE); + corrupted_pages.read_from_file(inc_filename); + } + if (!corrupted_pages.empty()) + corrupted_pages.zero_out_free_pages(); + if (corrupted_pages.empty()) + { + if (!xtrabackup_incremental && unlink(MB_CORRUPTED_PAGES_FILE) && + errno != ENOENT) + { + char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf), errno); + die("Error: unlink %s failed: %s", MB_CORRUPTED_PAGES_FILE, + errbuf); + } + } + else + corrupted_pages.print_to_file(NULL, MB_CORRUPTED_PAGES_FILE); + + if (ok) { + msg("Last binlog file %s, position %lld", + trx_sys.recovered_binlog_filename, + longlong(trx_sys.recovered_binlog_offset)); + } + + /* Check whether the log is applied enough or not. */ + if (recv_sys.lsn && recv_sys.lsn < target_lsn) { + msg("mariabackup: error: " + "The log was only applied up to LSN " LSN_PF + ", instead of " LSN_PF, recv_sys.lsn, target_lsn); + ok = false; + } +#ifdef WITH_WSREP + else if (ok) xb_write_galera_info(xtrabackup_incremental); +#endif + + innodb_shutdown(); + + innodb_free_param(); + + /* output to metadata file */ + if (ok) { + char filename[FN_REFLEN]; + + safe_strcpy(metadata_type, sizeof(metadata_type), + "log-applied"); + + if(xtrabackup_incremental + && metadata_to_lsn < incremental_to_lsn) + { + metadata_to_lsn = incremental_to_lsn; + metadata_last_lsn = incremental_last_lsn; + } + + sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME); + if (!xtrabackup_write_metadata(filename)) { + + msg("mariabackup: Error: failed to write metadata " + "to '%s'", filename); + ok = false; + } else if (xtrabackup_extra_lsndir) { + sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME); + if (!xtrabackup_write_metadata(filename)) { + msg("mariabackup: Error: failed to write " + "metadata to '%s'", filename); + ok = false; + } + } + } + + if (ok) ok = apply_log_finish(); + + if (ok && xtrabackup_export) + ok= (prepare_export() == 0); + +cleanup: + xb_filters_free(); + return ok && !ib::error::was_logged() && corrupted_pages.empty(); +} + +/************************************************************************** +Append group name to xb_load_default_groups list. */ +static +void +append_defaults_group(const char *group, const char *default_groups[], + size_t default_groups_size) +{ + uint i; + bool appended = false; + for (i = 0; i < default_groups_size - 1; i++) { + if (default_groups[i] == NULL) { + default_groups[i] = group; + appended = true; + break; + } + } + ut_a(appended); +} + +static const char* +normalize_privilege_target_name(const char* name) +{ + if (strcmp(name, "*") == 0) { + return "\\*"; + } + else { + /* should have no regex special characters. */ + ut_ad(strpbrk(name, ".()[]*+?") == 0); + } + return name; +} + +/******************************************************************//** +Check if specific privilege is granted. +Uses regexp magic to check if requested privilege is granted for given +database.table or database.* or *.* +or if user has 'ALL PRIVILEGES' granted. +@return true if requested privilege is granted, false otherwise. */ +static bool +has_privilege(const std::list<std::string> &granted, + const char* required, + const char* db_name, + const char* table_name) +{ + char buffer[1000]; + regex_t priv_re; + regmatch_t tables_regmatch[1]; + bool result = false; + + db_name = normalize_privilege_target_name(db_name); + table_name = normalize_privilege_target_name(table_name); + + int written = snprintf(buffer, sizeof(buffer), + "GRANT .*(%s)|(ALL PRIVILEGES).* ON (\\*|`%s`)\\.(\\*|`%s`)", + required, db_name, table_name); + if (written < 0 || written == sizeof(buffer) + || regcomp(&priv_re, buffer, REG_EXTENDED)) { + die("regcomp() failed for '%s'", buffer); + } + + typedef std::list<std::string>::const_iterator string_iter; + for (string_iter i = granted.begin(), e = granted.end(); i != e; ++i) { + int res = regexec(&priv_re, i->c_str(), + 1, tables_regmatch, 0); + + if (res != REG_NOMATCH) { + result = true; + break; + } + } + + xb_regfree(&priv_re); + return result; +} + +enum { + PRIVILEGE_OK = 0, + PRIVILEGE_WARNING = 1, + PRIVILEGE_ERROR = 2, +}; + +/******************************************************************//** +Check if specific privilege is granted. +Prints error message if required privilege is missing. +@return PRIVILEGE_OK if requested privilege is granted, error otherwise. */ +static +int check_privilege( + const std::list<std::string> &granted_priv, /* in: list of + granted privileges*/ + const char* required, /* in: required privilege name */ + const char* target_database, /* in: required privilege target + database name */ + const char* target_table, /* in: required privilege target + table name */ + int error = PRIVILEGE_ERROR) /* in: return value if privilege + is not granted */ +{ + if (!has_privilege(granted_priv, + required, target_database, target_table)) { + msg("%s: missing required privilege %s on %s.%s", + (error == PRIVILEGE_ERROR ? "Error" : "Warning"), + required, target_database, target_table); + return error; + } + return PRIVILEGE_OK; +} + + +/** +Check DB user privileges according to the intended actions. + +Fetches DB user privileges, determines intended actions based on +command-line arguments and prints missing privileges. +@return whether all the necessary privileges are granted */ +static bool check_all_privileges() +{ + if (!mysql_connection) { + /* Not connected, no queries is going to be executed. */ + return true; + } + + /* Fetch effective privileges. */ + std::list<std::string> granted_privileges; + MYSQL_RES* result = xb_mysql_query(mysql_connection, "SHOW GRANTS", + true); + while (MYSQL_ROW row = mysql_fetch_row(result)) { + granted_privileges.push_back(*row); + } + mysql_free_result(result); + + int check_result = PRIVILEGE_OK; + + /* FLUSH TABLES WITH READ LOCK */ + if (!opt_no_lock) + { + check_result |= check_privilege( + granted_privileges, + "RELOAD", "*", "*"); + check_result |= check_privilege( + granted_privileges, + "PROCESS", "*", "*"); + } + + /* KILL ... */ + if (!opt_no_lock && (opt_kill_long_queries_timeout || opt_kill_long_query_type)) { + check_result |= check_privilege( + granted_privileges, + "CONNECTION ADMIN", "*", "*", + PRIVILEGE_WARNING); + } + + /* START SLAVE SQL_THREAD */ + /* STOP SLAVE SQL_THREAD */ + if (opt_safe_slave_backup) { + check_result |= check_privilege( + granted_privileges, + "REPLICATION SLAVE ADMIN", "*", "*", + PRIVILEGE_WARNING); + } + + /* SHOW MASTER STATUS */ + /* SHOW SLAVE STATUS */ + if (opt_galera_info || opt_slave_info + || opt_safe_slave_backup) { + check_result |= check_privilege(granted_privileges, + "SLAVE MONITOR", "*", "*", + PRIVILEGE_WARNING); + } + + if (check_result & PRIVILEGE_ERROR) { + msg("Current privileges, as reported by 'SHOW GRANTS': "); + int n=1; + for (std::list<std::string>::const_iterator it = granted_privileges.begin(); + it != granted_privileges.end(); + it++,n++) { + msg(" %d.%s", n, it->c_str()); + } + return false; + } + + return true; +} + +bool +xb_init() +{ + const char *mixed_options[4] = {NULL, NULL, NULL, NULL}; + int n_mixed_options; + + /* sanity checks */ + + if (opt_slave_info + && opt_no_lock + && !opt_safe_slave_backup) { + msg("Error: --slave-info is used with --no-lock but " + "without --safe-slave-backup. The binlog position " + "cannot be consistent with the backup data."); + return(false); + } + + if (xtrabackup_backup && opt_rsync) + { + if (xtrabackup_stream_fmt) + { + msg("Error: --rsync doesn't work with --stream\n"); + return(false); + } + bool have_rsync = IF_WIN(false, (system("rsync --version > /dev/null 2>&1") == 0)); + if (!have_rsync) + { + msg("Error: rsync executable not found, cannot run backup with --rsync\n"); + return false; + } + } + + n_mixed_options = 0; + + if (opt_decompress) { + mixed_options[n_mixed_options++] = "--decompress"; + } + + if (xtrabackup_copy_back) { + mixed_options[n_mixed_options++] = "--copy-back"; + } + + if (xtrabackup_move_back) { + mixed_options[n_mixed_options++] = "--move-back"; + } + + if (xtrabackup_prepare) { + mixed_options[n_mixed_options++] = "--apply-log"; + } + + if (n_mixed_options > 1) { + msg("Error: %s and %s are mutually exclusive\n", + mixed_options[0], mixed_options[1]); + return(false); + } + + if (xtrabackup_backup) { + if ((mysql_connection = xb_mysql_connect()) == NULL) { + return(false); + } + + if (!get_mysql_vars(mysql_connection)) { + return(false); + } + + if (opt_check_privileges && !check_all_privileges()) { + return(false); + } + history_start_time = time(NULL); + } + + return(true); +} + + +extern void init_signals(void); + +#include <sql_locale.h> + + +void setup_error_messages() +{ + my_default_lc_messages = &my_locale_en_US; + if (init_errmessage()) + die("could not initialize error messages"); +} + +/** Handle mariabackup options. The options are handled with the following +order: + +1) Load server groups and process server options, ignore unknown options +2) Load client groups and process client options, ignore unknown options +3) Load backup groups and process client-server options, exit on unknown option +4) Process --mysqld-args options, ignore unknown options + +@param[in] argc arguments count +@param[in] argv arguments array +@param[out] argv_server server options including loaded from server groups +@param[out] argv_client client options including loaded from client groups +@param[out] argv_backup backup options including loaded from backup groups */ +void handle_options(int argc, char **argv, char ***argv_server, + char ***argv_client, char ***argv_backup) +{ + /* Setup some variables for Innodb.*/ + srv_operation = SRV_OPERATION_RESTORE; + + files_charset_info = &my_charset_utf8mb3_general_ci; + + + setup_error_messages(); + sys_var_init(); + plugin_mutex_init(); + mysql_prlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash); + opt_stack_trace = 1; + test_flags |= TEST_SIGINT; + init_signals(); +#ifndef _WIN32 + /* Exit process on SIGINT. */ + my_sigset(SIGINT, SIG_DFL); +#endif + + sf_leaking_memory = 1; /* don't report memory leaks on early exist */ + + int i; + int ho_error; + + char* target_dir = NULL; + bool prepare = false; + + char conf_file[FN_REFLEN]; + + // array_elements() will not work for load_defaults, as it is defined + // as external symbol, so let's use dynamic array to have ability to + // add new server default groups + std::vector<const char *> server_default_groups; + + for (const char **default_group= load_default_groups; *default_group; + ++default_group) + server_default_groups.push_back(*default_group); + + std::vector<char *> mysqld_args; + std::vector<char *> mariabackup_args; + mysqld_args.push_back(argv[0]); + mariabackup_args.push_back(argv[0]); + + /* scan options for group and config file to load defaults from */ + for (i= 1; i < argc; i++) + { + char *optend= strcend(argv[i], '='); + if (mysqld_args.size() > 1 || + strncmp(argv[i], "--mysqld-args", optend - argv[i]) == 0) + { + mysqld_args.push_back(argv[i]); + continue; + } + else + mariabackup_args.push_back(argv[i]); + + if (strncmp(argv[i], "--defaults-group", optend - argv[i]) == 0) + { + defaults_group= optend + 1; + server_default_groups.push_back(defaults_group); + } + else if (strncmp(argv[i], "--login-path", optend - argv[i]) == 0) + { + append_defaults_group(optend + 1, xb_client_default_groups, + array_elements(xb_client_default_groups)); + } + else if (!strncmp(argv[i], "--prepare", optend - argv[i])) + { + prepare= true; + } + else if (!strncmp(argv[i], "--apply-log", optend - argv[i])) + { + prepare= true; + } + else if (!strncmp(argv[i], "--incremental-dir", optend - argv[i]) && + *optend) + { + target_dir= optend + 1; + } + else if (!strncmp(argv[i], "--target-dir", optend - argv[i]) && + *optend && !target_dir) + { + target_dir= optend + 1; + } + else if (!*optend && argv[i][0] != '-' && !target_dir) + { + target_dir= argv[i]; + } + } + + server_default_groups.push_back(NULL); + snprintf(conf_file, sizeof(conf_file), "my"); + + if (prepare && target_dir) { + snprintf(conf_file, sizeof(conf_file), + "%s/backup-my.cnf", target_dir); + if (!strncmp(argv[1], "--defaults-file=", 16)) { + /* Remove defaults-file*/ + for (int i = 2; ; i++) { + if ((argv[i-1]= argv[i]) == 0) + break; + } + argc--; + } + } + + mariabackup_args.push_back(nullptr); + *argv_client= *argv_server= *argv_backup= &mariabackup_args[0]; + int argc_backup= static_cast<int>(mariabackup_args.size() - 1); + int argc_client= argc_backup; + int argc_server= argc_backup; + + /* 1) Load server groups and process server options, ignore unknown + options */ + + load_defaults_or_exit(conf_file, &server_default_groups[0], + &argc_server, argv_server); + + int n; + for (n = 0; (*argv_server)[n]; n++) {}; + argc_server = n; + + print_param_str << + "# This MySQL options file was generated by XtraBackup.\n" + "[" << defaults_group << "]\n"; + + /* We want xtrabackup to ignore unknown options, because it only + recognizes a small subset of server variables */ + my_getopt_skip_unknown = TRUE; + + /* Reset u_max_value for all options, as we don't want the + --maximum-... modifier to set the actual option values */ + for (my_option *optp= xb_server_options; optp->name; optp++) { + optp->u_max_value = (G_PTR *) &global_max_value; + } + + /* Throw a descriptive error if --defaults-file or --defaults-extra-file + is not the first command line argument */ + for (int i = 2 ; i < argc ; i++) { + char *optend = strcend((argv)[i], '='); + + if (optend - argv[i] == 15 && + !strncmp(argv[i], "--defaults-file", optend - argv[i])) { + die("--defaults-file must be specified first on the command line"); + } + if (optend - argv[i] == 21 && + !strncmp(argv[i], "--defaults-extra-file", + optend - argv[i])) { + die("--defaults-extra-file must be specified first on the command line"); + } + } + + if (argc_server > 0 + && (ho_error=handle_options(&argc_server, argv_server, + xb_server_options, xb_get_one_option))) + exit(ho_error); + + /* 2) Load client groups and process client options, ignore unknown + options */ + + load_defaults_or_exit(conf_file, xb_client_default_groups, + &argc_client, argv_client); + + for (n = 0; (*argv_client)[n]; n++) {}; + argc_client = n; + + if (innobackupex_mode && argc_client > 0) { + if (!ibx_handle_options(&argc_client, argv_client)) { + exit(EXIT_FAILURE); + } + } + + if (argc_client > 0 + && (ho_error=handle_options(&argc_client, argv_client, + xb_client_options, xb_get_one_option))) + exit(ho_error); + + /* 3) Load backup groups and process client-server options, exit on + unknown option */ + + load_defaults_or_exit(conf_file, backup_default_groups, &argc_backup, + argv_backup); + for (n= 0; (*argv_backup)[n]; n++) + { + }; + argc_backup= n; + + my_handle_options_init_variables = FALSE; + + if (argc_backup > 0 && + (ho_error= handle_options(&argc_backup, argv_backup, + xb_server_options, xb_get_one_option))) + exit(ho_error); + + /* Add back the program name handle_options removes */ + ++argc_backup; + --(*argv_backup); + + if (innobackupex_mode && argc_backup > 0 && + !ibx_handle_options(&argc_backup, argv_backup)) + exit(EXIT_FAILURE); + + my_getopt_skip_unknown = FALSE; + + if (argc_backup > 0 && + (ho_error= handle_options(&argc_backup, argv_backup, + xb_client_options, xb_get_one_option))) + exit(ho_error); + + if (opt_password) + { + char *argument= (char*) opt_password; + char *start= (char*) opt_password; + opt_password= my_strdup(PSI_NOT_INSTRUMENTED, opt_password, + MYF(MY_FAE)); + while (*argument) + *argument++= 'x'; // Destroy argument + if (*start) + start[1]= 0; + } + + /* 4) Process --mysqld-args options, ignore unknown options */ + + my_getopt_skip_unknown = TRUE; + + int argc_mysqld = static_cast<int>(mysqld_args.size()); + if (argc_mysqld > 1) + { + char **argv_mysqld= &mysqld_args[0]; + if ((ho_error= handle_options(&argc_mysqld, &argv_mysqld, + xb_server_options, xb_get_one_option))) + exit(ho_error); + } + + my_handle_options_init_variables = TRUE; + + /* Reject command line arguments that don't look like options, i.e. are + not of the form '-X' (single-character options) or '--option' (long + options) */ + for (int i = 0 ; i < argc_backup ; i++) { + const char * const opt = (*argv_backup)[i]; + + if (strncmp(opt, "--", 2) && + !(strlen(opt) == 2 && opt[0] == '-')) { + bool server_option = true; + + for (int j = 0; j < argc_backup; j++) { + if (opt == (*argv_backup)[j]) { + server_option = false; + break; + } + } + + if (!server_option) { + msg("mariabackup: Error:" + " unknown argument: '%s'", opt); + exit(EXIT_FAILURE); + } + } + } +} + +static int main_low(char** argv); +static int get_exepath(char *buf, size_t size, const char *argv0); + +/* ================= main =================== */ +int main(int argc, char **argv) +{ + char **server_defaults; + char **client_defaults; + char **backup_defaults; + + my_getopt_prefix_matching= 0; + + if (get_exepath(mariabackup_exe,FN_REFLEN, argv[0])) + strncpy(mariabackup_exe,argv[0], FN_REFLEN-1); + + + if (argc > 1 ) + { + /* In "prepare export", we need to start mysqld + Since it is not always be installed on the machine, + we start "mariabackup --mysqld", which acts as mysqld + */ + if (strcmp(argv[1], "--mysqld") == 0) + { + srv_operation= SRV_OPERATION_EXPORT_RESTORED; + extern int mysqld_main(int argc, char **argv); + argc--; + argv++; + argv[0]+=2; + return mysqld_main(argc, argv); + } + if(strcmp(argv[1], "--innobackupex") == 0) + { + argv++; + argc--; + innobackupex_mode = true; + } + } + + if (argc > 1) + strncpy(orig_argv1,argv[1],sizeof(orig_argv1) -1); + + init_signals(); + MY_INIT(argv[0]); + + xb_regex_init(); + + capture_tool_command(argc, argv); + + if (mysql_server_init(-1, NULL, NULL)) + { + die("mysql_server_init() failed"); + } + + system_charset_info = &my_charset_utf8mb3_general_ci; + key_map_full.set_all(); + + logger.init_base(); + logger.set_handlers(LOG_NONE, LOG_NONE); + mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log, + MY_MUTEX_INIT_FAST); + + handle_options(argc, argv, &server_defaults, &client_defaults, + &backup_defaults); + +#ifndef DBUG_OFF + if (dbug_option) { + DBUG_SET_INITIAL(dbug_option); + DBUG_SET(dbug_option); + } +#endif + /* Main functions for library */ + init_thr_timer(5); + + int status = main_low(server_defaults); + + end_thr_timer(); + backup_cleanup(); + + if (innobackupex_mode) { + ibx_cleanup(); + } + + free_defaults(server_defaults); + free_defaults(client_defaults); + free_defaults(backup_defaults); + +#ifndef DBUG_OFF + if (dbug_option) { + DBUG_END(); + } +#endif + + logger.cleanup_base(); + cleanup_errmsgs(); + free_error_messages(); + mysql_mutex_destroy(&LOCK_error_log); + + if (status == EXIT_SUCCESS) { + msg("completed OK!"); + } + + return status; +} + +static int main_low(char** argv) +{ + if (innobackupex_mode) { + if (!ibx_init()) { + return(EXIT_FAILURE); + } + } + + if (!xtrabackup_print_param && !xtrabackup_prepare + && !strcmp(mysql_data_home, "./")) { + if (!xtrabackup_print_param) + usage(); + msg("mariabackup: Error: Please set parameter 'datadir'"); + return(EXIT_FAILURE); + } + + /* Expand target-dir, incremental-basedir, etc. */ + + char cwd[FN_REFLEN]; + my_getwd(cwd, sizeof(cwd), MYF(0)); + + my_load_path(xtrabackup_real_target_dir, + xtrabackup_target_dir, cwd); + unpack_dirname(xtrabackup_real_target_dir, + xtrabackup_real_target_dir); + xtrabackup_target_dir= xtrabackup_real_target_dir; + + if (xtrabackup_incremental_basedir) { + my_load_path(xtrabackup_real_incremental_basedir, + xtrabackup_incremental_basedir, cwd); + unpack_dirname(xtrabackup_real_incremental_basedir, + xtrabackup_real_incremental_basedir); + xtrabackup_incremental_basedir = + xtrabackup_real_incremental_basedir; + } + + if (xtrabackup_incremental_dir) { + my_load_path(xtrabackup_real_incremental_dir, + xtrabackup_incremental_dir, cwd); + unpack_dirname(xtrabackup_real_incremental_dir, + xtrabackup_real_incremental_dir); + xtrabackup_incremental_dir = xtrabackup_real_incremental_dir; + } + + if (xtrabackup_extra_lsndir) { + my_load_path(xtrabackup_real_extra_lsndir, + xtrabackup_extra_lsndir, cwd); + unpack_dirname(xtrabackup_real_extra_lsndir, + xtrabackup_real_extra_lsndir); + xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir; + } + + /* get default temporary directory */ + if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) { + opt_mysql_tmpdir = getenv("TMPDIR"); +#if defined(_WIN32) + if (!opt_mysql_tmpdir) { + opt_mysql_tmpdir = getenv("TEMP"); + } + if (!opt_mysql_tmpdir) { + opt_mysql_tmpdir = getenv("TMP"); + } +#endif + if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) { + opt_mysql_tmpdir = const_cast<char*>(DEFAULT_TMPDIR); + } + } + + /* temporary setting of enough size */ + srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_MAX; + srv_page_size = UNIV_PAGE_SIZE_MAX; + if (xtrabackup_backup && xtrabackup_incremental) { + /* direct specification is only for --backup */ + /* and the lsn is prior to the other option */ + + char* endchar; + int error = 0; + incremental_lsn = strtoll(xtrabackup_incremental, &endchar, 10); + if (*endchar != '\0') + error = 1; + + if (error) { + msg("mariabackup: value '%s' may be wrong format for " + "incremental option.", xtrabackup_incremental); + return(EXIT_FAILURE); + } + } else if (xtrabackup_backup && xtrabackup_incremental_basedir) { + char filename[FN_REFLEN]; + + sprintf(filename, "%s/%s", xtrabackup_incremental_basedir, XTRABACKUP_METADATA_FILENAME); + + if (!xtrabackup_read_metadata(filename)) { + msg("mariabackup: error: failed to read metadata from " + "%s", filename); + return(EXIT_FAILURE); + } + + incremental_lsn = metadata_to_lsn; + xtrabackup_incremental = xtrabackup_incremental_basedir; //dummy + } else if (xtrabackup_prepare && xtrabackup_incremental_dir) { + char filename[FN_REFLEN]; + + sprintf(filename, "%s/%s", xtrabackup_incremental_dir, XTRABACKUP_METADATA_FILENAME); + + if (!xtrabackup_read_metadata(filename)) { + msg("mariabackup: error: failed to read metadata from " + "%s", filename); + return(EXIT_FAILURE); + } + + incremental_lsn = metadata_from_lsn; + incremental_to_lsn = metadata_to_lsn; + incremental_last_lsn = metadata_last_lsn; + xtrabackup_incremental = xtrabackup_incremental_dir; //dummy + + } else if (opt_incremental_history_name) { + xtrabackup_incremental = opt_incremental_history_name; + } else if (opt_incremental_history_uuid) { + xtrabackup_incremental = opt_incremental_history_uuid; + } else { + xtrabackup_incremental = NULL; + } + + if (xtrabackup_stream && !xtrabackup_backup) { + msg("Warning: --stream parameter is ignored, it only works together with --backup."); + } + + if (!xb_init()) { + return(EXIT_FAILURE); + } + + /* --print-param */ + if (xtrabackup_print_param) { + printf("%s", print_param_str.str().c_str()); + return(EXIT_SUCCESS); + } + + print_version(); + if (xtrabackup_incremental) { + msg("incremental backup from " LSN_PF " is enabled.", + incremental_lsn); + } + + if (xtrabackup_export && !srv_file_per_table) { + msg("mariabackup: auto-enabling --innodb-file-per-table due to " + "the --export option"); + srv_file_per_table = TRUE; + } + + /* cannot execute both for now */ + { + int num = 0; + + if (xtrabackup_backup) num++; + if (xtrabackup_prepare) num++; + if (xtrabackup_copy_back) num++; + if (xtrabackup_move_back) num++; + if (xtrabackup_decrypt_decompress) num++; + if (num != 1) { /* !XOR (for now) */ + usage(); + return(EXIT_FAILURE); + } + } + + ut_ad(!field_ref_zero); + if (auto b = aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) { + field_ref_zero = static_cast<byte*>( + memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX)); + } else { + msg("Can't allocate memory for field_ref_zero"); + return EXIT_FAILURE; + } + + auto _ = make_scope_exit([]() { + aligned_free(const_cast<byte*>(field_ref_zero)); + field_ref_zero = nullptr; + }); + + /* --backup */ + if (xtrabackup_backup && !xtrabackup_backup_func()) { + return(EXIT_FAILURE); + } + + /* --prepare */ + if (xtrabackup_prepare + && !xtrabackup_prepare_func(argv)) { + return(EXIT_FAILURE); + } + + if (xtrabackup_copy_back || xtrabackup_move_back) { + if (!check_if_param_set("datadir")) { + mysql_data_home = get_default_datadir(); + } + if (!copy_back()) + return(EXIT_FAILURE); + } + + if (xtrabackup_decrypt_decompress && !decrypt_decompress()) { + return(EXIT_FAILURE); + } + + return(EXIT_SUCCESS); +} + + +static int get_exepath(char *buf, size_t size, const char *argv0) +{ +#ifdef _WIN32 + DWORD ret = GetModuleFileNameA(NULL, buf, (DWORD)size); + if (ret > 0) + return 0; +#elif defined(__linux__) + ssize_t ret = readlink("/proc/self/exe", buf, size-1); + if(ret > 0) + return 0; +#elif defined(__APPLE__) + size_t ret = proc_pidpath(getpid(), buf, static_cast<uint32_t>(size)); + if (ret > 0) { + buf[ret] = 0; + return 0; + } +#elif defined(__FreeBSD__) + int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; + if (sysctl(mib, 4, buf, &size, NULL, 0) == 0) { + return 0; + } +#endif + + return my_realpath(buf, argv0, 0); +} + + +#if defined (__SANITIZE_ADDRESS__) && defined (__linux__) +/* Avoid LeakSanitizer's false positives. */ +const char* __asan_default_options() +{ + return "detect_leaks=0"; +} +#endif diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h new file mode 100644 index 00000000..53784a3f --- /dev/null +++ b/extra/mariabackup/xtrabackup.h @@ -0,0 +1,302 @@ +/****************************************************** +Copyright (c) 2011-2015 Percona LLC and/or its affiliates. + +Declarations for xtrabackup.cc + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*******************************************************/ + +#ifndef XB_XTRABACKUP_H +#define XB_XTRABACKUP_H + +#include <my_getopt.h> +#include "datasink.h" +#include "xbstream.h" +#include "changed_page_bitmap.h" +#include <set> + +#define XB_TOOL_NAME "mariadb-backup" +#define XB_HISTORY_TABLE "mysql.mariadb_backup_history" + +struct xb_delta_info_t +{ + xb_delta_info_t(ulint page_size, ulint zip_size, uint32_t space_id) + : page_size(page_size), zip_size(zip_size), space_id(space_id) {} + + ulint page_size; + ulint zip_size; + uint32_t space_id; +}; + +class CorruptedPages +{ +public: + CorruptedPages(); + ~CorruptedPages(); + void add_page(const char *file_name, page_id_t page_id); + bool contains(page_id_t page_id) const; + void drop_space(uint32_t space_id); + void rename_space(uint32_t space_id, const std::string &new_name); + bool print_to_file(ds_ctxt *ds_data, const char *file_name) const; + void read_from_file(const char *file_name); + bool empty() const; + void zero_out_free_pages(); + + void backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta); + +private: + void add_page_no_lock(const char *space_name, page_id_t page_id, + bool convert_space_name); + struct space_info_t { + std::string space_name; + std::set<uint32_t> pages; + }; + typedef std::map<uint32_t, space_info_t> container_t; + mutable pthread_mutex_t m_mutex; + container_t m_spaces; +}; + + +/* value of the --incremental option */ +extern lsn_t incremental_lsn; + +extern char *xtrabackup_target_dir; +extern char *xtrabackup_incremental_dir; +extern char *xtrabackup_incremental_basedir; +extern char *innobase_data_home_dir; +extern char *innobase_buffer_pool_filename; +extern char *aria_log_dir_path; +extern char *xb_plugin_dir; +extern char *xb_rocksdb_datadir; +extern my_bool xb_backup_rocksdb; + +extern uint opt_protocol; + +extern xb_page_bitmap *changed_page_bitmap; + +extern char *xtrabackup_incremental; +extern my_bool xtrabackup_incremental_force_scan; + +extern lsn_t metadata_to_lsn; + +extern xb_stream_fmt_t xtrabackup_stream_fmt; +extern ibool xtrabackup_stream; + +extern char *xtrabackup_tables; +extern char *xtrabackup_tables_file; +extern char *xtrabackup_databases; +extern char *xtrabackup_databases_file; +extern char *xtrabackup_tables_exclude; +extern char *xtrabackup_databases_exclude; + +extern uint xtrabackup_compress; + +extern my_bool xtrabackup_backup; +extern my_bool xtrabackup_prepare; +extern my_bool xtrabackup_copy_back; +extern my_bool xtrabackup_move_back; +extern my_bool xtrabackup_decrypt_decompress; + +extern char *innobase_data_file_path; +extern longlong innobase_page_size; + +extern int xtrabackup_parallel; + +extern my_bool xb_close_files; +extern const char *xtrabackup_compress_alg; + +extern uint xtrabackup_compress_threads; +extern ulonglong xtrabackup_compress_chunk_size; + +extern my_bool xtrabackup_export; +extern char *xtrabackup_extra_lsndir; +extern ulint xtrabackup_log_copy_interval; +extern char *xtrabackup_stream_str; +extern long xtrabackup_throttle; +extern longlong xtrabackup_use_memory; + +extern my_bool opt_galera_info; +extern my_bool opt_slave_info; +extern my_bool opt_no_lock; +extern my_bool opt_safe_slave_backup; +extern my_bool opt_rsync; +extern my_bool opt_force_non_empty_dirs; +extern my_bool opt_noversioncheck; +extern my_bool opt_no_backup_locks; +extern my_bool opt_decompress; +extern my_bool opt_remove_original; +extern my_bool opt_extended_validation; +extern my_bool opt_encrypted_backup; +extern my_bool opt_lock_ddl_per_table; +extern my_bool opt_log_innodb_page_corruption; + +extern char *opt_incremental_history_name; +extern char *opt_incremental_history_uuid; + +extern char *opt_user; +extern const char *opt_password; +extern char *opt_host; +extern char *opt_defaults_group; +extern char *opt_socket; +extern uint opt_port; +extern char *opt_log_bin; + +extern const char *query_type_names[]; + +enum query_type_t {QUERY_TYPE_ALL, QUERY_TYPE_UPDATE, + QUERY_TYPE_SELECT}; + +extern TYPELIB query_type_typelib; + +extern ulong opt_lock_wait_query_type; +extern ulong opt_kill_long_query_type; + +extern uint opt_kill_long_queries_timeout; +extern uint opt_lock_wait_timeout; +extern uint opt_lock_wait_threshold; +extern uint opt_debug_sleep_before_unlock; +extern uint opt_safe_slave_backup_timeout; + +extern const char *opt_history; + +enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_ON, + BINLOG_INFO_AUTO}; + +extern ulong opt_binlog_info; + +extern ulong xtrabackup_innodb_force_recovery; + +void xtrabackup_io_throttling(void); +my_bool xb_write_delta_metadata(ds_ctxt *ds_meta, + const char *filename, + const xb_delta_info_t *info); + +/************************************************************************ +Checks if a table specified as a name in the form "database/name" (InnoDB 5.6) +or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on +the --tables or --tables-file options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_table( +/******************/ + const char* name); /*!< in: path to the table */ + + +/************************************************************************ +Checks if a database specified by path should be skipped from backup based on +the --databases, --databases_file or --databases_exclude options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_database_by_path( + const char* path /*!< in: path to the db directory. */ +); + +/************************************************************************ +Check if parameter is set in defaults file or via command line argument +@return true if parameter is set. */ +bool +check_if_param_set(const char *param); + +#if defined(HAVE_OPENSSL) +extern my_bool opt_ssl_verify_server_cert; +#endif + + +my_bool +xb_get_one_option(int optid, + const struct my_option *opt __attribute__((unused)), + char *argument); + +const char* +xb_get_copy_action(const char *dflt = "Copying"); + +void mdl_lock_init(); +void mdl_lock_table(ulint space_id); +void mdl_unlock_all(); +bool ends_with(const char *str, const char *suffix); + +typedef void (*insert_entry_func_t)(const char*); + +/* Scan string and load filter entries from it. +@param[in] list string representing a list +@param[in] delimiters delimiters of entries +@param[in] ins callback to add entry */ +void xb_load_list_string(char *list, const char *delimiters, + insert_entry_func_t ins); +void register_ignore_db_dirs_filter(const char *name); + +#ifdef _WIN32 +typedef HANDLE os_file_dir_t; /*!< directory stream */ +/** The os_file_opendir() function opens a directory stream corresponding to the +directory named by the dirname argument. The directory stream is positioned +at the first entry. In both Unix and Windows we automatically skip the '.' +and '..' items at the start of the directory listing. + +@param[in] dirname directory name; it must not contain a trailing + '\' or '/' +@return directory stream +@retval INVALID_HANDLE_VALUE on error */ +HANDLE os_file_opendir(const char *dirname); +# define os_file_closedir(dir) static_cast<void>(FindClose(dir)) +# define os_file_closedir_failed(dir) !FindClose(dir) +#else +typedef DIR* os_file_dir_t; +# define os_file_opendir(dirname) opendir(dirname) +# define os_file_closedir(dir) static_cast<void>(closedir(dir)) +# define os_file_closedir_failed(dir) closedir(dir) +#endif + +/** This function returns information of the next file in the directory. We jump +over the '.' and '..' entries in the directory. +@param[in] dirname directory name or path +@param[in] dir directory stream +@param[out] info buffer where the info is returned +@return 0 if ok, -1 if error, 1 if at the end of the directory */ +int +os_file_readdir_next_file( + const char* dirname, + os_file_dir_t dir, + os_file_stat_t* info); + +/***********************************************************************//** +A fault-tolerant function that tries to read the next file name in the +directory. We retry 100 times if os_file_readdir_next_file() returns -1. The +idea is to read as much good data as we can and jump over bad data. +@return 0 if ok, -1 if error even after the retries, 1 if at the end +of the directory */ +int +fil_file_readdir_next_file( +/*=======================*/ + dberr_t* err, /*!< out: this is set to DB_ERROR if an error + was encountered, otherwise not changed */ + const char* dirname,/*!< in: directory name or path */ + os_file_dir_t dir, /*!< in: directory stream */ + os_file_stat_t* info); /*!< in/out: buffer where the + info is returned */ + +#ifndef DBUG_OFF +#include <fil0fil.h> +extern void dbug_mariabackup_event(const char *event, + const fil_space_t::name_type key); + +#define DBUG_MARIABACKUP_EVENT(A, B) \ + DBUG_EXECUTE_IF("mariabackup_events", dbug_mariabackup_event(A, B);) +#else +#define DBUG_MARIABACKUP_EVENT(A, B) /* empty */ +#endif // DBUG_OFF + +#endif /* XB_XTRABACKUP_H */ |