summaryrefslogtreecommitdiffstats
path: root/extra/mariabackup/xtrabackup.cc
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-13 12:24:36 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-13 12:24:36 +0000
commit06eaf7232e9a920468c0f8d74dcf2fe8b555501c (patch)
treee2c7b5777f728320e5b5542b6213fd3591ba51e2 /extra/mariabackup/xtrabackup.cc
parentInitial commit. (diff)
downloadmariadb-06eaf7232e9a920468c0f8d74dcf2fe8b555501c.tar.xz
mariadb-06eaf7232e9a920468c0f8d74dcf2fe8b555501c.zip
Adding upstream version 1:10.11.6.upstream/1%10.11.6
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'extra/mariabackup/xtrabackup.cc')
-rw-r--r--extra/mariabackup/xtrabackup.cc7083
1 files changed, 7083 insertions, 0 deletions
diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
new file mode 100644
index 00000000..9e359257
--- /dev/null
+++ b/extra/mariabackup/xtrabackup.cc
@@ -0,0 +1,7083 @@
+/******************************************************
+MariaBackup: hot backup tool for InnoDB
+(c) 2009-2017 Percona LLC and/or its affiliates
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+(c) 2017, 2022, MariaDB Corporation.
+Portions written by Marko Mäkelä.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
+
+*******************************************************
+
+This file incorporates work covered by the following copyright and
+permission notice:
+
+Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
+Street, Fifth Floor, Boston, MA 02110-1335 USA
+
+*******************************************************/
+
+//#define XTRABACKUP_TARGET_IS_PLUGIN
+
+#include <my_global.h>
+#include <my_config.h>
+#include <unireg.h>
+#include <mysql_version.h>
+#include <my_base.h>
+#include <my_getopt.h>
+#include <mysql_com.h>
+#include <my_default.h>
+#include <scope.h>
+#include <sql_class.h>
+
+#include <fcntl.h>
+#include <string.h>
+
+#ifdef __linux__
+# include <sys/prctl.h>
+# include <sys/resource.h>
+#endif
+
+#ifdef __APPLE__
+# include "libproc.h"
+#endif
+
+#ifdef __FreeBSD__
+# include <sys/sysctl.h>
+#endif
+
+
+#include <btr0sea.h>
+#include <lock0lock.h>
+#include <log0recv.h>
+#include <log0crypt.h>
+#include <row0mysql.h>
+#include <row0quiesce.h>
+#include <srv0start.h>
+#include "trx0sys.h"
+#include <buf0dblwr.h>
+#include <buf0flu.h>
+#include "ha_innodb.h"
+
+#include <list>
+#include <sstream>
+#include <set>
+#include <fstream>
+#include <mysql.h>
+
+#define G_PTR uchar*
+
+#include "common.h"
+#include "datasink.h"
+
+#include "xb_regex.h"
+#include "fil_cur.h"
+#include "write_filt.h"
+#include "xtrabackup.h"
+#include "ds_buffer.h"
+#include "ds_tmpfile.h"
+#include "xbstream.h"
+#include "changed_page_bitmap.h"
+#include "read_filt.h"
+#include "backup_wsrep.h"
+#include "innobackupex.h"
+#include "backup_mysql.h"
+#include "backup_copy.h"
+#include "backup_mysql.h"
+#include "xb_plugin.h"
+#include <sql_plugin.h>
+#include <srv0srv.h>
+#include <log.h>
+#include <derror.h>
+#include <thr_timer.h>
+#include "backup_debug.h"
+
+#define MB_CORRUPTED_PAGES_FILE "innodb_corrupted_pages"
+
+// disable server's systemd notification code
+extern "C" {
+int sd_notify() { return 0; }
+int sd_notifyf() { return 0; }
+}
+
+int sys_var_init();
+
+/* === xtrabackup specific options === */
+char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/";
+char *xtrabackup_target_dir= xtrabackup_real_target_dir;
+static my_bool xtrabackup_version;
+static my_bool verbose;
+my_bool xtrabackup_backup;
+my_bool xtrabackup_prepare;
+my_bool xtrabackup_copy_back;
+my_bool xtrabackup_move_back;
+my_bool xtrabackup_decrypt_decompress;
+my_bool xtrabackup_print_param;
+my_bool xtrabackup_mysqld_args;
+my_bool xtrabackup_help;
+
+my_bool xtrabackup_export;
+
+longlong xtrabackup_use_memory;
+
+uint opt_protocol;
+long xtrabackup_throttle; /* 0:unlimited */
+static lint io_ticket;
+static mysql_cond_t wait_throttle;
+static mysql_cond_t log_copying_stop;
+
+char *xtrabackup_incremental;
+lsn_t incremental_lsn;
+lsn_t incremental_to_lsn;
+lsn_t incremental_last_lsn;
+xb_page_bitmap *changed_page_bitmap;
+
+char *xtrabackup_incremental_basedir; /* for --backup */
+char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */
+char *xtrabackup_incremental_dir; /* for --prepare */
+
+char xtrabackup_real_incremental_basedir[FN_REFLEN];
+char xtrabackup_real_extra_lsndir[FN_REFLEN];
+char xtrabackup_real_incremental_dir[FN_REFLEN];
+
+
+char *xtrabackup_tmpdir;
+
+char *xtrabackup_tables;
+char *xtrabackup_tables_file;
+char *xtrabackup_tables_exclude;
+char *xb_rocksdb_datadir;
+my_bool xb_backup_rocksdb = 1;
+
+typedef std::list<regex_t> regex_list_t;
+static regex_list_t regex_include_list;
+static regex_list_t regex_exclude_list;
+
+static hash_table_t tables_include_hash;
+static hash_table_t tables_exclude_hash;
+
+char *xtrabackup_databases = NULL;
+char *xtrabackup_databases_file = NULL;
+char *xtrabackup_databases_exclude = NULL;
+static hash_table_t databases_include_hash;
+static hash_table_t databases_exclude_hash;
+
+static hash_table_t inc_dir_tables_hash;
+
+struct xb_filter_entry_t{
+ char* name;
+ ibool has_tables;
+ xb_filter_entry_t *name_hash;
+};
+
+/** whether log_copying_thread() is active; protected by recv_sys.mutex */
+static bool log_copying_running;
+
+int xtrabackup_parallel;
+
+char *xtrabackup_stream_str = NULL;
+xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE;
+ibool xtrabackup_stream = FALSE;
+
+const char *xtrabackup_compress_alg = NULL;
+uint xtrabackup_compress = FALSE;
+uint xtrabackup_compress_threads;
+ulonglong xtrabackup_compress_chunk_size = 0;
+
+/* sleep interval beetween log copy iterations in log copying thread
+in milliseconds (default is 1 second) */
+ulint xtrabackup_log_copy_interval = 1000;
+static ulong max_buf_pool_modified_pct;
+
+/* Ignored option (--log) for MySQL option compatibility */
+static char* log_ignored_opt;
+
+
+extern my_bool opt_use_ssl;
+extern char *opt_tls_version;
+my_bool opt_ssl_verify_server_cert;
+my_bool opt_extended_validation;
+my_bool opt_encrypted_backup;
+
+/* === metadata of backup === */
+#define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints"
+char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/
+static lsn_t metadata_from_lsn;
+lsn_t metadata_to_lsn;
+static lsn_t metadata_last_lsn;
+
+static ds_file_t* dst_log_file;
+
+static char mysql_data_home_buff[2];
+
+const char *defaults_group = "mysqld";
+
+/* === static parameters in ha_innodb.cc */
+
+#define HA_INNOBASE_ROWS_IN_TABLE 10000 /* to get optimization right */
+#define HA_INNOBASE_RANGE_COUNT 100
+
+/* The default values for the following, type long or longlong, start-up
+parameters are declared in mysqld.cc: */
+
+long innobase_buffer_pool_awe_mem_mb = 0;
+long innobase_file_io_threads = 4;
+ulong innobase_read_io_threads = 4;
+ulong innobase_write_io_threads = 4;
+
+/** Store the failed read of undo tablespace ids. Protected by
+recv_sys.mutex. */
+static std::set<uint32_t> fail_undo_ids;
+
+longlong innobase_page_size = (1LL << 14); /* 16KB */
+char* innobase_buffer_pool_filename = NULL;
+
+/* The default values for the following char* start-up parameters
+are determined in innobase_init below: */
+
+static char* innobase_ignored_opt;
+char* innobase_data_home_dir;
+char* innobase_data_file_path;
+
+char *aria_log_dir_path;
+
+my_bool xtrabackup_incremental_force_scan = FALSE;
+
+/*
+ * Ignore corrupt pages (disabled by default; used
+ * by "innobackupex" as a command line argument).
+ */
+ulong xtrabackup_innodb_force_recovery = 0;
+
+/* The flushed lsn which is read from data files */
+lsn_t flushed_lsn= 0;
+
+ulong xb_open_files_limit= 0;
+char *xb_plugin_dir;
+char *xb_plugin_load;
+my_bool xb_close_files;
+
+
+class Datasink_free_list
+{
+protected:
+ /*
+ Simple datasink creation tracking...
+ add datasinks in the reverse order you want them destroyed.
+ */
+#define XTRABACKUP_MAX_DATASINKS 10
+ ds_ctxt_t *m_datasinks_to_destroy[XTRABACKUP_MAX_DATASINKS];
+ uint m_actual_datasinks_to_destroy;
+public:
+ Datasink_free_list()
+ :m_actual_datasinks_to_destroy(0)
+ { }
+
+ void add_datasink_to_destroy(ds_ctxt_t *ds)
+ {
+ xb_ad(m_actual_datasinks_to_destroy < XTRABACKUP_MAX_DATASINKS);
+ m_datasinks_to_destroy[m_actual_datasinks_to_destroy] = ds;
+ m_actual_datasinks_to_destroy++;
+ }
+
+ /*
+ Destroy datasinks.
+ Destruction is done in the specific order to not violate their order in the
+ pipeline so that each datasink is able to flush data down the pipeline.
+ */
+ void destroy()
+ {
+ for (uint i= m_actual_datasinks_to_destroy; i > 0; i--)
+ {
+ ds_destroy(m_datasinks_to_destroy[i - 1]);
+ m_datasinks_to_destroy[i - 1] = NULL;
+ }
+ }
+};
+
+
+class Backup_datasinks: public Datasink_free_list
+{
+public:
+ ds_ctxt_t *m_data;
+ ds_ctxt_t *m_meta;
+ ds_ctxt_t *m_redo;
+
+ Backup_datasinks()
+ :m_data(NULL),
+ m_meta(NULL),
+ m_redo(NULL)
+ { }
+ void init();
+ void destroy()
+ {
+ Datasink_free_list::destroy();
+ *this= Backup_datasinks();
+ }
+ bool backup_low();
+};
+
+
+static bool innobackupex_mode = false;
+
+/* String buffer used by --print-param to accumulate server options as they are
+parsed from the defaults file */
+static std::ostringstream print_param_str;
+
+/* Set of specified parameters */
+std::set<std::string> param_set;
+
+static ulonglong global_max_value;
+
+extern "C" sig_handler handle_fatal_signal(int sig);
+extern LOGGER logger;
+
+my_bool opt_galera_info = FALSE;
+my_bool opt_slave_info = FALSE;
+my_bool opt_no_lock = FALSE;
+my_bool opt_safe_slave_backup = FALSE;
+my_bool opt_rsync = FALSE;
+my_bool opt_force_non_empty_dirs = FALSE;
+my_bool opt_noversioncheck = FALSE;
+my_bool opt_no_backup_locks = FALSE;
+my_bool opt_decompress = FALSE;
+my_bool opt_remove_original;
+my_bool opt_log_innodb_page_corruption;
+
+my_bool opt_lock_ddl_per_table = FALSE;
+static my_bool opt_check_privileges;
+
+extern const char *innodb_checksum_algorithm_names[];
+extern TYPELIB innodb_checksum_algorithm_typelib;
+extern const char *innodb_flush_method_names[];
+extern TYPELIB innodb_flush_method_typelib;
+
+static const char *binlog_info_values[] = {"off", "lockless", "on", "auto",
+ NullS};
+static TYPELIB binlog_info_typelib = {array_elements(binlog_info_values)-1, "",
+ binlog_info_values, NULL};
+ulong opt_binlog_info;
+
+char *opt_incremental_history_name;
+char *opt_incremental_history_uuid;
+
+char *opt_user;
+const char *opt_password;
+char *opt_host;
+char *opt_defaults_group;
+char *opt_socket;
+uint opt_port;
+char *opt_log_bin;
+
+const char *query_type_names[] = { "ALL", "UPDATE", "SELECT", NullS};
+
+TYPELIB query_type_typelib= {array_elements(query_type_names) - 1, "",
+ query_type_names, NULL};
+
+ulong opt_lock_wait_query_type;
+ulong opt_kill_long_query_type;
+
+uint opt_kill_long_queries_timeout = 0;
+uint opt_lock_wait_timeout = 0;
+uint opt_lock_wait_threshold = 0;
+uint opt_debug_sleep_before_unlock = 0;
+uint opt_safe_slave_backup_timeout = 0;
+
+const char *opt_history = NULL;
+
+
+char mariabackup_exe[FN_REFLEN];
+char orig_argv1[FN_REFLEN];
+
+pthread_cond_t scanned_lsn_cond;
+
+/** Store the deferred tablespace name during --backup */
+static std::set<std::string> defer_space_names;
+
+typedef std::map<space_id_t,std::string> space_id_to_name_t;
+
+struct ddl_tracker_t {
+ /** Tablspaces with their ID and name, as they were copied to backup.*/
+ space_id_to_name_t tables_in_backup;
+ /** Drop operations found in redo log. */
+ std::set<space_id_t> drops;
+ /* For DDL operation found in redo log, */
+ space_id_to_name_t id_to_name;
+ /** Deferred tablespaces with their ID and name which was
+ found in redo log of DDL operations */
+ space_id_to_name_t deferred_tables;
+
+ /** Insert the deferred tablespace id with the name */
+ void insert_defer_id(space_id_t space_id, std::string name)
+ {
+ auto it= defer_space_names.find(name);
+ if (it != defer_space_names.end())
+ {
+ deferred_tables[space_id]= name;
+ defer_space_names.erase(it);
+ }
+ }
+
+ /** Rename the deferred tablespace with new name */
+ void rename_defer(space_id_t space_id, std::string old_name,
+ std::string new_name)
+ {
+ if (deferred_tables.find(space_id) != deferred_tables.end())
+ deferred_tables[space_id] = new_name;
+ auto defer_end= defer_space_names.end();
+ auto defer= defer_space_names.find(old_name);
+ if (defer == defer_end)
+ defer= defer_space_names.find(new_name);
+
+ if (defer != defer_end)
+ {
+ deferred_tables[space_id]= new_name;
+ defer_space_names.erase(defer);
+ }
+ }
+
+ /** Delete the deferred tablespace */
+ void delete_defer(space_id_t space_id, std::string name)
+ {
+ deferred_tables.erase(space_id);
+ defer_space_names.erase(name);
+ }
+};
+
+static ddl_tracker_t ddl_tracker;
+
+/** Store the space ids of truncated undo log tablespaces. Protected
+by recv_sys.mutex */
+static std::set<uint32_t> undo_trunc_ids;
+
+/** Stores the space ids of page0 INIT_PAGE redo records. It is
+used to indicate whether the given deferred tablespace can
+be reconstructed. */
+static std::set<space_id_t> first_page_init_ids;
+
+// Convert non-null terminated filename to space name
+static std::string filename_to_spacename(const void *filename, size_t len);
+
+CorruptedPages::CorruptedPages() { ut_a(!pthread_mutex_init(&m_mutex, NULL)); }
+
+CorruptedPages::~CorruptedPages() { ut_a(!pthread_mutex_destroy(&m_mutex)); }
+
+void CorruptedPages::add_page_no_lock(const char *space_name,
+ page_id_t page_id,
+ bool convert_space_name)
+{
+ space_info_t &space_info = m_spaces[page_id.space()];
+ if (space_info.space_name.empty())
+ space_info.space_name= convert_space_name
+ ? filename_to_spacename(space_name, strlen(space_name))
+ : space_name;
+ (void)space_info.pages.insert(page_id.page_no());
+}
+
+void CorruptedPages::add_page(const char *file_name, page_id_t page_id)
+{
+ pthread_mutex_lock(&m_mutex);
+ add_page_no_lock(file_name, page_id, true);
+ pthread_mutex_unlock(&m_mutex);
+}
+
+bool CorruptedPages::contains(page_id_t page_id) const
+{
+ bool result = false;
+ ut_a(!pthread_mutex_lock(&m_mutex));
+ container_t::const_iterator space_it= m_spaces.find(page_id.space());
+ if (space_it != m_spaces.end())
+ result = space_it->second.pages.count(page_id.page_no());
+ ut_a(!pthread_mutex_unlock(&m_mutex));
+ return result;
+}
+
+void CorruptedPages::drop_space(uint32_t space_id)
+{
+ ut_a(!pthread_mutex_lock(&m_mutex));
+ m_spaces.erase(space_id);
+ ut_a(!pthread_mutex_unlock(&m_mutex));
+}
+
+void CorruptedPages::rename_space(uint32_t space_id,
+ const std::string &new_name)
+{
+ ut_a(!pthread_mutex_lock(&m_mutex));
+ container_t::iterator space_it = m_spaces.find(space_id);
+ if (space_it != m_spaces.end())
+ space_it->second.space_name = new_name;
+ ut_a(!pthread_mutex_unlock(&m_mutex));
+}
+
+bool CorruptedPages::print_to_file(ds_ctxt *ds_data,
+ const char *filename) const
+{
+ std::ostringstream out;
+ ut_a(!pthread_mutex_lock(&m_mutex));
+ if (!m_spaces.size())
+ {
+ ut_a(!pthread_mutex_unlock(&m_mutex));
+ return true;
+ }
+ for (container_t::const_iterator space_it=
+ m_spaces.begin();
+ space_it != m_spaces.end(); ++space_it)
+ {
+ out << space_it->second.space_name << " " << space_it->first << "\n";
+ bool first_page_no= true;
+ for (std::set<unsigned>::const_iterator page_it=
+ space_it->second.pages.begin();
+ page_it != space_it->second.pages.end(); ++page_it)
+ if (first_page_no)
+ {
+ out << *page_it;
+ first_page_no= false;
+ }
+ else
+ out << " " << *page_it;
+ out << "\n";
+ }
+ ut_a(!pthread_mutex_unlock(&m_mutex));
+ if (ds_data)
+ return ds_data->backup_file_print_buf(filename, out.str().c_str(),
+ static_cast<int>(out.str().size()));
+ std::ofstream outfile;
+ outfile.open(filename);
+ if (!outfile.is_open())
+ die("Can't open %s, error number: %d, error message: %s", filename, errno,
+ strerror(errno));
+ outfile << out.str();
+ return true;
+}
+
+void CorruptedPages::read_from_file(const char *file_name)
+{
+ MY_STAT mystat;
+ if (!my_stat(file_name, &mystat, MYF(0)))
+ return;
+ std::ifstream infile;
+ infile.open(file_name);
+ if (!infile.is_open())
+ die("Can't open %s, error number: %d, error message: %s", file_name, errno,
+ strerror(errno));
+ std::string line;
+ std::string space_name;
+ uint32_t space_id;
+ ulint line_number= 0;
+ while (std::getline(infile, line))
+ {
+ ++line_number;
+ std::istringstream iss(line);
+ if (line_number & 1) {
+ if (!(iss >> space_name))
+ die("Can't parse space name from corrupted pages file at "
+ "line " ULINTPF,
+ line_number);
+ if (!(iss >> space_id))
+ die("Can't parse space id from corrupted pages file at line " ULINTPF,
+ line_number);
+ }
+ else
+ {
+ std::istringstream iss(line);
+ unsigned page_no;
+ while ((iss >> page_no))
+ add_page_no_lock(space_name.c_str(), {space_id, page_no}, false);
+ if (!iss.eof())
+ die("Corrupted pages file parse error on line number " ULINTPF,
+ line_number);
+ }
+ }
+}
+
+bool CorruptedPages::empty() const
+{
+ ut_a(!pthread_mutex_lock(&m_mutex));
+ bool result= !m_spaces.size();
+ ut_a(!pthread_mutex_unlock(&m_mutex));
+ return result;
+}
+
+static void xb_load_single_table_tablespace(const std::string &space_name,
+ bool set_size,
+ uint32_t defer_space_id=0);
+static void xb_data_files_close();
+static fil_space_t* fil_space_get_by_name(const char* name);
+
+void CorruptedPages::zero_out_free_pages()
+{
+ container_t non_free_pages;
+ byte *zero_page=
+ static_cast<byte *>(aligned_malloc(srv_page_size, srv_page_size));
+ memset(zero_page, 0, srv_page_size);
+
+ ut_a(!pthread_mutex_lock(&m_mutex));
+ for (container_t::const_iterator space_it= m_spaces.begin();
+ space_it != m_spaces.end(); ++space_it)
+ {
+ uint32_t space_id = space_it->first;
+ const std::string &space_name = space_it->second.space_name;
+ // There is no need to close tablespaces explixitly as they will be closed
+ // in innodb_shutdown().
+ xb_load_single_table_tablespace(space_name, false);
+ fil_space_t *space = fil_space_t::get(space_id);
+ if (!space)
+ die("Can't find space object for space name %s to check corrupted page",
+ space_name.c_str());
+ for (std::set<unsigned>::const_iterator page_it=
+ space_it->second.pages.begin();
+ page_it != space_it->second.pages.end(); ++page_it)
+ {
+ if (fseg_page_is_allocated(space, *page_it))
+ {
+ space_info_t &space_info = non_free_pages[space_id];
+ space_info.pages.insert(*page_it);
+ if (space_info.space_name.empty())
+ space_info.space_name = space_name;
+ msg("Error: corrupted page " UINT32PF
+ " of tablespace %s can not be fixed",
+ *page_it, space_name.c_str());
+ }
+ else
+ {
+ space->reacquire();
+ auto err= space
+ ->io(IORequest(IORequest::PUNCH_RANGE),
+ *page_it * srv_page_size, srv_page_size, zero_page)
+ .err;
+ if (err != DB_SUCCESS)
+ die("Can't zero out corrupted page " UINT32PF " of tablespace %s",
+ *page_it, space_name.c_str());
+ msg("Corrupted page " UINT32PF
+ " of tablespace %s was successfully fixed.",
+ *page_it, space_name.c_str());
+ }
+ }
+ space->flush<true>();
+ space->release();
+ }
+ m_spaces.swap(non_free_pages);
+ ut_a(!pthread_mutex_unlock(&m_mutex));
+ aligned_free(zero_page);
+}
+
+typedef void (*process_single_tablespace_func_t)(const char *dirname,
+ const char *filname,
+ bool is_remote,
+ bool skip_node_page0,
+ uint32_t defer_space_id);
+static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback);
+
+/* ======== Datafiles iterator ======== */
+struct datafiles_iter_t {
+ space_list_t::iterator space = fil_system.space_list.end();
+ fil_node_t *node = nullptr;
+ bool started = false;
+ std::mutex mutex;
+};
+
+/* ======== Datafiles iterator ======== */
+static
+fil_node_t *
+datafiles_iter_next(datafiles_iter_t *it)
+{
+ fil_node_t *new_node;
+
+ std::lock_guard<std::mutex> _(it->mutex);
+
+ if (it->node == NULL) {
+ if (it->started)
+ goto end;
+ it->started = TRUE;
+ } else {
+ it->node = UT_LIST_GET_NEXT(chain, it->node);
+ if (it->node != NULL)
+ goto end;
+ }
+
+ it->space = (it->space == fil_system.space_list.end()) ?
+ fil_system.space_list.begin() :
+ std::next(it->space);
+
+ while (it->space != fil_system.space_list.end() &&
+ (it->space->purpose != FIL_TYPE_TABLESPACE ||
+ UT_LIST_GET_LEN(it->space->chain) == 0))
+ ++it->space;
+ if (it->space == fil_system.space_list.end())
+ goto end;
+
+ it->node = UT_LIST_GET_FIRST(it->space->chain);
+
+end:
+ new_node = it->node;
+
+ return new_node;
+}
+
+#ifndef DBUG_OFF
+struct dbug_thread_param_t
+{
+ MYSQL *con;
+ const char *query;
+ int expect_err;
+ int expect_errno;
+};
+
+
+/* Thread procedure used in dbug_start_query_thread. */
+static void *dbug_execute_in_new_connection(void *arg)
+{
+ mysql_thread_init();
+ dbug_thread_param_t *par= static_cast<dbug_thread_param_t*>(arg);
+ int err = mysql_query(par->con, par->query);
+ int err_no = mysql_errno(par->con);
+ if(par->expect_err != err)
+ {
+ msg("FATAL: dbug_execute_in_new_connection : mysql_query '%s' returns %d, instead of expected %d",
+ par->query, err, par->expect_err);
+ _exit(1);
+ }
+ if (err && par->expect_errno && par->expect_errno != err_no)
+ {
+ msg("FATAL: dbug_execute_in_new_connection: mysql_query '%s' returns mysql_errno %d, instead of expected %d",
+ par->query, err_no, par->expect_errno);
+ _exit(1);
+ }
+ mysql_close(par->con);
+ mysql_thread_end();
+ delete par;
+ return nullptr;
+}
+
+static pthread_t dbug_alter_thread;
+
+/*
+Execute query from a new connection, in own thread.
+
+@param query - query to be executed
+@param wait_state - if not NULL, wait until query from new connection
+ reaches this state (value of column State in I_S.PROCESSLIST)
+@param expected_err - if 0, query is supposed to finish successfully,
+ otherwise query should return error.
+@param expected_errno - if not 0, and query finished with error,
+ expected mysql_errno()
+*/
+static void dbug_start_query_thread(
+ const char *query,
+ const char *wait_state,
+ int expected_err,
+ int expected_errno)
+
+{
+ dbug_thread_param_t *par = new dbug_thread_param_t;
+ par->query = query;
+ par->expect_err = expected_err;
+ par->expect_errno = expected_errno;
+ par->con = xb_mysql_connect();
+
+ mysql_thread_create(0, &dbug_alter_thread, nullptr,
+ dbug_execute_in_new_connection, par);
+
+ if (!wait_state)
+ return;
+
+ char q[256];
+ snprintf(q, sizeof(q),
+ "SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST where ID=%lu"
+ " AND Command='Query' AND State='%s'",
+ mysql_thread_id(par->con), wait_state);
+ for (;;) {
+ MYSQL_RES *result = xb_mysql_query(mysql_connection,q, true, true);
+ bool exists = mysql_fetch_row(result) != NULL;
+ mysql_free_result(result);
+ if (exists) {
+ goto end;
+ }
+ msg("Waiting for query '%s' on connection %lu to "
+ " reach state '%s'", query, mysql_thread_id(par->con),
+ wait_state);
+ my_sleep(1000);
+ }
+end:
+ msg("query '%s' on connection %lu reached state '%s'", query,
+ mysql_thread_id(par->con), wait_state);
+}
+#endif
+
+void mdl_lock_all()
+{
+ mdl_lock_init();
+ datafiles_iter_t it;
+
+ while (fil_node_t *node= datafiles_iter_next(&it))
+ {
+ const auto id= node->space->id;
+ if (const char *name= (fil_is_user_tablespace_id(id) &&
+ node->space->chain.start)
+ ? node->space->chain.start->name : nullptr)
+ if (check_if_skip_table(filename_to_spacename(name,
+ strlen(name)).c_str()))
+ continue;
+ mdl_lock_table(id);
+ }
+}
+
+
+// Convert non-null terminated filename to space name
+// Note that in 10.6 the filename may be an undo file name
+static std::string filename_to_spacename(const void *filename, size_t len)
+{
+ char f[FN_REFLEN];
+ char *p= 0, *table, *db;
+ DBUG_ASSERT(len < FN_REFLEN);
+
+ strmake(f, (const char*) filename, len);
+
+#ifdef _WIN32
+ for (size_t i = 0; i < len; i++)
+ {
+ if (f[i] == '\\')
+ f[i] = '/';
+ }
+#endif
+
+ /* Remove extension, if exists */
+ if (!(p= strrchr(f, '.')))
+ goto err;
+ *p= 0;
+
+ /* Find table name */
+ if (!(table= strrchr(f, '/')))
+ goto err;
+ *table = 0;
+
+ /* Find database name */
+ db= strrchr(f, '/');
+ *table = '/';
+ if (!db)
+ goto err;
+ {
+ std::string s(db+1);
+ return s;
+ }
+
+err:
+ /* Not a database/table. Return original (converted) name */
+ if (p)
+ *p= '.'; // Restore removed extension
+ std::string s(f);
+ return s;
+}
+
+/** Report an operation to create, delete, or rename a file during backup.
+@param[in] space_id tablespace identifier
+@param[in] type redo log file operation type
+@param[in] name file name (not NUL-terminated)
+@param[in] len length of name, in bytes
+@param[in] new_name new file name (NULL if not rename)
+@param[in] new_len length of new_name, in bytes (0 if NULL) */
+static void backup_file_op(uint32_t space_id, int type,
+ const byte* name, ulint len,
+ const byte* new_name, ulint new_len)
+{
+
+ ut_ad(name);
+ ut_ad(len);
+ ut_ad(!new_name == !new_len);
+ mysql_mutex_assert_owner(&recv_sys.mutex);
+
+ switch(type) {
+ case FILE_CREATE:
+ {
+ std::string space_name = filename_to_spacename(name, len);
+ ddl_tracker.id_to_name[space_id] = space_name;
+ ddl_tracker.delete_defer(space_id, space_name);
+ msg("DDL tracking : create %u \"%.*s\"", space_id, int(len), name);
+ }
+ break;
+ case FILE_MODIFY:
+ ddl_tracker.insert_defer_id(
+ space_id, filename_to_spacename(name, len));
+ break;
+ case FILE_RENAME:
+ {
+ std::string new_space_name = filename_to_spacename(
+ new_name, new_len);
+ std::string old_space_name = filename_to_spacename(
+ name, len);
+ ddl_tracker.id_to_name[space_id] = new_space_name;
+ ddl_tracker.rename_defer(space_id, old_space_name,
+ new_space_name);
+ msg("DDL tracking : rename %u \"%.*s\",\"%.*s\"",
+ space_id, int(len), name, int(new_len), new_name);
+ }
+ break;
+ case FILE_DELETE:
+ ddl_tracker.drops.insert(space_id);
+ ddl_tracker.delete_defer(
+ space_id, filename_to_spacename(name, len));
+ msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name);
+ break;
+ default:
+ ut_ad(0);
+ break;
+ }
+}
+
+
+/*
+ This callback is called if DDL operation is detected,
+ at the end of backup
+
+ Normally, DDL operations are blocked due to FTWRL,
+ but in rare cases of --no-lock, they are not.
+
+ We will abort backup in this case.
+*/
+static void backup_file_op_fail(uint32_t space_id, int type,
+ const byte* name, ulint len,
+ const byte* new_name, ulint new_len)
+{
+ bool fail = false;
+ switch(type) {
+ case FILE_CREATE:
+ msg("DDL tracking : create %u \"%.*s\"", space_id, int(len), name);
+ fail = !check_if_skip_table(
+ filename_to_spacename(name, len).c_str());
+ break;
+ case FILE_MODIFY:
+ break;
+ case FILE_RENAME:
+ msg("DDL tracking : rename %u \"%.*s\",\"%.*s\"",
+ space_id, int(len), name, int(new_len), new_name);
+ fail = !check_if_skip_table(
+ filename_to_spacename(name, len).c_str())
+ || !check_if_skip_table(
+ filename_to_spacename(new_name, new_len).c_str());
+ break;
+ case FILE_DELETE:
+ fail = !check_if_skip_table(
+ filename_to_spacename(name, len).c_str());
+ msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name);
+ break;
+ default:
+ ut_ad(0);
+ break;
+ }
+
+ if (fail) {
+ ut_a(opt_no_lock);
+ die("DDL operation detected in the late phase of backup."
+ "Backup is inconsistent. Remove --no-lock option to fix.");
+ }
+}
+
+static void backup_undo_trunc(uint32_t space_id)
+{
+ undo_trunc_ids.insert(space_id);
+}
+
+/* Function to store the space id of page0 INIT_PAGE
+@param space_id space id which has page0 init page */
+static void backup_first_page_op(space_id_t space_id)
+{
+ first_page_init_ids.insert(space_id);
+}
+
+/*
+ Retrieve default data directory, to be used with --copy-back.
+
+ On Windows, default datadir is ..\data, relative to the
+ directory where mariabackup.exe is located(usually "bin")
+
+ Elsewhere, the compiled-in constant MYSQL_DATADIR is used.
+*/
+static char *get_default_datadir() {
+ static char ddir[] = MYSQL_DATADIR;
+#ifdef _WIN32
+ static char buf[MAX_PATH];
+ DWORD size = (DWORD)sizeof(buf) - 1;
+ if (GetModuleFileName(NULL, buf, size) <= size)
+ {
+ char *p;
+ if ((p = strrchr(buf, '\\')))
+ {
+ *p = 0;
+ if ((p = strrchr(buf, '\\')))
+ {
+ strncpy(p + 1, "data", buf + MAX_PATH - p);
+ return buf;
+ }
+ }
+ }
+#endif
+ return ddir;
+}
+
+
+/* ======== Date copying thread context ======== */
+
+typedef struct {
+ datafiles_iter_t *it;
+ uint num;
+ uint *count;
+ pthread_mutex_t* count_mutex;
+ CorruptedPages *corrupted_pages;
+ Backup_datasinks *datasinks;
+} data_thread_ctxt_t;
+
+/* ======== for option and variables ======== */
+#include <../../client/client_priv.h>
+
+enum options_xtrabackup
+{
+ OPT_XTRA_TARGET_DIR= 1000, /* make sure it is larger
+ than OPT_MAX_CLIENT_OPTION */
+ OPT_XTRA_BACKUP,
+ OPT_XTRA_PREPARE,
+ OPT_XTRA_EXPORT,
+ OPT_XTRA_PRINT_PARAM,
+ OPT_XTRA_USE_MEMORY,
+ OPT_XTRA_THROTTLE,
+ OPT_XTRA_LOG_COPY_INTERVAL,
+ OPT_XTRA_INCREMENTAL,
+ OPT_XTRA_INCREMENTAL_BASEDIR,
+ OPT_XTRA_EXTRA_LSNDIR,
+ OPT_XTRA_INCREMENTAL_DIR,
+ OPT_XTRA_TABLES,
+ OPT_XTRA_TABLES_FILE,
+ OPT_XTRA_DATABASES,
+ OPT_XTRA_DATABASES_FILE,
+ OPT_XTRA_PARALLEL,
+ OPT_XTRA_EXTENDED_VALIDATION,
+ OPT_XTRA_ENCRYPTED_BACKUP,
+ OPT_XTRA_STREAM,
+ OPT_XTRA_COMPRESS,
+ OPT_XTRA_COMPRESS_THREADS,
+ OPT_XTRA_COMPRESS_CHUNK_SIZE,
+ OPT_LOG,
+ OPT_INNODB,
+ OPT_INNODB_DATA_FILE_PATH,
+ OPT_INNODB_DATA_HOME_DIR,
+ OPT_INNODB_ADAPTIVE_HASH_INDEX,
+ OPT_INNODB_DOUBLEWRITE,
+ OPT_INNODB_FILE_PER_TABLE,
+ OPT_INNODB_FLUSH_METHOD,
+ OPT_INNODB_LOG_GROUP_HOME_DIR,
+ OPT_INNODB_MAX_DIRTY_PAGES_PCT,
+ OPT_INNODB_MAX_PURGE_LAG,
+ OPT_INNODB_STATUS_FILE,
+ OPT_INNODB_AUTOEXTEND_INCREMENT,
+ OPT_INNODB_BUFFER_POOL_SIZE,
+ OPT_INNODB_COMMIT_CONCURRENCY,
+ OPT_INNODB_CONCURRENCY_TICKETS,
+ OPT_INNODB_FILE_IO_THREADS,
+ OPT_INNODB_IO_CAPACITY,
+ OPT_INNODB_READ_IO_THREADS,
+ OPT_INNODB_WRITE_IO_THREADS,
+ OPT_INNODB_USE_NATIVE_AIO,
+ OPT_INNODB_PAGE_SIZE,
+ OPT_INNODB_BUFFER_POOL_FILENAME,
+ OPT_INNODB_LOCK_WAIT_TIMEOUT,
+ OPT_INNODB_LOG_BUFFER_SIZE,
+#if defined __linux__ || defined _WIN32
+ OPT_INNODB_LOG_FILE_BUFFERING,
+#endif
+ OPT_INNODB_LOG_FILE_SIZE,
+ OPT_INNODB_OPEN_FILES,
+ OPT_XTRA_DEBUG_SYNC,
+ OPT_INNODB_CHECKSUM_ALGORITHM,
+ OPT_INNODB_UNDO_DIRECTORY,
+ OPT_INNODB_UNDO_TABLESPACES,
+ OPT_XTRA_INCREMENTAL_FORCE_SCAN,
+ OPT_DEFAULTS_GROUP,
+ OPT_CLOSE_FILES,
+ OPT_CORE_FILE,
+
+ OPT_COPY_BACK,
+ OPT_MOVE_BACK,
+ OPT_GALERA_INFO,
+ OPT_SLAVE_INFO,
+ OPT_NO_LOCK,
+ OPT_SAFE_SLAVE_BACKUP,
+ OPT_RSYNC,
+ OPT_FORCE_NON_EMPTY_DIRS,
+ OPT_NO_VERSION_CHECK,
+ OPT_NO_BACKUP_LOCKS,
+ OPT_DECOMPRESS,
+ OPT_INCREMENTAL_HISTORY_NAME,
+ OPT_INCREMENTAL_HISTORY_UUID,
+ OPT_REMOVE_ORIGINAL,
+ OPT_LOCK_WAIT_QUERY_TYPE,
+ OPT_KILL_LONG_QUERY_TYPE,
+ OPT_HISTORY,
+ OPT_KILL_LONG_QUERIES_TIMEOUT,
+ OPT_LOCK_WAIT_TIMEOUT,
+ OPT_LOCK_WAIT_THRESHOLD,
+ OPT_DEBUG_SLEEP_BEFORE_UNLOCK,
+ OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
+ OPT_BINLOG_INFO,
+ OPT_XB_SECURE_AUTH,
+
+ OPT_XTRA_TABLES_EXCLUDE,
+ OPT_XTRA_DATABASES_EXCLUDE,
+ OPT_PROTOCOL,
+ OPT_INNODB_COMPRESSION_LEVEL,
+ OPT_LOCK_DDL_PER_TABLE,
+ OPT_ROCKSDB_DATADIR,
+ OPT_BACKUP_ROCKSDB,
+ OPT_XTRA_CHECK_PRIVILEGES,
+ OPT_XTRA_MYSQLD_ARGS,
+ OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION,
+ OPT_INNODB_FORCE_RECOVERY,
+ OPT_ARIA_LOG_DIR_PATH
+};
+
+struct my_option xb_client_options[]= {
+ {"verbose", 'V', "display verbose output", (G_PTR *) &verbose,
+ (G_PTR *) &verbose, 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0},
+ {"version", 'v', "print version information",
+ (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL,
+ NO_ARG, 0, 0, 0, 0, 0, 0},
+ {"target-dir", OPT_XTRA_TARGET_DIR, "destination directory",
+ (G_PTR *) &xtrabackup_target_dir, (G_PTR *) &xtrabackup_target_dir, 0,
+ GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"backup", OPT_XTRA_BACKUP, "take backup to target-dir",
+ (G_PTR *) &xtrabackup_backup, (G_PTR *) &xtrabackup_backup, 0, GET_BOOL,
+ NO_ARG, 0, 0, 0, 0, 0, 0},
+ {"prepare", OPT_XTRA_PREPARE,
+ "prepare a backup for starting mysql server on the backup.",
+ (G_PTR *) &xtrabackup_prepare, (G_PTR *) &xtrabackup_prepare, 0, GET_BOOL,
+ NO_ARG, 0, 0, 0, 0, 0, 0},
+ {"export", OPT_XTRA_EXPORT,
+ "create files to import to another database when prepare.",
+ (G_PTR *) &xtrabackup_export, (G_PTR *) &xtrabackup_export, 0, GET_BOOL,
+ NO_ARG, 0, 0, 0, 0, 0, 0},
+ {"print-param", OPT_XTRA_PRINT_PARAM,
+ "print parameter of mysqld needed for copyback.",
+ (G_PTR *) &xtrabackup_print_param, (G_PTR *) &xtrabackup_print_param, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+ {"use-memory", OPT_XTRA_USE_MEMORY,
+ "The value is used in place of innodb_buffer_pool_size. "
+ "This option is only relevant when the --prepare option is specified.",
+ (G_PTR *) &xtrabackup_use_memory, (G_PTR *) &xtrabackup_use_memory, 0,
+ GET_LL, REQUIRED_ARG, 100 * 1024 * 1024L, 1024 * 1024L, LONGLONG_MAX, 0,
+ 1024 * 1024L, 0},
+ {"throttle", OPT_XTRA_THROTTLE,
+ "limit count of IO operations (pairs of read&write) per second to IOS "
+ "values (for '--backup')",
+ (G_PTR *) &xtrabackup_throttle, (G_PTR *) &xtrabackup_throttle, 0,
+ GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0},
+ {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
+ (G_PTR *) &log_ignored_opt, (G_PTR *) &log_ignored_opt, 0, GET_STR,
+ OPT_ARG, 0, 0, 0, 0, 0, 0},
+ {"log-copy-interval", OPT_XTRA_LOG_COPY_INTERVAL,
+ "time interval between checks done by log copying thread in milliseconds "
+ "(default is 1 second).",
+ (G_PTR *) &xtrabackup_log_copy_interval,
+ (G_PTR *) &xtrabackup_log_copy_interval, 0, GET_LONG, REQUIRED_ARG, 1000,
+ 0, LONG_MAX, 0, 1, 0},
+ {"extra-lsndir", OPT_XTRA_EXTRA_LSNDIR,
+ "(for --backup): save an extra copy of the xtrabackup_checkpoints file "
+ "in this directory.",
+ (G_PTR *) &xtrabackup_extra_lsndir, (G_PTR *) &xtrabackup_extra_lsndir, 0,
+ GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"incremental-lsn", OPT_XTRA_INCREMENTAL,
+ "(for --backup): copy only .ibd pages newer than specified LSN "
+ "'high:low'. ##ATTENTION##: If a wrong LSN value is specified, it is "
+ "impossible to diagnose this, causing the backup to be unusable. Be "
+ "careful!",
+ (G_PTR *) &xtrabackup_incremental, (G_PTR *) &xtrabackup_incremental, 0,
+ GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"incremental-basedir", OPT_XTRA_INCREMENTAL_BASEDIR,
+ "(for --backup): copy only .ibd pages newer than backup at specified "
+ "directory.",
+ (G_PTR *) &xtrabackup_incremental_basedir,
+ (G_PTR *) &xtrabackup_incremental_basedir, 0, GET_STR, REQUIRED_ARG, 0, 0,
+ 0, 0, 0, 0},
+ {"incremental-dir", OPT_XTRA_INCREMENTAL_DIR,
+ "(for --prepare): apply .delta files and logfile in the specified "
+ "directory.",
+ (G_PTR *) &xtrabackup_incremental_dir,
+ (G_PTR *) &xtrabackup_incremental_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
+ 0, 0, 0},
+ {"tables", OPT_XTRA_TABLES, "filtering by regexp for table names.",
+ (G_PTR *) &xtrabackup_tables, (G_PTR *) &xtrabackup_tables, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"tables_file", OPT_XTRA_TABLES_FILE,
+ "filtering by list of the exact database.table name in the file.",
+ (G_PTR *) &xtrabackup_tables_file, (G_PTR *) &xtrabackup_tables_file, 0,
+ GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.",
+ (G_PTR *) &xtrabackup_databases, (G_PTR *) &xtrabackup_databases, 0,
+ GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"databases_file", OPT_XTRA_DATABASES_FILE,
+ "filtering by list of databases in the file.",
+ (G_PTR *) &xtrabackup_databases_file,
+ (G_PTR *) &xtrabackup_databases_file, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
+ 0, 0, 0},
+ {"tables-exclude", OPT_XTRA_TABLES_EXCLUDE,
+ "filtering by regexp for table names. "
+ "Operates the same way as --tables, but matched names are excluded from "
+ "backup. "
+ "Note that this option has a higher priority than --tables.",
+ (G_PTR *) &xtrabackup_tables_exclude,
+ (G_PTR *) &xtrabackup_tables_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0, 0,
+ 0, 0, 0},
+ {"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE,
+ "Excluding databases based on name, "
+ "Operates the same way as --databases, but matched names are excluded "
+ "from backup. "
+ "Note that this option has a higher priority than --databases.",
+ (G_PTR *) &xtrabackup_databases_exclude,
+ (G_PTR *) &xtrabackup_databases_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0,
+ 0, 0, 0, 0},
+
+ {"stream", OPT_XTRA_STREAM,
+ "Stream all backup files to the standard output "
+ "in the specified format."
+ "Supported format is 'mbstream' or 'xbstream'.",
+ (G_PTR *) &xtrabackup_stream_str, (G_PTR *) &xtrabackup_stream_str, 0,
+ GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"compress", OPT_XTRA_COMPRESS,
+ "Compress individual backup files using the "
+ "specified compression algorithm. Currently the only supported algorithm "
+ "is 'quicklz'. It is also the default algorithm, i.e. the one used when "
+ "--compress is used without an argument.",
+ (G_PTR *) &xtrabackup_compress_alg, (G_PTR *) &xtrabackup_compress_alg, 0,
+ GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"compress-threads", OPT_XTRA_COMPRESS_THREADS,
+ "Number of threads for parallel data compression. The default value is "
+ "1.",
+ (G_PTR *) &xtrabackup_compress_threads,
+ (G_PTR *) &xtrabackup_compress_threads, 0, GET_UINT, REQUIRED_ARG, 1, 1,
+ UINT_MAX, 0, 0, 0},
+
+ {"compress-chunk-size", OPT_XTRA_COMPRESS_CHUNK_SIZE,
+ "Size of working buffer(s) for compression threads in bytes. The default "
+ "value is 64K.",
+ (G_PTR *) &xtrabackup_compress_chunk_size,
+ (G_PTR *) &xtrabackup_compress_chunk_size, 0, GET_ULL, REQUIRED_ARG,
+ (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},
+
+ {"incremental-force-scan", OPT_XTRA_INCREMENTAL_FORCE_SCAN,
+ "Perform a full-scan incremental backup even in the presence of changed "
+ "page bitmap data",
+ (G_PTR *) &xtrabackup_incremental_force_scan,
+ (G_PTR *) &xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG, 0, 0,
+ 0, 0, 0, 0},
+
+ {"close_files", OPT_CLOSE_FILES,
+ "do not keep files opened. Use at your own "
+ "risk.",
+ (G_PTR *) &xb_close_files, (G_PTR *) &xb_close_files, 0, GET_BOOL, NO_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"core-file", OPT_CORE_FILE, "Write core on fatal signals", 0, 0, 0,
+ GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"copy-back", OPT_COPY_BACK,
+ "Copy all the files in a previously made "
+ "backup from the backup directory to their original locations.",
+ (uchar *) &xtrabackup_copy_back, (uchar *) &xtrabackup_copy_back, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"move-back", OPT_MOVE_BACK,
+ "Move all the files in a previously made "
+ "backup from the backup directory to the actual datadir location. "
+ "Use with caution, as it removes backup files.",
+ (uchar *) &xtrabackup_move_back, (uchar *) &xtrabackup_move_back, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"galera-info", OPT_GALERA_INFO,
+ "This options creates the "
+ "xtrabackup_galera_info file which contains the local node state at "
+ "the time of the backup. Option should be used when performing the "
+ "backup of MariaDB Galera Cluster. Has no effect when backup locks "
+ "are used to create the backup.",
+ (uchar *) &opt_galera_info, (uchar *) &opt_galera_info, 0, GET_BOOL,
+ NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"slave-info", OPT_SLAVE_INFO,
+ "This option is useful when backing "
+ "up a replication slave server. It prints the binary log position "
+ "and name of the master server. It also writes this information to "
+ "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. "
+ "A new slave for this master can be set up by starting a slave server "
+ "on this backup and issuing a \"CHANGE MASTER\" command with the "
+ "binary log position saved in the \"xtrabackup_slave_info\" file.",
+ (uchar *) &opt_slave_info, (uchar *) &opt_slave_info, 0, GET_BOOL, NO_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"no-lock", OPT_NO_LOCK,
+ "Use this option to disable table lock "
+ "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your "
+ "tables are InnoDB and you DO NOT CARE about the binary log "
+ "position of the backup. This option shouldn't be used if there "
+ "are any DDL statements being executed or if any updates are "
+ "happening on non-InnoDB tables (this includes the system MyISAM "
+ "tables in the mysql database), otherwise it could lead to an "
+ "inconsistent backup. If you are considering to use --no-lock "
+ "because your backups are failing to acquire the lock, this could "
+ "be because of incoming replication events preventing the lock "
+ "from succeeding. Please try using --safe-slave-backup to "
+ "momentarily stop the replication slave thread, this may help "
+ "the backup to succeed and you then don't need to resort to "
+ "using this option.",
+ (uchar *) &opt_no_lock, (uchar *) &opt_no_lock, 0, GET_BOOL, NO_ARG, 0, 0,
+ 0, 0, 0, 0},
+
+ {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP,
+ "Stop slave SQL thread "
+ "and wait to start backup until Slave_open_temp_tables in "
+ "\"SHOW STATUS\" is zero. If there are no open temporary tables, "
+ "the backup will take place, otherwise the SQL thread will be "
+ "started and stopped until there are no open temporary tables. "
+ "The backup will fail if Slave_open_temp_tables does not become "
+ "zero after --safe-slave-backup-timeout seconds. The slave SQL "
+ "thread will be restarted when the backup finishes.",
+ (uchar *) &opt_safe_slave_backup, (uchar *) &opt_safe_slave_backup, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"rsync", OPT_RSYNC,
+ "Uses the rsync utility to optimize local file "
+ "transfers. When this option is specified, " XB_TOOL_NAME " uses rsync "
+ "to copy all non-InnoDB files instead of spawning a separate cp for "
+ "each file, which can be much faster for servers with a large number "
+ "of databases or tables. This option cannot be used together with "
+ "--stream.",
+ (uchar *) &opt_rsync, (uchar *) &opt_rsync, 0, GET_BOOL, NO_ARG, 0, 0, 0,
+ 0, 0, 0},
+
+ {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS,
+ "This "
+ "option, when specified, makes --copy-back or --move-back transfer "
+ "files to non-empty directories. Note that no existing files will be "
+ "overwritten. If --copy-back or --move-back has to copy a file from "
+ "the backup directory which already exists in the destination "
+ "directory, it will still fail with an error.",
+ (uchar *) &opt_force_non_empty_dirs, (uchar *) &opt_force_non_empty_dirs,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"no-version-check", OPT_NO_VERSION_CHECK,
+ "This option disables the "
+ "version check which is enabled by the --version-check option.",
+ (uchar *) &opt_noversioncheck, (uchar *) &opt_noversioncheck, 0, GET_BOOL,
+ NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"no-backup-locks", OPT_NO_BACKUP_LOCKS,
+ "This option controls if "
+ "backup locks should be used instead of FLUSH TABLES WITH READ LOCK "
+ "on the backup stage. The option has no effect when backup locks are "
+ "not supported by the server. This option is enabled by default, "
+ "disable with --no-backup-locks.",
+ (uchar *) &opt_no_backup_locks, (uchar *) &opt_no_backup_locks, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"decompress", OPT_DECOMPRESS,
+ "Decompresses all files with the .qp "
+ "extension in a backup previously made with the --compress option.",
+ (uchar *) &opt_decompress, (uchar *) &opt_decompress, 0, GET_BOOL, NO_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"user", 'u',
+ "This option specifies the MySQL username used "
+ "when connecting to the server, if that's not the current user. "
+ "The option accepts a string argument. See mysql --help for details.",
+ (uchar *) &opt_user, (uchar *) &opt_user, 0, GET_STR, REQUIRED_ARG, 0, 0,
+ 0, 0, 0, 0},
+
+ {"host", 'H',
+ "This option specifies the host to use when "
+ "connecting to the database server with TCP/IP. The option accepts "
+ "a string argument. See mysql --help for details.",
+ (uchar *) &opt_host, (uchar *) &opt_host, 0, GET_STR, REQUIRED_ARG, 0, 0,
+ 0, 0, 0, 0},
+
+ {"port", 'P',
+ "This option specifies the port to use when "
+ "connecting to the database server with TCP/IP. The option accepts "
+ "a string argument. See mysql --help for details.",
+ &opt_port, &opt_port, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"password", 'p',
+ "This option specifies the password to use "
+ "when connecting to the database. It accepts a string argument. "
+ "See mysql --help for details.",
+ 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"protocol", OPT_PROTOCOL,
+ "The protocol to use for connection (tcp, socket, pipe, memory).", 0, 0,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"socket", 'S',
+ "This option specifies the socket to use when "
+ "connecting to the local database server with a UNIX domain socket. "
+ "The option accepts a string argument. See mysql --help for details.",
+ (uchar *) &opt_socket, (uchar *) &opt_socket, 0, GET_STR, REQUIRED_ARG, 0,
+ 0, 0, 0, 0, 0},
+
+ {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME,
+ "This option specifies the name of the backup series stored in the "
+ XB_HISTORY_TABLE " history record to base an "
+ "incremental backup on. Xtrabackup will search the history table "
+ "looking for the most recent (highest innodb_to_lsn), successful "
+ "backup in the series and take the to_lsn value to use as the "
+ "starting lsn for the incremental backup. This will be mutually "
+ "exclusive with --incremental-history-uuid, --incremental-basedir "
+ "and --incremental-lsn. If no valid lsn can be found (no series by "
+ "that name, no successful backups by that name), an error will be returned."
+ " It is used with the --incremental option.",
+ (uchar *) &opt_incremental_history_name,
+ (uchar *) &opt_incremental_history_name, 0, GET_STR, REQUIRED_ARG, 0, 0,
+ 0, 0, 0, 0},
+
+ {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID,
+ "This option specifies the UUID of the specific history record "
+ "stored in the " XB_HISTORY_TABLE " table to base an "
+ "incremental backup on. --incremental-history-name, "
+ "--incremental-basedir and --incremental-lsn. If no valid lsn can be "
+ "found (no success record with that uuid), an error will be returned."
+ " It is used with the --incremental option.",
+ (uchar *) &opt_incremental_history_uuid,
+ (uchar *) &opt_incremental_history_uuid, 0, GET_STR, REQUIRED_ARG, 0, 0,
+ 0, 0, 0, 0},
+
+ {"remove-original", OPT_REMOVE_ORIGINAL,
+ "Remove .qp files after decompression.", (uchar *) &opt_remove_original,
+ (uchar *) &opt_remove_original, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE,
+ "This option specifies which types of queries are allowed to complete "
+ "before " XB_TOOL_NAME " will issue the global lock. Default is all.",
+ (uchar *) &opt_lock_wait_query_type, (uchar *) &opt_lock_wait_query_type,
+ &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0,
+ 0},
+
+ {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE,
+ "This option specifies which types of queries should be killed to "
+ "unblock the global lock. Default is \"all\".",
+ (uchar *) &opt_kill_long_query_type, (uchar *) &opt_kill_long_query_type,
+ &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0,
+ 0, 0},
+
+ {"history", OPT_HISTORY,
+ "This option enables the tracking of backup history in the "
+ XB_HISTORY_TABLE " table. An optional history "
+ "series name may be specified that will be placed with the history "
+ "record for the current backup being taken.",
+ NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT,
+ "This option specifies the number of seconds " XB_TOOL_NAME " waits "
+ "between starting FLUSH TABLES WITH READ LOCK and killing those "
+ "queries that block it. Default is 0 seconds, which means "
+ XB_TOOL_NAME " will not attempt to kill any queries.",
+ (uchar *) &opt_kill_long_queries_timeout,
+ (uchar *) &opt_kill_long_queries_timeout, 0, GET_UINT, REQUIRED_ARG, 0, 0,
+ 0, 0, 0, 0},
+
+ {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT,
+ "This option specifies time in seconds that " XB_TOOL_NAME " should wait "
+ "for queries that would block FTWRL before running it. If there are "
+ "still such queries when the timeout expires, " XB_TOOL_NAME " terminates "
+ "with an error. Default is 0, in which case " XB_TOOL_NAME " does not "
+ "wait for queries to complete and starts FTWRL immediately.",
+ (uchar *) &opt_lock_wait_timeout, (uchar *) &opt_lock_wait_timeout, 0,
+ GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD,
+ "This option specifies the query run time threshold which is used by "
+ XB_TOOL_NAME " to detect long-running queries with a non-zero value "
+ "of --ftwrl-wait-timeout. FTWRL is not started until such "
+ "long-running queries exist. This option has no effect if "
+ "--ftwrl-wait-timeout is 0. Default value is 60 seconds.",
+ (uchar *) &opt_lock_wait_threshold, (uchar *) &opt_lock_wait_threshold, 0,
+ GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0},
+
+
+ {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
+ "How many seconds --safe-slave-backup should wait for "
+ "Slave_open_temp_tables to become zero. (default 300)",
+ (uchar *) &opt_safe_slave_backup_timeout,
+ (uchar *) &opt_safe_slave_backup_timeout, 0, GET_UINT, REQUIRED_ARG, 300,
+ 0, 0, 0, 0, 0},
+
+ {"binlog-info", OPT_BINLOG_INFO,
+ "This option controls how backup should retrieve server's binary log "
+ "coordinates corresponding to the backup. Possible values are OFF, ON, "
+ "LOCKLESS and AUTO.",
+ &opt_binlog_info, &opt_binlog_info, &binlog_info_typelib, GET_ENUM,
+ OPT_ARG, BINLOG_INFO_AUTO, 0, 0, 0, 0, 0},
+
+ {"secure-auth", OPT_XB_SECURE_AUTH,
+ "Refuse client connecting to server if it"
+ " uses old (pre-4.1.1) protocol.",
+ &opt_secure_auth, &opt_secure_auth, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0,
+ 0},
+
+ {"log-innodb-page-corruption", OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION,
+ "Continue backup if innodb corrupted pages are found. The pages are "
+ "logged in " MB_CORRUPTED_PAGES_FILE
+ " and backup is finished with error. "
+ "--prepare will try to fix corrupted pages. If " MB_CORRUPTED_PAGES_FILE
+ " exists after --prepare in base backup directory, backup still contains "
+ "corrupted pages and can not be considered as consistent.",
+ &opt_log_innodb_page_corruption, &opt_log_innodb_page_corruption, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+#define MYSQL_CLIENT
+#include "sslopt-longopts.h"
+#undef MYSQL_CLIENT
+ {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};
+
+uint xb_client_options_count = array_elements(xb_client_options);
+
+#ifndef DBUG_OFF
+/** Parameters to DBUG */
+static const char *dbug_option;
+#endif
+
+#ifdef HAVE_URING
+extern const char *io_uring_may_be_unsafe;
+bool innodb_use_native_aio_default();
+#endif
+
+struct my_option xb_server_options[] =
+{
+ {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home,
+ (G_PTR*) &mysql_data_home, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"tmpdir", 't',
+ "Path for temporary files. Several paths may be specified, separated by a "
+#if defined(_WIN32)
+ "semicolon (;)"
+#else
+ "colon (:)"
+#endif
+ ", in this case they are used in a round-robin fashion.",
+ (G_PTR*) &opt_mysql_tmpdir,
+ (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"parallel", OPT_XTRA_PARALLEL,
+ "Number of threads to use for parallel datafiles transfer. "
+ "The default value is 1.",
+ (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT,
+ REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
+
+ {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION,
+ "Enable extended validation for Innodb data pages during backup phase. "
+ "Will slow down backup considerably, in case encryption is used. "
+ "May fail if tables are created during the backup.",
+ (G_PTR*)&opt_extended_validation,
+ (G_PTR*)&opt_extended_validation,
+ 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0},
+
+ {"encrypted_backup", OPT_XTRA_ENCRYPTED_BACKUP,
+ "In --backup, assume that nonzero key_version implies that the page"
+ " is encrypted. Use --backup --skip-encrypted-backup to allow"
+ " copying unencrypted that were originally created before MySQL 5.1.48.",
+ (G_PTR*)&opt_encrypted_backup,
+ (G_PTR*)&opt_encrypted_backup,
+ 0, GET_BOOL, NO_ARG, TRUE, 0, 0, 0, 0, 0},
+
+ {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
+ (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0,
+ GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"log_bin", OPT_LOG, "Base name for the log sequence",
+ &opt_log_bin, &opt_log_bin, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility",
+ (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0,
+ GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
+#ifdef BTR_CUR_HASH_ADAPT
+ {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX,
+ "Enable InnoDB adaptive hash index (disabled by default).",
+ &btr_search_enabled,
+ &btr_search_enabled,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+#endif /* BTR_CUR_HASH_ADAPT */
+ {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT,
+ "Data file autoextend increment in megabytes",
+ (G_PTR*) &sys_tablespace_auto_extend_increment,
+ (G_PTR*) &sys_tablespace_auto_extend_increment,
+ 0, GET_UINT, REQUIRED_ARG, 8, 1, 1000, 0, 1, 0},
+ {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH,
+ "Path to individual files and their sizes.", &innobase_data_file_path,
+ &innobase_data_file_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR,
+ "The common part for InnoDB table spaces.", &innobase_data_home_dir,
+ &innobase_data_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE,
+ "Enable InnoDB doublewrite buffer during --prepare.",
+ (G_PTR*) &srv_use_doublewrite_buf,
+ (G_PTR*) &srv_use_doublewrite_buf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+ {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY,
+ "Number of IOPs the server can do. Tunes the background IO rate",
+ (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity,
+ 0, GET_ULONG, OPT_ARG, 200, 100, ~0UL, 0, 0, 0},
+ {"innodb_file_io_threads", OPT_INNODB_FILE_IO_THREADS,
+ "Number of file I/O threads in InnoDB.", (G_PTR*) &innobase_file_io_threads,
+ (G_PTR*) &innobase_file_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 4, 64, 0,
+ 1, 0},
+ {"innodb_read_io_threads", OPT_INNODB_READ_IO_THREADS,
+ "Number of background read I/O threads in InnoDB.", (G_PTR*) &innobase_read_io_threads,
+ (G_PTR*) &innobase_read_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
+ 1, 0},
+ {"innodb_write_io_threads", OPT_INNODB_WRITE_IO_THREADS,
+ "Number of background write I/O threads in InnoDB.", (G_PTR*) &innobase_write_io_threads,
+ (G_PTR*) &innobase_write_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
+ 1, 0},
+ {"innodb_file_per_table", OPT_INNODB_FILE_PER_TABLE,
+ "Stores each InnoDB table to an .ibd file in the database dir.",
+ (G_PTR*) &srv_file_per_table,
+ (G_PTR*) &srv_file_per_table, 0, GET_BOOL, NO_ARG,
+ FALSE, 0, 0, 0, 0, 0},
+
+ {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD,
+ "With which method to flush data.",
+ &srv_file_flush_method, &srv_file_flush_method,
+ &innodb_flush_method_typelib, GET_ENUM, REQUIRED_ARG,
+ IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_O_DIRECT), 0, 0, 0, 0, 0},
+
+ {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE,
+ "Redo log buffer size in bytes.",
+ (G_PTR*) &log_sys.buf_size, (G_PTR*) &log_sys.buf_size, 0,
+ IF_WIN(GET_ULL,GET_ULONG), REQUIRED_ARG, 2U << 20,
+ 2U << 20, SIZE_T_MAX, 0, 4096, 0},
+#if defined __linux__ || defined _WIN32
+ {"innodb_log_file_buffering", OPT_INNODB_LOG_FILE_BUFFERING,
+ "Whether the file system cache for ib_logfile0 is enabled during --backup",
+ (G_PTR*) &log_sys.log_buffered,
+ (G_PTR*) &log_sys.log_buffered, 0, GET_BOOL, NO_ARG,
+ TRUE, 0, 0, 0, 0, 0},
+#endif
+ {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE,
+ "Ignored for mysqld option compatibility",
+ (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0,
+ GET_ULL, REQUIRED_ARG, 96 << 20, 4 << 20,
+ std::numeric_limits<ulonglong>::max(), 0, 4096, 0},
+ {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR,
+ "Path to InnoDB log files.", &srv_log_group_home_dir,
+ &srv_log_group_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT,
+ "Percentage of dirty pages allowed in bufferpool.", (G_PTR*) &srv_max_buf_pool_modified_pct,
+ (G_PTR*) &srv_max_buf_pool_modified_pct, 0, GET_ULONG, REQUIRED_ARG, 90, 0, 100, 0, 0, 0},
+ {"innodb_use_native_aio", OPT_INNODB_USE_NATIVE_AIO,
+ "Use native AIO if supported on this platform.",
+ (G_PTR*) &srv_use_native_aio,
+ (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG,
+#ifdef HAVE_URING
+ innodb_use_native_aio_default(),
+#else
+ TRUE,
+#endif
+ 0, 0, 0, 0, 0},
+ {"innodb_page_size", OPT_INNODB_PAGE_SIZE,
+ "The universal page size of the database.",
+ (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0,
+ /* Use GET_LL to support numeric suffixes in 5.6 */
+ GET_LL, REQUIRED_ARG,
+ (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0},
+ {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME,
+ "Ignored for mysqld option compatibility",
+ (G_PTR*) &innobase_buffer_pool_filename,
+ (G_PTR*) &innobase_buffer_pool_filename,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+#ifndef DBUG_OFF /* unfortunately "debug" collides with existing options */
+ {"dbug", '#', "Built in DBUG debugger.",
+ &dbug_option, &dbug_option, 0, GET_STR, OPT_ARG,
+ 0, 0, 0, 0, 0, 0},
+#endif
+
+ {"innodb_checksum_algorithm", OPT_INNODB_CHECKSUM_ALGORITHM,
+ "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, "
+ "FULL_CRC32, STRICT_FULL_CRC32]", &srv_checksum_algorithm,
+ &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM,
+ REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_CRC32, 0, 0, 0, 0, 0},
+
+ {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY,
+ "Directory where undo tablespace files live, this path can be absolute.",
+ &srv_undo_dir, &srv_undo_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0,
+ 0},
+
+ {"innodb_undo_tablespaces", OPT_INNODB_UNDO_TABLESPACES,
+ "Number of undo tablespaces to use.",
+ (G_PTR*)&srv_undo_tablespaces, (G_PTR*)&srv_undo_tablespaces,
+ 0, GET_UINT, REQUIRED_ARG, 0, 0, 126, 0, 1, 0},
+
+ {"innodb_compression_level", OPT_INNODB_COMPRESSION_LEVEL,
+ "Compression level used for zlib compression.",
+ (G_PTR*)&page_zip_level, (G_PTR*)&page_zip_level,
+ 0, GET_UINT, REQUIRED_ARG, 6, 0, 9, 0, 0, 0},
+
+ {"defaults_group", OPT_DEFAULTS_GROUP, "defaults group in config file (default \"mysqld\").",
+ (G_PTR*) &defaults_group, (G_PTR*) &defaults_group,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"plugin-dir", OPT_PLUGIN_DIR,
+ "Server plugin directory. Used to load plugins during 'prepare' phase."
+ "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)",
+ &xb_plugin_dir, &xb_plugin_dir,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+
+ {"aria_log_dir_path", OPT_ARIA_LOG_DIR_PATH,
+ "Path to individual files and their sizes.",
+ &aria_log_dir_path, &aria_log_dir_path,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file "
+ "descriptors to reserve with setrlimit().",
+ (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG,
+ REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0},
+
+ {"lock-ddl-per-table", OPT_LOCK_DDL_PER_TABLE, "Lock DDL for each table "
+ "before backup starts to copy it and until the backup is completed.",
+ (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory."
+ "This option is only used with --copy-back or --move-back option",
+ &xb_rocksdb_datadir, &xb_rocksdb_datadir,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+
+ {"rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed."
+ "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data",
+ &xb_backup_rocksdb, &xb_backup_rocksdb,
+ 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
+
+ {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user "
+ "privileges fro the backup user",
+ &opt_check_privileges, &opt_check_privileges,
+ 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
+
+ {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY,
+ "(for --prepare): Crash recovery mode (ignores "
+ "page corruption; for emergencies only).",
+ (G_PTR*)&srv_force_recovery,
+ (G_PTR*)&srv_force_recovery,
+ 0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0},
+
+ {"mysqld-args", OPT_XTRA_MYSQLD_ARGS,
+ "All arguments that follow this argument are considered as server "
+ "options, and if some of them are not supported by mariabackup, they "
+ "will be ignored.",
+ (G_PTR *) &xtrabackup_mysqld_args, (G_PTR *) &xtrabackup_mysqld_args, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"help", '?',
+ "Display this help and exit.",
+ (G_PTR *) &xtrabackup_help, (G_PTR *) &xtrabackup_help, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
+};
+
+uint xb_server_options_count = array_elements(xb_server_options);
+
+
+static std::set<std::string> tables_for_export;
+
+static void append_export_table(const char *dbname, const char *tablename,
+ bool is_remote, bool skip_node_page0,
+ uint32_t defer_space_id)
+{
+ if(dbname && tablename && !is_remote)
+ {
+ char buf[3*FN_REFLEN];
+ snprintf(buf,sizeof(buf),"%s/%s",dbname, tablename);
+ // trim .ibd
+ char *p=strrchr(buf, '.');
+ if (p) *p=0;
+
+ std::string name=ut_get_name(0, buf);
+ /* Strip partition name comment from table name, if any */
+ if (ends_with(name.c_str(), "*/"))
+ {
+ size_t pos= name.rfind("/*");
+ if (pos != std::string::npos)
+ name.resize(pos);
+ }
+ tables_for_export.insert(name);
+ }
+}
+
+
+#define BOOTSTRAP_FILENAME "mariabackup_prepare_for_export.sql"
+
+static int create_bootstrap_file()
+{
+ FILE *f= fopen(BOOTSTRAP_FILENAME,"wb");
+ if(!f)
+ return -1;
+
+ fputs("SET NAMES UTF8;\n",f);
+ enumerate_ibd_files(append_export_table);
+ for (std::set<std::string>::iterator it = tables_for_export.begin();
+ it != tables_for_export.end(); it++)
+ {
+ const char *tab = it->c_str();
+ fprintf(f,
+ "BEGIN NOT ATOMIC "
+ "DECLARE CONTINUE HANDLER FOR NOT FOUND,SQLEXCEPTION BEGIN END;"
+ "FLUSH TABLES %s FOR EXPORT;"
+ "END;\n"
+ "UNLOCK TABLES;\n",
+ tab);
+ }
+ fclose(f);
+ return 0;
+}
+
+static int prepare_export()
+{
+ int err= -1;
+
+ char cmdline[2*FN_REFLEN];
+ FILE *outf;
+
+ if (create_bootstrap_file())
+ return -1;
+
+ // Process defaults-file , it can have some --lc-language stuff,
+ // which is* unfortunately* still necessary to get mysqld up
+ if (strncmp(orig_argv1,"--defaults-file=", 16) == 0)
+ {
+ snprintf(cmdline, sizeof cmdline,
+ IF_WIN("\"","") "\"%s\" --mysqld \"%s\""
+ " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
+ " --innodb --innodb-fast-shutdown=0 --loose-partition"
+ " --innodb-buffer-pool-size=%llu"
+ " --console --skip-log-error --skip-log-bin --bootstrap %s< "
+ BOOTSTRAP_FILENAME IF_WIN("\"",""),
+ mariabackup_exe,
+ orig_argv1, (my_defaults_group_suffix?my_defaults_group_suffix:""),
+ xtrabackup_use_memory,
+ (srv_force_recovery ? "--innodb-force-recovery=1 " : ""));
+ }
+ else
+ {
+ snprintf(cmdline, sizeof cmdline,
+ IF_WIN("\"","") "\"%s\" --mysqld"
+ " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
+ " --innodb --innodb-fast-shutdown=0 --loose-partition"
+ " --innodb-buffer-pool-size=%llu"
+ " --console --log-error= --skip-log-bin --bootstrap %s< "
+ BOOTSTRAP_FILENAME IF_WIN("\"",""),
+ mariabackup_exe,
+ (my_defaults_group_suffix?my_defaults_group_suffix:""),
+ xtrabackup_use_memory,
+ (srv_force_recovery ? "--innodb-force-recovery=1 " : ""));
+ }
+
+ msg("Prepare export : executing %s\n", cmdline);
+ fflush(stderr);
+
+ outf= popen(cmdline,"r");
+ if (!outf)
+ goto end;
+
+ char outline[FN_REFLEN];
+ while (fgets(outline, FN_REFLEN - 1, outf))
+ fprintf(stderr,"%s",outline);
+
+ err = pclose(outf);
+end:
+ unlink(BOOTSTRAP_FILENAME);
+ return err;
+}
+
+static const char *xb_client_default_groups[]= {
+ "client", "client-server", "client-mariadb", 0, 0, 0};
+
+static const char *backup_default_groups[]= {
+ "xtrabackup", "mariabackup", "mariadb-backup", 0, 0, 0};
+
+static void print_version(void)
+{
+ fprintf(stderr, "%s based on MariaDB server %s %s (%s)\n",
+ my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE);
+}
+
+static void concatenate_default_groups(std::vector<const char*> &backup_load_groups, const char **default_groups)
+{
+ for ( ; *default_groups ; default_groups++)
+ backup_load_groups.push_back(*default_groups);
+}
+
+static void usage(void)
+{
+ puts("Open source backup tool for InnoDB and XtraDB\n\
+\n\
+Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\
+Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\
+\n\
+This program is free software; you can redistribute it and/or\n\
+modify it under the terms of the GNU General Public License\n\
+as published by the Free Software Foundation version 2\n\
+of the License.\n\
+\n\
+This program is distributed in the hope that it will be useful,\n\
+but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\
+GNU General Public License for more details.\n\
+\n\
+You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n");
+
+ printf("Usage: %s [--defaults-file=#] [--backup | --prepare | --copy-back | --move-back] [OPTIONS]\n",my_progname);
+ std::vector<const char*> backup_load_default_groups;
+ concatenate_default_groups(backup_load_default_groups, backup_default_groups);
+ concatenate_default_groups(backup_load_default_groups, load_default_groups);
+ backup_load_default_groups.push_back(nullptr);
+ print_defaults("my", &backup_load_default_groups[0]);
+ my_print_help(xb_client_options);
+ my_print_help(xb_server_options);
+ my_print_variables(xb_server_options);
+ my_print_variables(xb_client_options);
+}
+
+#define ADD_PRINT_PARAM_OPT(value) \
+ { \
+ print_param_str << opt->name << "=" << value << "\n"; \
+ param_set.insert(opt->name); \
+ }
+
+/************************************************************************
+Check if parameter is set in defaults file or via command line argument
+@return true if parameter is set. */
+bool
+check_if_param_set(const char *param)
+{
+ return param_set.find(param) != param_set.end();
+}
+
+my_bool
+xb_get_one_option(const struct my_option *opt,
+ const char *argument, const char *)
+{
+ switch(opt->id) {
+ case 'h':
+ strmake(mysql_real_data_home,argument, FN_REFLEN - 1);
+ mysql_data_home= mysql_real_data_home;
+
+ ADD_PRINT_PARAM_OPT(mysql_real_data_home);
+ break;
+
+ case 't':
+
+ ADD_PRINT_PARAM_OPT(opt_mysql_tmpdir);
+ break;
+
+ case OPT_INNODB_DATA_HOME_DIR:
+
+ ADD_PRINT_PARAM_OPT(innobase_data_home_dir);
+ break;
+
+ case OPT_INNODB_DATA_FILE_PATH:
+
+ ADD_PRINT_PARAM_OPT(innobase_data_file_path);
+ break;
+
+ case OPT_INNODB_LOG_GROUP_HOME_DIR:
+
+ ADD_PRINT_PARAM_OPT(srv_log_group_home_dir);
+ break;
+
+ case OPT_INNODB_FLUSH_METHOD:
+#ifdef _WIN32
+ /* From: storage/innobase/handler/ha_innodb.cc:innodb_init_params */
+ switch (srv_file_flush_method) {
+ case SRV_ALL_O_DIRECT_FSYNC + 1 /* "async_unbuffered"="unbuffered" */:
+ srv_file_flush_method= SRV_ALL_O_DIRECT_FSYNC;
+ break;
+ case SRV_ALL_O_DIRECT_FSYNC + 2 /* "normal"="fsync" */:
+ srv_file_flush_method= SRV_FSYNC;
+ break;
+ }
+#endif
+ ut_a(srv_file_flush_method
+ <= IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_O_DIRECT_NO_FSYNC));
+ ADD_PRINT_PARAM_OPT(innodb_flush_method_names[srv_file_flush_method]);
+ break;
+
+ case OPT_INNODB_PAGE_SIZE:
+
+ ADD_PRINT_PARAM_OPT(innobase_page_size);
+ break;
+
+ case OPT_INNODB_UNDO_DIRECTORY:
+
+ ADD_PRINT_PARAM_OPT(srv_undo_dir);
+ break;
+
+ case OPT_INNODB_UNDO_TABLESPACES:
+
+ ADD_PRINT_PARAM_OPT(srv_undo_tablespaces);
+ break;
+
+ case OPT_INNODB_CHECKSUM_ALGORITHM:
+
+ ut_a(srv_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32);
+
+ ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]);
+ break;
+
+ case OPT_INNODB_COMPRESSION_LEVEL:
+ ADD_PRINT_PARAM_OPT(page_zip_level);
+ break;
+
+ case OPT_INNODB_BUFFER_POOL_FILENAME:
+
+ ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename);
+ break;
+
+ case OPT_INNODB_FORCE_RECOVERY:
+
+ if (srv_force_recovery) {
+ ADD_PRINT_PARAM_OPT(srv_force_recovery);
+ }
+ break;
+
+ case OPT_ARIA_LOG_DIR_PATH:
+ ADD_PRINT_PARAM_OPT(aria_log_dir_path);
+ break;
+
+ case OPT_XTRA_TARGET_DIR:
+ strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1);
+ xtrabackup_target_dir= xtrabackup_real_target_dir;
+ break;
+ case OPT_XTRA_STREAM:
+ if (!strcasecmp(argument, "mbstream") ||
+ !strcasecmp(argument, "xbstream"))
+ xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM;
+ else
+ {
+ msg("Invalid --stream argument: %s", argument);
+ return 1;
+ }
+ xtrabackup_stream = TRUE;
+ break;
+ case OPT_XTRA_COMPRESS:
+ if (argument == NULL)
+ xtrabackup_compress_alg = "quicklz";
+ else if (strcasecmp(argument, "quicklz"))
+ {
+ msg("Invalid --compress argument: %s", argument);
+ return 1;
+ }
+ xtrabackup_compress = TRUE;
+ break;
+ case OPT_DECOMPRESS:
+ opt_decompress = TRUE;
+ xtrabackup_decrypt_decompress = true;
+ break;
+ case (int) OPT_CORE_FILE:
+ test_flags |= TEST_CORE_ON_SIGNAL;
+ break;
+ case OPT_HISTORY:
+ if (argument) {
+ opt_history = argument;
+ } else {
+ opt_history = "";
+ }
+ break;
+ case 'p':
+ opt_password = argument;
+ break;
+ case OPT_PROTOCOL:
+ if (argument)
+ {
+ if ((opt_protocol= find_type_with_warning(argument, &sql_protocol_typelib,
+ opt->name)) <= 0)
+ {
+ sf_leaking_memory= 1; /* no memory leak reports here */
+ exit(1);
+ }
+ }
+ break;
+#define MYSQL_CLIENT
+#include "sslopt-case.h"
+#undef MYSQL_CLIENT
+
+ case '?':
+ usage();
+ exit(EXIT_SUCCESS);
+ break;
+ case 'v':
+ print_version();
+ exit(EXIT_SUCCESS);
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static bool innodb_init_param()
+{
+ srv_is_being_started = TRUE;
+ /* === some variables from mysqld === */
+ memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list));
+
+ if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) {
+ msg("init_tmpdir() failed");
+ return true;
+ }
+ xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list);
+ /* dummy for initialize all_charsets[] */
+ get_charset_name(0);
+
+ srv_page_size = 0;
+ srv_page_size_shift = 0;
+#ifdef BTR_CUR_HASH_ADAPT
+ btr_ahi_parts = 1;
+#endif /* BTR_CUR_HASH_ADAPT */
+
+ if (innobase_page_size != (1LL << 14)) {
+ size_t n_shift = get_bit_shift(size_t(innobase_page_size));
+
+ if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) {
+ srv_page_size_shift = uint32_t(n_shift);
+ srv_page_size = 1U << n_shift;
+ msg("InnoDB: The universal page size of the "
+ "database is set to %lu.", srv_page_size);
+ } else {
+ msg("invalid value of "
+ "innobase_page_size: %lld", innobase_page_size);
+ goto error;
+ }
+ } else {
+ srv_page_size_shift = 14;
+ srv_page_size = 1U << 14;
+ }
+
+ /* Check that values don't overflow on 32-bit systems. */
+ if (sizeof(ulint) == 4) {
+ if (xtrabackup_use_memory > UINT_MAX32) {
+ msg("mariabackup: use-memory can't be over 4GB"
+ " on 32-bit systems");
+ }
+ }
+
+ static char default_path[2] = { FN_CURLIB, 0 };
+ fil_path_to_mysql_datadir = default_path;
+
+ /* Set InnoDB initialization parameters according to the values
+ read from MySQL .cnf file */
+
+ if (xtrabackup_backup) {
+ msg("mariabackup: using the following InnoDB configuration:");
+ } else {
+ msg("mariabackup: using the following InnoDB configuration "
+ "for recovery:");
+ }
+
+ /*--------------- Data files -------------------------*/
+
+ /* The default dir for data files is the datadir of MySQL */
+
+ srv_data_home = (xtrabackup_backup && innobase_data_home_dir
+ ? innobase_data_home_dir : default_path);
+ msg("innodb_data_home_dir = %s", srv_data_home);
+
+ /* Set default InnoDB data file size to 10 MB and let it be
+ auto-extending. Thus users can use InnoDB in >= 4.0 without having
+ to specify any startup options. */
+
+ if (!innobase_data_file_path) {
+ innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
+ }
+ msg("innodb_data_file_path = %s",
+ innobase_data_file_path);
+
+ srv_sys_space.set_space_id(TRX_SYS_SPACE);
+ srv_sys_space.set_path(srv_data_home);
+ switch (srv_checksum_algorithm) {
+ case SRV_CHECKSUM_ALGORITHM_FULL_CRC32:
+ case SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32:
+ srv_sys_space.set_flags(FSP_FLAGS_FCRC32_MASK_MARKER
+ | FSP_FLAGS_FCRC32_PAGE_SSIZE());
+ break;
+ default:
+ srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE());
+ }
+
+ if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
+ goto error;
+ }
+
+ srv_sys_space.normalize_size();
+ srv_lock_table_size = 5 * (srv_buf_pool_size >> srv_page_size_shift);
+
+ /* -------------- Log files ---------------------------*/
+
+ /* The default dir for log files is the datadir of MySQL */
+
+ if (!(xtrabackup_backup && srv_log_group_home_dir)) {
+ srv_log_group_home_dir = default_path;
+ }
+ if (xtrabackup_prepare && xtrabackup_incremental_dir) {
+ srv_log_group_home_dir = xtrabackup_incremental_dir;
+ }
+ msg("innodb_log_group_home_dir = %s",
+ srv_log_group_home_dir);
+
+ if (strchr(srv_log_group_home_dir, ';')) {
+ msg("syntax error in innodb_log_group_home_dir, ");
+ goto error;
+ }
+
+ srv_adaptive_flushing = FALSE;
+
+ /* We set srv_pool_size here in units of 1 kB. InnoDB internally
+ changes the value so that it becomes the number of database pages. */
+
+ srv_buf_pool_size = (ulint) xtrabackup_use_memory;
+ srv_buf_pool_chunk_unit = srv_buf_pool_size;
+
+ srv_n_read_io_threads = (uint) innobase_read_io_threads;
+ srv_n_write_io_threads = (uint) innobase_write_io_threads;
+
+ srv_max_n_open_files = ULINT_UNDEFINED - 5;
+
+ srv_print_verbose_log = verbose ? 2 : 1;
+
+ /* Store the default charset-collation number of this MySQL
+ installation */
+
+ /* We cannot treat characterset here for now!! */
+ data_mysql_default_charset_coll = (ulint)default_charset_info->number;
+
+ ut_ad(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number);
+
+#ifdef _WIN32
+ srv_use_native_aio = TRUE;
+
+#elif defined(LINUX_NATIVE_AIO)
+
+ if (srv_use_native_aio) {
+ msg("InnoDB: Using Linux native AIO");
+ }
+#elif defined(HAVE_URING)
+ if (!srv_use_native_aio) {
+ } else if (io_uring_may_be_unsafe) {
+ msg("InnoDB: Using liburing on this kernel %s may cause hangs;"
+ " see https://jira.mariadb.org/browse/MDEV-26674",
+ io_uring_may_be_unsafe);
+ } else {
+ msg("InnoDB: Using liburing");
+ }
+#else
+ /* Currently native AIO is supported only on windows and linux
+ and that also when the support is compiled in. In all other
+ cases, we ignore the setting of innodb_use_native_aio. */
+ srv_use_native_aio = FALSE;
+
+#endif
+
+ /* Assign the default value to srv_undo_dir if it's not specified, as
+ my_getopt does not support default values for string options. We also
+ ignore the option and override innodb_undo_directory on --prepare,
+ because separate undo tablespaces are copied to the root backup
+ directory. */
+
+ if (!srv_undo_dir || !xtrabackup_backup) {
+ srv_undo_dir = (char*) ".";
+ }
+
+ compile_time_assert(SRV_FORCE_IGNORE_CORRUPT == 1);
+
+ /*
+ * This option can be read both from the command line, and the
+ * defaults file. The assignment should account for both cases,
+ * and for "--innobackupex". Since the command line argument is
+ * parsed after the defaults file, it takes precedence.
+ */
+ if (xtrabackup_innodb_force_recovery) {
+ srv_force_recovery = xtrabackup_innodb_force_recovery;
+ }
+
+ if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) {
+ if (!xtrabackup_prepare) {
+ msg("mariabackup: The option \"innodb_force_recovery\""
+ " should only be used with \"%s\".",
+ (innobackupex_mode ? "--apply-log" : "--prepare"));
+ goto error;
+ } else {
+ msg("innodb_force_recovery = %lu", srv_force_recovery);
+ }
+ }
+
+#ifdef _WIN32
+ srv_use_native_aio = TRUE;
+#endif
+ return false;
+
+error:
+ msg("mariabackup: innodb_init_param(): Error occurred.\n");
+ return true;
+}
+
+static byte log_hdr_buf[log_t::START_OFFSET + SIZE_OF_FILE_CHECKPOINT];
+
+/** Initialize an InnoDB log file header in log_hdr_buf[] */
+static void log_hdr_init()
+{
+ memset(log_hdr_buf, 0, sizeof log_hdr_buf);
+ mach_write_to_4(LOG_HEADER_FORMAT + log_hdr_buf, log_t::FORMAT_10_8);
+ mach_write_to_8(LOG_HEADER_START_LSN + log_hdr_buf,
+ log_sys.next_checkpoint_lsn);
+ snprintf(reinterpret_cast<char*>(LOG_HEADER_CREATOR + log_hdr_buf),
+ 16, "Backup %u.%u.%u",
+ MYSQL_VERSION_ID / 10000, MYSQL_VERSION_ID / 100 % 100,
+ MYSQL_VERSION_ID % 100);
+ if (log_sys.is_encrypted())
+ log_crypt_write_header(log_hdr_buf + LOG_HEADER_CREATOR_END);
+ mach_write_to_4(508 + log_hdr_buf, my_crc32c(0, log_hdr_buf, 508));
+ mach_write_to_8(log_hdr_buf + 0x1000, log_sys.next_checkpoint_lsn);
+ mach_write_to_8(log_hdr_buf + 0x1008, recv_sys.lsn);
+ mach_write_to_4(log_hdr_buf + 0x103c,
+ my_crc32c(0, log_hdr_buf + 0x1000, 60));
+}
+
+static bool innodb_init()
+{
+ bool create_new_db= false;
+
+ srv_max_io_capacity= srv_io_capacity >= SRV_MAX_IO_CAPACITY_LIMIT / 2
+ ? SRV_MAX_IO_CAPACITY_LIMIT : std::max(2 * srv_io_capacity, 2000UL);
+
+ /* Check if the data files exist or not. */
+ dberr_t err= srv_sys_space.check_file_spec(&create_new_db, 5U << 20);
+
+ if (create_new_db)
+ {
+ msg("mariadb-backup: InnoDB files do not exist");
+ return true;
+ }
+
+ if (err == DB_SUCCESS)
+ err= srv_start(false);
+
+ if (err != DB_SUCCESS)
+ {
+ msg("mariadb-backup: srv_start() returned %d (%s).", err, ut_strerr(err));
+ return true;
+ }
+
+ ut_ad(srv_force_recovery <= SRV_FORCE_IGNORE_CORRUPT);
+ ut_ad(recv_no_log_write);
+ buf_flush_sync();
+ recv_sys.debug_free();
+ ut_ad(!os_aio_pending_reads());
+ ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex));
+ ut_ad(!buf_pool.get_oldest_modification(0));
+ ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex));
+ /* os_aio_pending_writes() may hold here if some write_io_callback()
+ did not release the slot yet. However, the page write itself must
+ have completed, because the buf_pool.flush_list is empty. In debug
+ builds, we wait for this to happen, hoping to get a hung process if
+ this assumption does not hold. */
+ ut_d(os_aio_wait_until_no_pending_writes(false));
+ log_sys.close_file();
+
+ if (xtrabackup_incremental)
+ /* Reset the ib_logfile0 in --target-dir, not --incremental-dir. */
+ srv_log_group_home_dir= xtrabackup_target_dir;
+
+ bool ret;
+ const std::string ib_logfile0{get_log_file_path()};
+ os_file_delete_if_exists_func(ib_logfile0.c_str(), nullptr);
+ os_file_t file= os_file_create_func(ib_logfile0.c_str(),
+ OS_FILE_CREATE, OS_FILE_NORMAL,
+ OS_DATA_FILE_NO_O_DIRECT, false, &ret);
+ if (!ret)
+ {
+ invalid_log:
+ msg("mariadb-backup: Cannot create %s", ib_logfile0.c_str());
+ return true;
+ }
+
+ recv_sys.lsn= log_sys.next_checkpoint_lsn=
+ log_sys.get_lsn() - SIZE_OF_FILE_CHECKPOINT;
+ log_sys.set_latest_format(false); // not encrypted
+ log_hdr_init();
+ byte *b= &log_hdr_buf[log_t::START_OFFSET];
+ b[0]= FILE_CHECKPOINT | 10;
+ mach_write_to_8(b + 3, recv_sys.lsn);
+ b[11]= 1;
+ mach_write_to_4(b + 12, my_crc32c(0, b, 11));
+ static_assert(12 + 4 == SIZE_OF_FILE_CHECKPOINT, "compatibility");
+
+#ifdef _WIN32
+ DWORD len;
+ ret= WriteFile(file, log_hdr_buf, sizeof log_hdr_buf,
+ &len, nullptr) && len == sizeof log_hdr_buf;
+#else
+ ret= sizeof log_hdr_buf == write(file, log_hdr_buf, sizeof log_hdr_buf);
+#endif
+ if (!os_file_close_func(file) || !ret)
+ goto invalid_log;
+ return false;
+}
+
+/* ================= common ================= */
+
+/***********************************************************************
+Read backup meta info.
+@return TRUE on success, FALSE on failure. */
+static
+my_bool
+xtrabackup_read_metadata(char *filename)
+{
+ FILE *fp;
+ my_bool r = TRUE;
+
+ fp = fopen(filename,"r");
+ if(!fp) {
+ msg("Error: cannot open %s", filename);
+ return(FALSE);
+ }
+
+ if (fscanf(fp, "backup_type = %29s\n", metadata_type)
+ != 1) {
+ r = FALSE;
+ goto end;
+ }
+ /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
+ format. */
+ if (fscanf(fp, "from_lsn = " UINT64PF "\n", &metadata_from_lsn)
+ != 1) {
+ r = FALSE;
+ goto end;
+ }
+ if (fscanf(fp, "to_lsn = " UINT64PF "\n", &metadata_to_lsn)
+ != 1) {
+ r = FALSE;
+ goto end;
+ }
+ if (fscanf(fp, "last_lsn = " UINT64PF "\n", &metadata_last_lsn)
+ != 1) {
+ metadata_last_lsn = 0;
+ }
+ /* Optional fields */
+
+end:
+ fclose(fp);
+
+ return(r);
+}
+
+/***********************************************************************
+Print backup meta info to a specified buffer. */
+static
+void
+xtrabackup_print_metadata(char *buf, size_t buf_len)
+{
+ /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
+ format. */
+ snprintf(buf, buf_len,
+ "backup_type = %s\n"
+ "from_lsn = " UINT64PF "\n"
+ "to_lsn = " UINT64PF "\n"
+ "last_lsn = " UINT64PF "\n",
+ metadata_type,
+ metadata_from_lsn,
+ metadata_to_lsn,
+ metadata_last_lsn);
+}
+
+/***********************************************************************
+Stream backup meta info to a specified datasink.
+@return TRUE on success, FALSE on failure. */
+static
+my_bool
+xtrabackup_stream_metadata(ds_ctxt_t *ds_ctxt)
+{
+ char buf[1024];
+ size_t len;
+ ds_file_t *stream;
+ MY_STAT mystat;
+ my_bool rc = TRUE;
+
+ xtrabackup_print_metadata(buf, sizeof(buf));
+
+ len = strlen(buf);
+
+ mystat.st_size = len;
+ mystat.st_mtime = my_time(0);
+
+ stream = ds_open(ds_ctxt, XTRABACKUP_METADATA_FILENAME, &mystat);
+ if (stream == NULL) {
+ msg("Error: cannot open output stream for %s", XTRABACKUP_METADATA_FILENAME);
+ return(FALSE);
+ }
+
+ if (ds_write(stream, buf, len)) {
+ rc = FALSE;
+ }
+
+ if (ds_close(stream)) {
+ rc = FALSE;
+ }
+
+ return(rc);
+}
+
+/***********************************************************************
+Write backup meta info to a specified file.
+@return TRUE on success, FALSE on failure. */
+static
+my_bool
+xtrabackup_write_metadata(const char *filepath)
+{
+ char buf[1024];
+ size_t len;
+ FILE *fp;
+
+ xtrabackup_print_metadata(buf, sizeof(buf));
+
+ len = strlen(buf);
+
+ fp = fopen(filepath, "w");
+ if(!fp) {
+ msg("Error: cannot open %s", filepath);
+ return(FALSE);
+ }
+ if (fwrite(buf, len, 1, fp) < 1) {
+ fclose(fp);
+ return(FALSE);
+ }
+
+ fclose(fp);
+
+ return(TRUE);
+}
+
+/***********************************************************************
+Read meta info for an incremental delta.
+@return TRUE on success, FALSE on failure. */
+static my_bool
+xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info)
+{
+ FILE* fp;
+ char key[51];
+ char value[51];
+ my_bool r = TRUE;
+
+ /* set defaults */
+ ulint page_size = ULINT_UNDEFINED, zip_size = 0;
+ info->space_id = UINT32_MAX;
+
+ fp = fopen(filepath, "r");
+ if (!fp) {
+ /* Meta files for incremental deltas are optional */
+ return(TRUE);
+ }
+
+ while (!feof(fp)) {
+ if (fscanf(fp, "%50s = %50s\n", key, value) == 2) {
+ if (strcmp(key, "page_size") == 0) {
+ page_size = strtoul(value, NULL, 10);
+ } else if (strcmp(key, "zip_size") == 0) {
+ zip_size = strtoul(value, NULL, 10);
+ } else if (strcmp(key, "space_id") == 0) {
+ info->space_id = static_cast<uint32_t>
+ (strtoul(value, NULL, 10));
+ }
+ }
+ }
+
+ fclose(fp);
+
+ if (page_size == ULINT_UNDEFINED) {
+ msg("page_size is required in %s", filepath);
+ r = FALSE;
+ } else {
+ info->page_size = page_size;
+ info->zip_size = zip_size;
+ }
+
+ if (info->space_id == UINT32_MAX) {
+ msg("mariabackup: Warning: This backup was taken with XtraBackup 2.0.1 "
+ "or earlier, some DDL operations between full and incremental "
+ "backups may be handled incorrectly");
+ }
+
+ return(r);
+}
+
+/***********************************************************************
+Write meta info for an incremental delta.
+@return TRUE on success, FALSE on failure. */
+my_bool
+xb_write_delta_metadata(ds_ctxt *ds_meta,
+ const char *filename, const xb_delta_info_t *info)
+{
+ ds_file_t *f;
+ char buf[64];
+ my_bool ret;
+ size_t len;
+ MY_STAT mystat;
+
+ snprintf(buf, sizeof(buf),
+ "page_size = " ULINTPF "\n"
+ "zip_size = " ULINTPF " \n"
+ "space_id = %u\n",
+ info->page_size,
+ info->zip_size,
+ info->space_id);
+ len = strlen(buf);
+
+ mystat.st_size = len;
+ mystat.st_mtime = my_time(0);
+
+ f = ds_open(ds_meta, filename, &mystat);
+ if (f == NULL) {
+ msg("Error: Can't open output stream for %s",filename);
+ return(FALSE);
+ }
+
+ ret = (ds_write(f, buf, len) == 0);
+
+ if (ds_close(f)) {
+ ret = FALSE;
+ }
+
+ return(ret);
+}
+
+/* ================= backup ================= */
+void xtrabackup_io_throttling()
+{
+ if (!xtrabackup_backup || !xtrabackup_throttle)
+ return;
+
+ mysql_mutex_lock(&recv_sys.mutex);
+ if (io_ticket-- < 0)
+ mysql_cond_wait(&wait_throttle, &recv_sys.mutex);
+ mysql_mutex_unlock(&recv_sys.mutex);
+}
+
+static
+my_bool regex_list_check_match(
+ const regex_list_t& list,
+ const char* name)
+{
+ regmatch_t tables_regmatch[1];
+ for (regex_list_t::const_iterator i = list.begin(), end = list.end();
+ i != end; ++i) {
+ const regex_t& regex = *i;
+ int regres = regexec(&regex, name, 1, tables_regmatch, 0);
+
+ if (regres != REG_NOMATCH) {
+ return(TRUE);
+ }
+ }
+ return(FALSE);
+}
+
+static
+my_bool
+find_filter_in_hashtable(
+ const char* name,
+ hash_table_t* table,
+ xb_filter_entry_t** result
+)
+{
+ xb_filter_entry_t* found = NULL;
+ const ulint fold = my_crc32c(0, name, strlen(name));
+ HASH_SEARCH(name_hash, table, fold,
+ xb_filter_entry_t*,
+ found, (void) 0,
+ !strcmp(found->name, name));
+
+ if (found && result) {
+ *result = found;
+ }
+ return (found != NULL);
+}
+
+/************************************************************************
+Checks if a given table name matches any of specifications given in
+regex_list or tables_hash.
+
+@return TRUE on match or both regex_list and tables_hash are empty.*/
+static my_bool
+check_if_table_matches_filters(const char *name,
+ const regex_list_t& regex_list,
+ hash_table_t* tables_hash)
+{
+ if (regex_list.empty() && !tables_hash->array) {
+ return(FALSE);
+ }
+
+ if (regex_list_check_match(regex_list, name)) {
+ return(TRUE);
+ }
+
+ return tables_hash->array &&
+ find_filter_in_hashtable(name, tables_hash, NULL);
+}
+
+enum skip_database_check_result {
+ DATABASE_SKIP,
+ DATABASE_SKIP_SOME_TABLES,
+ DATABASE_DONT_SKIP,
+ DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED,
+};
+
+/************************************************************************
+Checks if a database specified by name should be skipped from backup based on
+the --databases, --databases_file or --databases_exclude options.
+
+@return TRUE if entire database should be skipped,
+ FALSE otherwise.
+*/
+static
+skip_database_check_result
+check_if_skip_database(
+ const char* name /*!< in: path to the database */
+)
+{
+ /* There are some filters for databases, check them */
+ xb_filter_entry_t* database = NULL;
+
+ if (databases_exclude_hash.array &&
+ find_filter_in_hashtable(name, &databases_exclude_hash,
+ &database) &&
+ (!database->has_tables || !databases_include_hash.array)) {
+ /* Database is found and there are no tables specified,
+ skip entire db. */
+ return DATABASE_SKIP;
+ }
+
+ if (databases_include_hash.array) {
+ if (!find_filter_in_hashtable(name, &databases_include_hash,
+ &database)) {
+ /* Database isn't found, skip the database */
+ return DATABASE_SKIP;
+ } else if (database->has_tables) {
+ return DATABASE_SKIP_SOME_TABLES;
+ } else {
+ return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED;
+ }
+ }
+
+ return DATABASE_DONT_SKIP;
+}
+
+/************************************************************************
+Checks if a database specified by path should be skipped from backup based on
+the --databases, --databases_file or --databases_exclude options.
+
+@return TRUE if the table should be skipped. */
+my_bool
+check_if_skip_database_by_path(
+ const char* path /*!< in: path to the db directory. */
+)
+{
+ if (!databases_include_hash.array && !databases_exclude_hash.array) {
+ return(FALSE);
+ }
+
+ const char* db_name = strrchr(path, '/');
+#ifdef _WIN32
+ if (const char* last = strrchr(path, '\\')) {
+ if (!db_name || last > db_name) {
+ db_name = last;
+ }
+ }
+#endif
+
+ if (db_name == NULL) {
+ db_name = path;
+ } else {
+ ++db_name;
+ }
+
+ return check_if_skip_database(db_name) == DATABASE_SKIP;
+}
+
+/************************************************************************
+Checks if a table specified as a name in the form "database/name" (InnoDB 5.6)
+or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on
+the --tables or --tables-file options.
+
+@return TRUE if the table should be skipped. */
+my_bool
+check_if_skip_table(
+/******************/
+ const char* name) /*!< in: path to the table */
+{
+ char buf[FN_REFLEN];
+ const char *dbname, *tbname;
+ const char *ptr;
+ char *eptr;
+
+
+ dbname = NULL;
+ tbname = name;
+ for (;;) {
+ ptr= strchr(tbname, '/');
+#ifdef _WIN32
+ if (!ptr) {
+ ptr= strchr(tbname,'\\');
+ }
+#endif
+ if (!ptr) {
+ break;
+ }
+ dbname = tbname;
+ tbname = ptr + 1;
+ }
+
+ if (strncmp(tbname, tmp_file_prefix, tmp_file_prefix_length) == 0) {
+ return TRUE;
+ }
+
+ if (regex_exclude_list.empty() &&
+ regex_include_list.empty() &&
+ !tables_include_hash.array &&
+ !tables_exclude_hash.array &&
+ !databases_include_hash.array &&
+ !databases_exclude_hash.array) {
+ return(FALSE);
+ }
+
+ if (dbname == NULL) {
+ return(FALSE);
+ }
+
+ strncpy(buf, dbname, FN_REFLEN - 1);
+ buf[FN_REFLEN - 1] = '\0';
+ buf[tbname - 1 - dbname] = '\0';
+
+ const skip_database_check_result skip_database =
+ check_if_skip_database(buf);
+ if (skip_database == DATABASE_SKIP) {
+ return (TRUE);
+ }
+
+ buf[tbname - 1 - dbname] = '.';
+
+ /* Check if there's a suffix in the table name. If so, truncate it. We
+ rely on the fact that a dot cannot be a part of a table name (it is
+ encoded by the server with the @NNNN syntax). */
+ if ((eptr = strchr(&buf[tbname - dbname], '.')) != NULL) {
+
+ *eptr = '\0';
+ }
+
+ /* For partitioned tables first try to match against the regexp
+ without truncating the #P#... suffix so we can backup individual
+ partitions with regexps like '^test[.]t#P#p5' */
+ if (check_if_table_matches_filters(buf, regex_exclude_list,
+ &tables_exclude_hash)) {
+ return(TRUE);
+ }
+ if (check_if_table_matches_filters(buf, regex_include_list,
+ &tables_include_hash)) {
+ return(FALSE);
+ }
+ if ((eptr = strstr(buf, "#P#")) != NULL) {
+ *eptr = 0;
+
+ if (check_if_table_matches_filters(buf, regex_exclude_list,
+ &tables_exclude_hash)) {
+ return (TRUE);
+ }
+ if (check_if_table_matches_filters(buf, regex_include_list,
+ &tables_include_hash)) {
+ return(FALSE);
+ }
+ }
+
+ if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) {
+ /* Database is in include-list, and qualified name wasn't
+ found in any of exclusion filters.*/
+ return (FALSE);
+ }
+
+ if (skip_database == DATABASE_SKIP_SOME_TABLES ||
+ !regex_include_list.empty() ||
+ tables_include_hash.array) {
+
+ /* Include lists are present, but qualified name
+ failed to match any.*/
+ return(TRUE);
+ }
+
+ return(FALSE);
+}
+
+const char*
+xb_get_copy_action(const char *dflt)
+{
+ const char *action;
+
+ if (xtrabackup_stream) {
+ if (xtrabackup_compress) {
+ action = "Compressing and streaming";
+ } else {
+ action = "Streaming";
+ }
+ } else {
+ if (xtrabackup_compress) {
+ action = "Compressing";
+ } else {
+ action = dflt;
+ }
+ }
+
+ return(action);
+}
+
+
+/** Copy innodb data file to the specified destination.
+
+@param[in] node file node of a tablespace
+@param[in] thread_n thread id, used in the text of diagnostic messages
+@param[in] dest_name destination file name
+@param[in] write_filter write filter to copy data, can be pass-through filter
+for full backup, pages filter for incremental backup, etc.
+
+@return FALSE on success and TRUE on error */
+static my_bool xtrabackup_copy_datafile(ds_ctxt *ds_data,
+ ds_ctxt *ds_meta,
+ fil_node_t *node, uint thread_n,
+ const char *dest_name,
+ const xb_write_filt_t &write_filter,
+ CorruptedPages &corrupted_pages)
+{
+ char dst_name[FN_REFLEN];
+ ds_file_t *dstfile = NULL;
+ xb_fil_cur_t cursor;
+ xb_fil_cur_result_t res;
+ xb_write_filt_ctxt_t write_filt_ctxt;
+ const char *action;
+ xb_read_filt_t *read_filter;
+ my_bool rc = FALSE;
+
+ if (fil_is_user_tablespace_id(node->space->id)
+ && check_if_skip_table(filename_to_spacename(node->name,
+ strlen(node->name)).
+ c_str())) {
+ msg(thread_n, "Skipping %s.", node->name);
+ return(FALSE);
+ }
+
+ memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t));
+
+ bool was_dropped;
+ mysql_mutex_lock(&recv_sys.mutex);
+ was_dropped = (ddl_tracker.drops.find(node->space->id) != ddl_tracker.drops.end());
+ mysql_mutex_unlock(&recv_sys.mutex);
+ if (was_dropped) {
+ if (node->is_open()) {
+ mysql_mutex_lock(&fil_system.mutex);
+ node->close();
+ mysql_mutex_unlock(&fil_system.mutex);
+ }
+ goto skip;
+ }
+
+ if (!changed_page_bitmap) {
+ read_filter = &rf_pass_through;
+ }
+ else {
+ read_filter = &rf_bitmap;
+ }
+
+ res = xb_fil_cur_open(&cursor, read_filter, node, thread_n, ULLONG_MAX);
+ if (res == XB_FIL_CUR_SKIP) {
+ goto skip;
+ } else if (res == XB_FIL_CUR_ERROR) {
+ goto error;
+ }
+
+ strncpy(dst_name, dest_name ? dest_name : cursor.rel_path,
+ sizeof dst_name - 1);
+ dst_name[sizeof dst_name - 1] = '\0';
+
+ ut_a(write_filter.process != NULL);
+
+ if (write_filter.init != NULL &&
+ !write_filter.init(ds_meta, &write_filt_ctxt, dst_name, &cursor,
+ opt_log_innodb_page_corruption ? &corrupted_pages : NULL)) {
+ msg (thread_n, "mariabackup: error: failed to initialize page write filter.");
+ goto error;
+ }
+
+ dstfile = ds_open(ds_data, dst_name, &cursor.statinfo);
+ if (dstfile == NULL) {
+ msg(thread_n,"mariabackup: error: can't open the destination stream for %s", dst_name);
+ goto error;
+ }
+
+ action = xb_get_copy_action();
+
+ if (xtrabackup_stream) {
+ msg(thread_n, "%s %s", action, node->name);
+ } else {
+ msg(thread_n, "%s %s to %s", action, node->name,
+ dstfile->path);
+ }
+
+ /* The main copy loop */
+ while (1) {
+ res = xb_fil_cur_read(&cursor, corrupted_pages);
+ if (res == XB_FIL_CUR_ERROR) {
+ goto error;
+ }
+
+ if (res == XB_FIL_CUR_EOF) {
+ break;
+ }
+
+ if (!write_filter.process(&write_filt_ctxt, dstfile)) {
+ goto error;
+ }
+
+ if (res == XB_FIL_CUR_SKIP) {
+ mysql_mutex_lock(&recv_sys.mutex);
+ fail_undo_ids.insert(
+ static_cast<uint32_t>(cursor.space_id));
+ mysql_mutex_unlock(&recv_sys.mutex);
+ break;
+ }
+ }
+
+ if (write_filter.finalize
+ && !write_filter.finalize(&write_filt_ctxt, dstfile)) {
+ goto error;
+ } else {
+ const fil_space_t::name_type name = node->space->name();
+
+ mysql_mutex_lock(&recv_sys.mutex);
+ ddl_tracker.tables_in_backup.emplace(node->space->id,
+ std::string(name.data(),
+ name.size()));
+ mysql_mutex_unlock(&recv_sys.mutex);
+ }
+
+ /* close */
+ msg(thread_n," ...done");
+ xb_fil_cur_close(&cursor);
+ if (ds_close(dstfile)) {
+ rc = TRUE;
+ }
+ if (write_filter.deinit) {
+ write_filter.deinit(&write_filt_ctxt);
+ }
+ return(rc);
+
+error:
+ xb_fil_cur_close(&cursor);
+ if (dstfile != NULL) {
+ ds_close(dstfile);
+ }
+ if (write_filter.deinit) {
+ write_filter.deinit(&write_filt_ctxt);;
+ }
+ msg(thread_n, "mariabackup: xtrabackup_copy_datafile() failed.");
+ return(TRUE); /*ERROR*/
+
+skip:
+
+ if (dstfile != NULL) {
+ ds_close(dstfile);
+ }
+ if (write_filter.deinit) {
+ write_filter.deinit(&write_filt_ctxt);
+ }
+ msg(thread_n,"Warning: We assume the table was dropped during xtrabackup execution and ignore the tablespace %s", node->name);
+ return(FALSE);
+}
+
+/** Copy redo log until the current end of the log is reached
+@return whether the operation failed */
+static bool xtrabackup_copy_logfile()
+{
+ mysql_mutex_assert_owner(&recv_sys.mutex);
+ DBUG_EXECUTE_IF("log_checksum_mismatch", return false;);
+
+ ut_a(dst_log_file);
+ ut_ad(recv_sys.is_initialised());
+ const size_t sequence_offset{log_sys.is_encrypted() ? 8U + 5U : 5U};
+ const size_t block_size_1{log_sys.get_block_size() - 1};
+
+ ut_ad(!log_sys.is_pmem());
+
+ {
+ recv_sys.offset= size_t(recv_sys.lsn - log_sys.get_first_lsn()) &
+ block_size_1;
+ recv_sys.len= 0;
+ }
+
+ for (unsigned retry_count{0};;)
+ {
+ recv_sys_t::parse_mtr_result r;
+ size_t start_offset{recv_sys.offset};
+
+ {
+ {
+ auto source_offset=
+ log_sys.calc_lsn_offset(recv_sys.lsn + recv_sys.len -
+ recv_sys.offset);
+ source_offset&= ~block_size_1;
+ size_t size{log_sys.buf_size - recv_sys.len};
+ if (UNIV_UNLIKELY(source_offset + size > log_sys.file_size))
+ {
+ const size_t first{size_t(log_sys.file_size - source_offset)};
+ ut_ad(first <= log_sys.buf_size);
+ log_sys.log.read(source_offset, {log_sys.buf, first});
+ size-= first;
+ if (log_sys.START_OFFSET + size > source_offset)
+ size= size_t(source_offset - log_sys.START_OFFSET);
+ if (size)
+ log_sys.log.read(log_sys.START_OFFSET,
+ {log_sys.buf + first, size});
+ size+= first;
+ }
+ else
+ log_sys.log.read(source_offset, {log_sys.buf, size});
+ recv_sys.len= size;
+ }
+
+ if (log_sys.buf[recv_sys.offset] <= 1)
+ break;
+
+ if (recv_sys.parse_mtr<false>(false) == recv_sys_t::OK)
+ {
+ do
+ {
+ /* Set the sequence bit (the backed-up log will not wrap around) */
+ byte *seq= &log_sys.buf[recv_sys.offset - sequence_offset];
+ ut_ad(*seq == log_sys.get_sequence_bit(recv_sys.lsn -
+ sequence_offset));
+ *seq= 1;
+ }
+ while ((r= recv_sys.parse_mtr<false>(false)) == recv_sys_t::OK);
+
+ if (ds_write(dst_log_file, log_sys.buf + start_offset,
+ recv_sys.offset - start_offset))
+ {
+ msg("Error: write to ib_logfile0 failed");
+ return true;
+ }
+ else
+ {
+ const auto ofs= recv_sys.offset & ~block_size_1;
+ memmove_aligned<64>(log_sys.buf, log_sys.buf + ofs,
+ recv_sys.len - ofs);
+ recv_sys.len-= ofs;
+ recv_sys.offset&= block_size_1;
+ }
+
+ pthread_cond_broadcast(&scanned_lsn_cond);
+
+ if (r == recv_sys_t::GOT_EOF)
+ break;
+
+ if (recv_sys.offset < log_sys.get_block_size())
+ break;
+
+ if (xtrabackup_throttle && io_ticket-- < 0)
+ mysql_cond_wait(&wait_throttle, &recv_sys.mutex);
+
+ retry_count= 0;
+ continue;
+ }
+ else
+ {
+ recv_sys.len= recv_sys.offset & ~block_size_1;
+ if (retry_count == 100)
+ break;
+
+ mysql_mutex_unlock(&recv_sys.mutex);
+ if (!retry_count++)
+ msg("Retrying read of log at LSN=" LSN_PF, recv_sys.lsn);
+ my_sleep(1000);
+ }
+ }
+ mysql_mutex_lock(&recv_sys.mutex);
+ }
+
+ if (verbose)
+ msg(">> log scanned up to (" LSN_PF ")", recv_sys.lsn);
+ return false;
+}
+
+/**
+Wait until redo log copying thread processes given lsn
+*/
+void backup_wait_for_lsn(lsn_t lsn)
+{
+ mysql_mutex_lock(&recv_sys.mutex);
+ for (lsn_t last_lsn{recv_sys.lsn}; last_lsn < lsn; )
+ {
+ timespec abstime;
+ set_timespec(abstime, 5);
+ if (my_cond_timedwait(&scanned_lsn_cond, &recv_sys.mutex.m_mutex,
+ &abstime) &&
+ last_lsn == recv_sys.lsn)
+ die("Was only able to copy log from " LSN_PF " to " LSN_PF
+ ", not " LSN_PF "; try increasing innodb_log_file_size",
+ log_sys.next_checkpoint_lsn, last_lsn, lsn);
+ last_lsn= recv_sys.lsn;
+ }
+ mysql_mutex_unlock(&recv_sys.mutex);
+}
+
+extern lsn_t server_lsn_after_lock;
+
+static void log_copying_thread()
+{
+ my_thread_init();
+ mysql_mutex_lock(&recv_sys.mutex);
+ while (!xtrabackup_copy_logfile() &&
+ (!metadata_to_lsn || metadata_to_lsn > recv_sys.lsn))
+ {
+ timespec abstime;
+ set_timespec_nsec(abstime, 1000000ULL * xtrabackup_log_copy_interval);
+ mysql_cond_timedwait(&log_copying_stop, &recv_sys.mutex, &abstime);
+ }
+ log_copying_running= false;
+ mysql_mutex_unlock(&recv_sys.mutex);
+ my_thread_end();
+}
+
+/** whether io_watching_thread() is active; protected by recv_sys.mutex */
+static bool have_io_watching_thread;
+
+/* io throttle watching (rough) */
+static void io_watching_thread()
+{
+ my_thread_init();
+ /* currently, for --backup only */
+ ut_ad(xtrabackup_backup);
+
+ mysql_mutex_lock(&recv_sys.mutex);
+ ut_ad(have_io_watching_thread);
+
+ while (log_copying_running && !metadata_to_lsn)
+ {
+ timespec abstime;
+ set_timespec(abstime, 1);
+ mysql_cond_timedwait(&log_copying_stop, &recv_sys.mutex, &abstime);
+ io_ticket= xtrabackup_throttle;
+ mysql_cond_broadcast(&wait_throttle);
+ }
+
+ /* stop io throttle */
+ xtrabackup_throttle= 0;
+ have_io_watching_thread= false;
+ mysql_cond_broadcast(&wait_throttle);
+ mysql_mutex_unlock(&recv_sys.mutex);
+ my_thread_end();
+}
+
+#ifndef DBUG_OFF
+char *dbug_mariabackup_get_val(const char *event,
+ const fil_space_t::name_type key)
+{
+ char envvar[FN_REFLEN];
+ strncpy(envvar, event, sizeof envvar - 1);
+ envvar[(sizeof envvar) - 1] = '\0';
+
+ if (key.size() && key.size() + strlen(envvar) < (sizeof envvar) - 2)
+ {
+ strcat(envvar, "_");
+ strncat(envvar, key.data(), key.size());
+ if (char *slash= strchr(envvar, '/'))
+ *slash= '_';
+ }
+
+ char *val = getenv(envvar);
+ return val && *val ? val : nullptr;
+}
+
+/*
+In debug mode, execute SQL statement that was passed via environment.
+To use this facility, you need to
+
+1. Add code DBUG_EXECUTE_MARIABACKUP_EVENT("my_event_name", key););
+ to the code. key is usually a table name
+2. Set environment variable my_event_name_$key SQL statement you want to execute
+ when event occurs, in DBUG_EXECUTE_IF from above.
+ In mtr , you can set environment via 'let' statement (do not use $ as the first char
+ for the variable)
+3. start mariabackup with --dbug=+d,debug_mariabackup_events
+*/
+void dbug_mariabackup_event(const char *event,
+ const fil_space_t::name_type key)
+{
+ char *sql = dbug_mariabackup_get_val(event, key);
+ if (sql && *sql) {
+ msg("dbug_mariabackup_event : executing '%s'", sql);
+ xb_mysql_query(mysql_connection, sql, false, true);
+ }
+}
+#endif // DBUG_OFF
+
+/** Datafiles copying thread.*/
+static void data_copy_thread_func(data_thread_ctxt_t *ctxt) /* thread context */
+{
+ uint num = ctxt->num;
+ fil_node_t* node;
+ ut_ad(ctxt->corrupted_pages);
+
+ /*
+ Initialize mysys thread-specific memory so we can
+ use mysys functions in this thread.
+ */
+ my_thread_init();
+
+ while ((node = datafiles_iter_next(ctxt->it)) != NULL) {
+ DBUG_MARIABACKUP_EVENT("before_copy", node->space->name());
+ DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy",
+ node->space->name(),
+ backup_wait_for_lsn(get_current_lsn(mysql_connection)););
+ /* copy the datafile */
+ if (xtrabackup_copy_datafile(ctxt->datasinks->m_data,
+ ctxt->datasinks->m_meta, node, num, NULL,
+ xtrabackup_incremental ? wf_incremental : wf_write_through,
+ *ctxt->corrupted_pages))
+ die("failed to copy datafile.");
+
+ DBUG_MARIABACKUP_EVENT("after_copy", node->space->name());
+ }
+
+ pthread_mutex_lock(ctxt->count_mutex);
+ (*ctxt->count)--;
+ pthread_mutex_unlock(ctxt->count_mutex);
+
+ my_thread_end();
+}
+
+/************************************************************************
+Initialize the appropriate datasink(s). Both local backups and streaming in the
+'xbstream' format allow parallel writes so we can write directly.
+
+Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks
+for the data stream (and don't allow parallel data copying) and for metainfo
+files (including ib_logfile0). The second datasink writes to temporary
+files first, and then streams them in a serialized way when closed. */
+void Backup_datasinks::init()
+{
+ /* Start building out the pipelines from the terminus back */
+ if (xtrabackup_stream) {
+ /* All streaming goes to stdout */
+ m_data = m_meta = m_redo = ds_create(xtrabackup_target_dir,
+ DS_TYPE_STDOUT);
+ } else {
+ /* Local filesystem */
+ m_data = m_meta = m_redo = ds_create(xtrabackup_target_dir,
+ DS_TYPE_LOCAL);
+ }
+
+ /* Track it for destruction */
+ add_datasink_to_destroy(m_data);
+
+ /* Stream formatting */
+ if (xtrabackup_stream) {
+ ds_ctxt_t *ds;
+
+ ut_a(xtrabackup_stream_fmt == XB_STREAM_FMT_XBSTREAM);
+ ds = ds_create(xtrabackup_target_dir, DS_TYPE_XBSTREAM);
+
+ add_datasink_to_destroy(ds);
+
+ ds_set_pipe(ds, m_data);
+ m_data = ds;
+
+
+ m_redo = m_meta = m_data;
+ }
+
+ /* Compression for m_data and m_redo */
+ if (xtrabackup_compress) {
+ ds_ctxt_t *ds;
+
+ /* Use a 1 MB buffer for compressed output stream */
+ ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
+ ds_buffer_set_size(ds, 1024 * 1024);
+ add_datasink_to_destroy(ds);
+ ds_set_pipe(ds, m_data);
+ if (m_data != m_redo) {
+ m_data = ds;
+ ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
+ ds_buffer_set_size(ds, 1024 * 1024);
+ add_datasink_to_destroy(ds);
+ ds_set_pipe(ds, m_redo);
+ m_redo = ds;
+ } else {
+ m_redo = m_data = ds;
+ }
+
+ ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
+ add_datasink_to_destroy(ds);
+ ds_set_pipe(ds, m_data);
+ if (m_data != m_redo) {
+ m_data = ds;
+ ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
+ add_datasink_to_destroy(ds);
+ ds_set_pipe(ds, m_redo);
+ m_redo = ds;
+ } else {
+ m_redo = m_data = ds;
+ }
+ }
+}
+
+#define SRV_MAX_N_PENDING_SYNC_IOS 100
+
+/** Initialize the tablespace cache subsystem. */
+static
+void
+xb_fil_io_init()
+{
+ fil_system.create(srv_file_per_table ? 50000 : 5000);
+ fil_system.freeze_space_list = 1;
+ fil_system.space_id_reuse_warned = true;
+}
+
+/** Load tablespace.
+
+@param[in] dirname directory name of the tablespace to open
+@param[in] filname file name of the tablespece to open
+@param[in] is_remote true if tablespace file is .isl
+@param[in] skip_node_page0 true if we don't need to read node page 0. Otherwise
+node page0 will be read, and it's size and free pages limit
+will be set from page 0, what is neccessary for checking and fixing corrupted
+pages.
+@param[in] defer_space_id use the space id to create space object
+when there is deferred tablespace
+*/
+static void xb_load_single_table_tablespace(const char *dirname,
+ const char *filname,
+ bool is_remote,
+ bool skip_node_page0,
+ uint32_t defer_space_id)
+{
+ ut_ad(srv_operation == SRV_OPERATION_BACKUP
+ || srv_operation == SRV_OPERATION_RESTORE_DELTA
+ || srv_operation == SRV_OPERATION_RESTORE
+ || srv_operation == SRV_OPERATION_BACKUP_NO_DEFER);
+ /* Ignore .isl files on XtraBackup recovery. All tablespaces must be
+ local. */
+ if (is_remote && srv_operation == SRV_OPERATION_RESTORE_DELTA) {
+ return;
+ }
+ if (check_if_skip_table(filname)) {
+ return;
+ }
+
+ /* The name ends in .ibd or .isl;
+ try opening the file */
+ char* name;
+ size_t dirlen = dirname == NULL ? 0 : strlen(dirname);
+ size_t namelen = strlen(filname);
+ ulint pathlen = dirname == NULL ? namelen + 1: dirlen + namelen + 2;
+ dberr_t err;
+ fil_space_t *space;
+ bool defer = false;
+
+ name = static_cast<char*>(ut_malloc_nokey(pathlen));
+
+ if (dirname != NULL) {
+ snprintf(name, pathlen, "%s/%s", dirname, filname);
+ name[pathlen - 5] = 0;
+ } else {
+ snprintf(name, pathlen, "%s", filname);
+ name[pathlen - 5] = 0;
+ }
+
+ const fil_space_t::name_type n{name, pathlen - 5};
+ Datafile *file;
+
+ if (is_remote) {
+ RemoteDatafile* rf = new RemoteDatafile();
+ if (!rf->open_link_file(n)) {
+ die("Can't open datafile %s", name);
+ }
+ file = rf;
+ } else {
+ file = new Datafile();
+ file->make_filepath(".", n, IBD);
+ }
+
+ if (file->open_read_only(true) != DB_SUCCESS) {
+ die("Can't open datafile %s", name);
+ }
+
+ for (int i = 0; i < 10; i++) {
+ file->m_defer = false;
+ err = file->validate_first_page();
+
+ if (file->m_defer) {
+ if (defer_space_id) {
+ defer = true;
+ file->set_space_id(defer_space_id);
+ file->set_flags(FSP_FLAGS_PAGE_SSIZE());
+ err = DB_SUCCESS;
+ break;
+ }
+ } else if (err != DB_CORRUPTION) {
+ break;
+ }
+
+ my_sleep(1000);
+ }
+
+ if (!defer && file->m_defer) {
+ const char *file_path = file->filepath();
+ defer_space_names.insert(
+ filename_to_spacename(
+ file_path, strlen(file_path)));
+ delete file;
+ ut_free(name);
+ return;
+ }
+
+ bool is_empty_file = file->exists() && file->is_empty_file();
+
+ if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) {
+ mysql_mutex_lock(&fil_system.mutex);
+ space = fil_space_t::create(
+ file->space_id(), file->flags(),
+ FIL_TYPE_TABLESPACE, nullptr/* TODO: crypt_data */,
+ FIL_ENCRYPTION_DEFAULT,
+ file->handle() != OS_FILE_CLOSED);
+ ut_ad(space);
+ fil_node_t* node= space->add(
+ file->filepath(),
+ skip_node_page0 ? file->detach() : pfs_os_file_t(),
+ 0, false, false);
+ node->deferred= defer;
+ if (!space->read_page0())
+ err = DB_CANNOT_OPEN_FILE;
+ mysql_mutex_unlock(&fil_system.mutex);
+
+ if (srv_operation == SRV_OPERATION_RESTORE_DELTA
+ || xb_close_files) {
+ space->close();
+ }
+ }
+
+ delete file;
+
+ if (err != DB_SUCCESS && xtrabackup_backup && !is_empty_file) {
+ die("Failed to validate first page of the file %s, error %d",name, (int)err);
+ }
+
+ ut_free(name);
+}
+
+static void xb_load_single_table_tablespace(const std::string &space_name,
+ bool skip_node_page0,
+ uint32_t defer_space_id)
+{
+ std::string name(space_name);
+ bool is_remote= access((name + ".ibd").c_str(), R_OK) != 0;
+ const char *extension= is_remote ? ".isl" : ".ibd";
+ name.append(extension);
+ char buf[FN_REFLEN];
+ strncpy(buf, name.c_str(), sizeof buf - 1);
+ buf[sizeof buf - 1]= '\0';
+ const char *dbname= buf;
+ char *p= strchr(buf, '/');
+ if (!p)
+ die("Unexpected tablespace %s filename %s", space_name.c_str(),
+ name.c_str());
+ *p= 0;
+ const char *tablename= p + 1;
+ xb_load_single_table_tablespace(dbname, tablename, is_remote,
+ skip_node_page0, defer_space_id);
+}
+
+#ifdef _WIN32
+/**
+The os_file_opendir() function opens a directory stream corresponding to the
+directory named by the dirname argument. The directory stream is positioned
+at the first entry. In both Unix and Windows we automatically skip the '.'
+and '..' items at the start of the directory listing.
+@param[in] dirname directory name; it must not contain a trailing
+ '\' or '/'
+@return directory stream, NULL if error */
+os_file_dir_t os_file_opendir(const char *dirname)
+{
+ char path[OS_FILE_MAX_PATH + 3];
+
+ ut_a(strlen(dirname) < OS_FILE_MAX_PATH);
+
+ strcpy(path, dirname);
+ strcpy(path + strlen(path), "\\*");
+
+ /* Note that in Windows opening the 'directory stream' also retrieves
+ the first entry in the directory. Since it is '.', that is no problem,
+ as we will skip over the '.' and '..' entries anyway. */
+
+ LPWIN32_FIND_DATA lpFindFileData= static_cast<LPWIN32_FIND_DATA>
+ (ut_malloc_nokey(sizeof(WIN32_FIND_DATA)));
+ os_file_dir_t dir= FindFirstFile((LPCTSTR) path, lpFindFileData);
+ ut_free(lpFindFileData);
+
+ return dir;
+}
+#endif
+
+/** This function returns information of the next file in the directory. We jump
+over the '.' and '..' entries in the directory.
+@param[in] dirname directory name or path
+@param[in] dir directory stream
+@param[out] info buffer where the info is returned
+@return 0 if ok, -1 if error, 1 if at the end of the directory */
+int
+os_file_readdir_next_file(
+ const char* dirname,
+ os_file_dir_t dir,
+ os_file_stat_t* info)
+{
+#ifdef _WIN32
+ BOOL ret;
+ int status;
+ WIN32_FIND_DATA find_data;
+
+next_file:
+ ret = FindNextFile(dir, &find_data);
+
+ if (ret > 0) {
+
+ const char* name;
+
+ name = static_cast<const char*>(find_data.cFileName);
+
+ ut_a(strlen(name) < OS_FILE_MAX_PATH);
+
+ if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) {
+
+ goto next_file;
+ }
+
+ strcpy(info->name, name);
+
+ info->size = find_data.nFileSizeHigh;
+ info->size <<= 32;
+ info->size |= find_data.nFileSizeLow;
+
+ if (find_data.dwFileAttributes
+ & FILE_ATTRIBUTE_REPARSE_POINT) {
+
+ /* TODO: test Windows symlinks */
+ /* TODO: MySQL has apparently its own symlink
+ implementation in Windows, dbname.sym can
+ redirect a database directory:
+ REFMAN "windows-symbolic-links.html" */
+
+ info->type = OS_FILE_TYPE_LINK;
+
+ } else if (find_data.dwFileAttributes
+ & FILE_ATTRIBUTE_DIRECTORY) {
+
+ info->type = OS_FILE_TYPE_DIR;
+
+ } else {
+
+ /* It is probably safest to assume that all other
+ file types are normal. Better to check them rather
+ than blindly skip them. */
+
+ info->type = OS_FILE_TYPE_FILE;
+ }
+
+ status = 0;
+
+ } else {
+ DWORD err = GetLastError();
+ if (err == ERROR_NO_MORE_FILES) {
+ status = 1;
+ } else {
+ msg("FindNextFile in %s returned %lu", dirname, err);
+ status = -1;
+ }
+ }
+
+ return(status);
+#else
+ struct dirent* ent;
+ char* full_path;
+ int ret;
+ struct stat statinfo;
+
+next_file:
+
+ ent = readdir(dir);
+
+ if (ent == NULL) {
+
+ return(1);
+ }
+
+ ut_a(strlen(ent->d_name) < OS_FILE_MAX_PATH);
+
+ if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) {
+
+ goto next_file;
+ }
+
+ strcpy(info->name, ent->d_name);
+
+ full_path = static_cast<char*>(
+ ut_malloc_nokey(strlen(dirname) + strlen(ent->d_name) + 10));
+ if (!full_path) {
+ return -1;
+ }
+
+ sprintf(full_path, "%s/%s", dirname, ent->d_name);
+
+ ret = stat(full_path, &statinfo);
+
+ if (ret) {
+
+ if (errno == ENOENT) {
+ /* readdir() returned a file that does not exist,
+ it must have been deleted in the meantime. Do what
+ would have happened if the file was deleted before
+ readdir() - ignore and go to the next entry.
+ If this is the last entry then info->name will still
+ contain the name of the deleted file when this
+ function returns, but this is not an issue since the
+ caller shouldn't be looking at info when end of
+ directory is returned. */
+
+ ut_free(full_path);
+
+ goto next_file;
+ }
+
+ msg("stat %s: Got error %d", full_path, errno);
+
+ ut_free(full_path);
+
+ return(-1);
+ }
+
+ info->size = statinfo.st_size;
+
+ if (S_ISDIR(statinfo.st_mode)) {
+ info->type = OS_FILE_TYPE_DIR;
+ } else if (S_ISLNK(statinfo.st_mode)) {
+ info->type = OS_FILE_TYPE_LINK;
+ } else if (S_ISREG(statinfo.st_mode)) {
+ info->type = OS_FILE_TYPE_FILE;
+ } else {
+ info->type = OS_FILE_TYPE_UNKNOWN;
+ }
+
+ ut_free(full_path);
+ return(0);
+#endif
+}
+
+/***********************************************************************//**
+A fault-tolerant function that tries to read the next file name in the
+directory. We retry 100 times if os_file_readdir_next_file() returns -1. The
+idea is to read as much good data as we can and jump over bad data.
+@return 0 if ok, -1 if error even after the retries, 1 if at the end
+of the directory */
+int
+fil_file_readdir_next_file(
+/*=======================*/
+ dberr_t* err, /*!< out: this is set to DB_ERROR if an error
+ was encountered, otherwise not changed */
+ const char* dirname,/*!< in: directory name or path */
+ os_file_dir_t dir, /*!< in: directory stream */
+ os_file_stat_t* info) /*!< in/out: buffer where the
+ info is returned */
+{
+ for (ulint i = 0; i < 100; i++) {
+ int ret = os_file_readdir_next_file(dirname, dir, info);
+
+ if (ret != -1) {
+
+ return(ret);
+ }
+
+ ib::error() << "os_file_readdir_next_file() returned -1 in"
+ " directory " << dirname
+ << ", crash recovery may have failed"
+ " for some .ibd files!";
+
+ *err = DB_ERROR;
+ }
+
+ return(-1);
+}
+
+/** Scan the database directories under the MySQL datadir, looking for
+.ibd files and determining the space id in each of them.
+@return DB_SUCCESS or error number */
+
+static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback)
+{
+ int ret;
+ char* dbpath = NULL;
+ ulint dbpath_len = 100;
+ os_file_dir_t dir;
+ os_file_dir_t dbdir;
+ os_file_stat_t dbinfo;
+ os_file_stat_t fileinfo;
+ dberr_t err = DB_SUCCESS;
+ size_t len;
+
+ /* The datadir of MySQL is always the default directory of mysqld */
+
+ dir = os_file_opendir(fil_path_to_mysql_datadir);
+
+ if (UNIV_UNLIKELY(dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr))) {
+ msg("cannot open dir %s", fil_path_to_mysql_datadir);
+ return(DB_ERROR);
+ }
+
+ dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));
+
+ /* Scan all directories under the datadir. They are the database
+ directories of MySQL. */
+
+ ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir,
+ &dbinfo);
+ while (ret == 0) {
+
+ /* General tablespaces are always at the first level of the
+ data home dir */
+ if (dbinfo.type != OS_FILE_TYPE_FILE) {
+ const bool is_isl = ends_with(dbinfo.name, ".isl");
+ if (is_isl || ends_with(dbinfo.name,".ibd")) {
+ (*callback)(nullptr, dbinfo.name, is_isl,
+ false, 0);
+ }
+ }
+
+ if (dbinfo.type == OS_FILE_TYPE_FILE
+ || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
+
+ goto next_datadir_item;
+ }
+
+ /* We found a symlink or a directory; try opening it to see
+ if a symlink is a directory */
+
+ len = strlen(fil_path_to_mysql_datadir)
+ + strlen (dbinfo.name) + 2;
+ if (len > dbpath_len) {
+ dbpath_len = len;
+
+ if (dbpath) {
+ ut_free(dbpath);
+ }
+
+ dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));
+ }
+ snprintf(dbpath, dbpath_len,
+ "%s/%s", fil_path_to_mysql_datadir, dbinfo.name);
+
+ if (check_if_skip_database_by_path(dbpath)) {
+ fprintf(stderr, "Skipping db: %s\n", dbpath);
+ goto next_datadir_item;
+ }
+
+ dbdir = os_file_opendir(dbpath);
+
+ if (UNIV_UNLIKELY(dbdir != IF_WIN(INVALID_HANDLE_VALUE,NULL))){
+ /* We found a database directory; loop through it,
+ looking for possible .ibd files in it */
+
+ for (ret = fil_file_readdir_next_file(&err, dbpath,
+ dbdir,
+ &fileinfo);
+ ret == 0;
+ ret = fil_file_readdir_next_file(&err, dbpath,
+ dbdir,
+ &fileinfo)) {
+ if (fileinfo.type == OS_FILE_TYPE_DIR) {
+ continue;
+ }
+
+ /* We found a symlink or a file */
+ if (strlen(fileinfo.name) > 4) {
+ bool is_isl= false;
+ if (ends_with(fileinfo.name, ".ibd") || ((is_isl = ends_with(fileinfo.name, ".isl"))))
+ (*callback)(dbinfo.name, fileinfo.name, is_isl, false, 0);
+ }
+ }
+
+ if (os_file_closedir_failed(dbdir)) {
+ fprintf(stderr, "InnoDB: Warning: could not"
+ " close database directory %s\n",
+ dbpath);
+
+ err = DB_ERROR;
+ }
+
+ } else {
+ msg("Can't open dir %s", dbpath);
+ err = DB_ERROR;
+ break;
+
+ }
+
+next_datadir_item:
+ ret = fil_file_readdir_next_file(&err,
+ fil_path_to_mysql_datadir,
+ dir, &dbinfo);
+ }
+
+ ut_free(dbpath);
+
+ if (os_file_closedir_failed(dir)) {
+ fprintf(stderr,
+ "InnoDB: Error: could not close MariaDB datadir\n");
+ return(DB_ERROR);
+ }
+
+ return(err);
+}
+
+/** Close all undo tablespaces while applying incremental delta */
+static void xb_close_undo_tablespaces()
+{
+ if (srv_undo_space_id_start == 0)
+ return;
+ for (uint32_t space_id= srv_undo_space_id_start;
+ space_id < srv_undo_space_id_start + srv_undo_tablespaces_open;
+ space_id++)
+ {
+ fil_space_t *space= fil_space_get(space_id);
+ ut_ad(space);
+ space->close();
+ }
+}
+
+/****************************************************************************
+Populates the tablespace memory cache by scanning for and opening data files.
+@returns DB_SUCCESS or error code.*/
+static
+dberr_t
+xb_load_tablespaces()
+{
+ bool create_new_db;
+ dberr_t err;
+ ulint sum_of_new_sizes;
+
+ ut_ad(srv_operation == SRV_OPERATION_BACKUP
+ || srv_operation == SRV_OPERATION_RESTORE_DELTA);
+
+ err = srv_sys_space.check_file_spec(&create_new_db, 0);
+
+ /* create_new_db must not be true. */
+ if (err != DB_SUCCESS || create_new_db) {
+ msg("Could not find data files at the specified datadir");
+ return(DB_ERROR);
+ }
+
+ for (int i= 0; i < 10; i++) {
+ err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes);
+ if (err == DB_PAGE_CORRUPTED || err == DB_CORRUPTION) {
+ my_sleep(1000);
+ }
+ else
+ break;
+ }
+
+ if (err != DB_SUCCESS) {
+ msg("Could not open data files.\n");
+ return(err);
+ }
+
+ /* Add separate undo tablespaces to fil_system */
+ err = srv_undo_tablespaces_init(false, nullptr);
+
+ if (err != DB_SUCCESS) {
+ return(err);
+ }
+
+ /* It is important to call xb_load_single_table_tablespaces() after
+ srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() *
+ relies on srv_undo_tablespaces_open to be properly initialized */
+
+ msg("mariabackup: Generating a list of tablespaces");
+
+ err = enumerate_ibd_files(xb_load_single_table_tablespace);
+ if (err != DB_SUCCESS) {
+ return(err);
+ }
+
+ if (srv_operation == SRV_OPERATION_RESTORE_DELTA) {
+ xb_close_undo_tablespaces();
+ }
+
+ DBUG_MARIABACKUP_EVENT("after_load_tablespaces", {});
+ return(DB_SUCCESS);
+}
+
+/** Destroy the tablespace memory cache. */
+static void xb_data_files_close()
+{
+ fil_space_t::close_all();
+ buf_dblwr.close();
+}
+
+/***********************************************************************
+Allocate and initialize the entry for databases and tables filtering
+hash tables. If memory allocation is not successful, terminate program.
+@return pointer to the created entry. */
+static
+xb_filter_entry_t *
+xb_new_filter_entry(
+/*================*/
+ const char* name) /*!< in: name of table/database */
+{
+ xb_filter_entry_t *entry;
+ ulint namelen = strlen(name);
+
+ ut_a(namelen <= NAME_LEN * 2 + 1);
+
+ entry = static_cast<xb_filter_entry_t *>
+ (malloc(sizeof(xb_filter_entry_t) + namelen + 1));
+ memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1);
+ entry->name = ((char*)entry) + sizeof(xb_filter_entry_t);
+ strcpy(entry->name, name);
+ entry->has_tables = FALSE;
+
+ return entry;
+}
+
+/***********************************************************************
+Add entry to hash table. If hash table is NULL, allocate and initialize
+new hash table */
+static
+xb_filter_entry_t*
+xb_add_filter(
+ const char* name, /*!< in: name of table/database */
+ hash_table_t* hash) /*!< in/out: hash to insert into */
+{
+ xb_filter_entry_t* entry = xb_new_filter_entry(name);
+
+ if (UNIV_UNLIKELY(!hash->array)) {
+ hash->create(1000);
+ }
+ const ulint fold = my_crc32c(0, entry->name, strlen(entry->name));
+ HASH_INSERT(xb_filter_entry_t, name_hash, hash, fold, entry);
+ return entry;
+}
+
+/***********************************************************************
+Validate name of table or database. If name is invalid, program will
+be finished with error code */
+static
+void
+xb_validate_name(
+/*=============*/
+ const char* name, /*!< in: name */
+ size_t len) /*!< in: length of name */
+{
+ const char* p;
+
+ /* perform only basic validation. validate length and
+ path symbols */
+ if (len > NAME_LEN) {
+ die("name `%s` is too long.", name);
+ }
+ p = strpbrk(name, "/\\~");
+ if (p && (uint) (p - name) < NAME_LEN) {
+ die("name `%s` is not valid.", name);
+ }
+}
+
+/***********************************************************************
+Register new filter entry which can be either database
+or table name. */
+static
+void
+xb_register_filter_entry(
+/*=====================*/
+ const char* name, /*!< in: name */
+ hash_table_t* databases_hash,
+ hash_table_t* tables_hash
+ )
+{
+ const char* p;
+ size_t namelen;
+ xb_filter_entry_t* db_entry = NULL;
+
+ namelen = strlen(name);
+ if ((p = strchr(name, '.')) != NULL) {
+ char dbname[NAME_LEN + 1];
+
+ xb_validate_name(name, p - name);
+ xb_validate_name(p + 1, namelen - (p - name));
+
+ strncpy(dbname, name, p - name);
+ dbname[p - name] = 0;
+
+ if (databases_hash && databases_hash->array) {
+ const ulint fold = my_crc32c(0, dbname, p - name);
+ HASH_SEARCH(name_hash, databases_hash,
+ fold,
+ xb_filter_entry_t*,
+ db_entry, (void) 0,
+ !strcmp(db_entry->name, dbname));
+ }
+ if (!db_entry) {
+ db_entry = xb_add_filter(dbname, databases_hash);
+ }
+ db_entry->has_tables = TRUE;
+ xb_add_filter(name, tables_hash);
+ } else {
+ xb_validate_name(name, namelen);
+
+ xb_add_filter(name, databases_hash);
+ }
+}
+
+static
+void
+xb_register_include_filter_entry(
+ const char* name
+)
+{
+ xb_register_filter_entry(name, &databases_include_hash,
+ &tables_include_hash);
+}
+
+static
+void
+xb_register_exclude_filter_entry(
+ const char* name
+)
+{
+ xb_register_filter_entry(name, &databases_exclude_hash,
+ &tables_exclude_hash);
+}
+
+void register_ignore_db_dirs_filter(const char *name)
+{
+ xb_add_filter(name, &databases_exclude_hash);
+}
+
+/***********************************************************************
+Register new table for the filter. */
+static
+void
+xb_register_table(
+/*==============*/
+ const char* name) /*!< in: name of table */
+{
+ if (strchr(name, '.') == NULL) {
+ die("`%s` is not fully qualified name.", name);
+ }
+
+ xb_register_include_filter_entry(name);
+}
+
+static
+void
+xb_add_regex_to_list(
+ const char* regex, /*!< in: regex */
+ const char* error_context, /*!< in: context to error message */
+ regex_list_t* list) /*! in: list to put new regex to */
+{
+ char errbuf[100];
+ int ret;
+
+ regex_t compiled_regex;
+ ret = regcomp(&compiled_regex, regex, REG_EXTENDED);
+
+ if (ret != 0) {
+ regerror(ret, &compiled_regex, errbuf, sizeof(errbuf));
+ msg("mariabackup: error: %s regcomp(%s): %s",
+ error_context, regex, errbuf);
+ exit(EXIT_FAILURE);
+ }
+
+ list->push_back(compiled_regex);
+}
+
+/***********************************************************************
+Register new regex for the include filter. */
+static
+void
+xb_register_include_regex(
+/*==============*/
+ const char* regex) /*!< in: regex */
+{
+ xb_add_regex_to_list(regex, "tables", &regex_include_list);
+}
+
+/***********************************************************************
+Register new regex for the exclude filter. */
+static
+void
+xb_register_exclude_regex(
+/*==============*/
+ const char* regex) /*!< in: regex */
+{
+ xb_add_regex_to_list(regex, "tables-exclude", &regex_exclude_list);
+}
+
+typedef void (*insert_entry_func_t)(const char*);
+
+/* Scan string and load filter entries from it.
+@param[in] list string representing a list
+@param[in] delimiters delimiters of entries
+@param[in] ins callback to add entry */
+void xb_load_list_string(char *list, const char *delimiters,
+ insert_entry_func_t ins)
+{
+ char *p;
+ char *saveptr;
+
+ p= strtok_r(list, delimiters, &saveptr);
+ while (p)
+ {
+
+ ins(p);
+
+ p= strtok_r(NULL, delimiters, &saveptr);
+ }
+}
+
+/***********************************************************************
+Scan file and load filter entries from it. */
+static
+void
+xb_load_list_file(
+/*==============*/
+ const char* filename, /*!< in: name of file */
+ insert_entry_func_t ins) /*!< in: callback to add entry */
+{
+ char name_buf[NAME_LEN*2+2];
+ FILE* fp;
+
+ /* read and store the filenames */
+ fp = fopen(filename, "r");
+ if (!fp) {
+ die("Can't open %s",
+ filename);
+ }
+ while (fgets(name_buf, sizeof(name_buf), fp) != NULL) {
+ char* p = strchr(name_buf, '\n');
+ if (p) {
+ *p = '\0';
+ } else {
+ die("`%s...` name is too long", name_buf);
+ }
+
+ ins(name_buf);
+ }
+
+ fclose(fp);
+}
+
+
+static
+void
+xb_filters_init()
+{
+ if (xtrabackup_databases) {
+ xb_load_list_string(xtrabackup_databases, " \t",
+ xb_register_include_filter_entry);
+ }
+
+ if (xtrabackup_databases_file) {
+ xb_load_list_file(xtrabackup_databases_file,
+ xb_register_include_filter_entry);
+ }
+
+ if (xtrabackup_databases_exclude) {
+ xb_load_list_string(xtrabackup_databases_exclude, " \t",
+ xb_register_exclude_filter_entry);
+ }
+
+ if (xtrabackup_tables) {
+ xb_load_list_string(xtrabackup_tables, ",",
+ xb_register_include_regex);
+ }
+
+ if (xtrabackup_tables_file) {
+ xb_load_list_file(xtrabackup_tables_file, xb_register_table);
+ }
+
+ if (xtrabackup_tables_exclude) {
+ xb_load_list_string(xtrabackup_tables_exclude, ",",
+ xb_register_exclude_regex);
+ }
+}
+
+static
+void
+xb_filter_hash_free(hash_table_t* hash)
+{
+ ulint i;
+
+ /* free the hash elements */
+ for (i = 0; i < hash->n_cells; i++) {
+ xb_filter_entry_t* table;
+
+ table = static_cast<xb_filter_entry_t *>
+ (HASH_GET_FIRST(hash, i));
+
+ while (table) {
+ xb_filter_entry_t* prev_table = table;
+
+ table = static_cast<xb_filter_entry_t *>
+ (HASH_GET_NEXT(name_hash, prev_table));
+ const ulint fold = my_crc32c(0, prev_table->name,
+ strlen(prev_table->name));
+ HASH_DELETE(xb_filter_entry_t, name_hash, hash,
+ fold, prev_table);
+ free(prev_table);
+ }
+ }
+
+ hash->free();
+}
+
+static void xb_regex_list_free(regex_list_t* list)
+{
+ while (list->size() > 0) {
+ xb_regfree(&list->front());
+ list->pop_front();
+ }
+}
+
+/************************************************************************
+Destroy table filters for partial backup. */
+static
+void
+xb_filters_free()
+{
+ xb_regex_list_free(&regex_include_list);
+ xb_regex_list_free(&regex_exclude_list);
+
+ if (tables_include_hash.array) {
+ xb_filter_hash_free(&tables_include_hash);
+ }
+
+ if (tables_exclude_hash.array) {
+ xb_filter_hash_free(&tables_exclude_hash);
+ }
+
+ if (databases_include_hash.array) {
+ xb_filter_hash_free(&databases_include_hash);
+ }
+
+ if (databases_exclude_hash.array) {
+ xb_filter_hash_free(&databases_exclude_hash);
+ }
+}
+
+#ifdef RLIMIT_NOFILE
+/**
+Set the open files limit. Based on set_max_open_files().
+@param max_file_limit requested open files limit
+@return the resulting open files limit. May be less or more than the requested
+value. */
+static ulong xb_set_max_open_files(rlim_t max_file_limit)
+{
+ struct rlimit rlimit;
+ rlim_t old_cur;
+
+ if (getrlimit(RLIMIT_NOFILE, &rlimit)) {
+
+ goto end;
+ }
+
+ old_cur = rlimit.rlim_cur;
+
+ if (rlimit.rlim_cur == RLIM_INFINITY) {
+
+ rlimit.rlim_cur = max_file_limit;
+ }
+
+ if (rlimit.rlim_cur >= max_file_limit) {
+
+ max_file_limit = rlimit.rlim_cur;
+ goto end;
+ }
+
+ rlimit.rlim_cur = rlimit.rlim_max = max_file_limit;
+
+ if (setrlimit(RLIMIT_NOFILE, &rlimit)) {
+ /* Use original value */
+ max_file_limit = static_cast<ulong>(old_cur);
+ } else {
+
+ rlimit.rlim_cur = 0; /* Safety if next call fails */
+
+ (void) getrlimit(RLIMIT_NOFILE, &rlimit);
+
+ if (rlimit.rlim_cur) {
+
+ /* If call didn't fail */
+ max_file_limit = rlimit.rlim_cur;
+ }
+ }
+
+end:
+ return static_cast<ulong>(max_file_limit);
+}
+#else
+# define xb_set_max_open_files(x) 0UL
+#endif
+
+static void stop_backup_threads()
+{
+ mysql_cond_broadcast(&log_copying_stop);
+
+ if (log_copying_running || have_io_watching_thread)
+ {
+ mysql_mutex_unlock(&recv_sys.mutex);
+ fputs("mariabackup: Stopping log copying thread", stderr);
+ fflush(stderr);
+ mysql_mutex_lock(&recv_sys.mutex);
+ while (log_copying_running || have_io_watching_thread)
+ {
+ mysql_cond_broadcast(&log_copying_stop);
+ mysql_mutex_unlock(&recv_sys.mutex);
+ putc('.', stderr);
+ fflush(stderr);
+ std::this_thread::sleep_for(std::chrono::milliseconds(200));
+ mysql_mutex_lock(&recv_sys.mutex);
+ }
+ putc('\n', stderr);
+ }
+
+ mysql_cond_destroy(&log_copying_stop);
+}
+
+/** Implement the core of --backup
+@return whether the operation succeeded */
+bool Backup_datasinks::backup_low()
+{
+ mysql_mutex_lock(&recv_sys.mutex);
+ ut_ad(!metadata_to_lsn);
+
+ /* read the latest checkpoint lsn */
+ {
+ const lsn_t lsn = recv_sys.lsn;
+ if (recv_sys.find_checkpoint() == DB_SUCCESS
+ && log_sys.is_latest()) {
+ metadata_to_lsn = log_sys.next_checkpoint_lsn;
+ msg("mariabackup: The latest check point"
+ " (for incremental): '" LSN_PF "'",
+ metadata_to_lsn);
+ } else {
+ msg("Error: recv_sys.find_checkpoint() failed.");
+ }
+
+ recv_sys.lsn = lsn;
+ stop_backup_threads();
+ }
+
+ if (metadata_to_lsn && xtrabackup_copy_logfile()) {
+ mysql_mutex_unlock(&recv_sys.mutex);
+ ds_close(dst_log_file);
+ dst_log_file = NULL;
+ return false;
+ }
+
+ mysql_mutex_unlock(&recv_sys.mutex);
+
+ if (ds_close(dst_log_file) || !metadata_to_lsn) {
+ dst_log_file = NULL;
+ return false;
+ }
+
+ dst_log_file = NULL;
+
+ std::vector<uint32_t> failed_ids;
+ std::set_difference(
+ fail_undo_ids.begin(), fail_undo_ids.end(),
+ undo_trunc_ids.begin(), undo_trunc_ids.end(),
+ std::inserter(failed_ids, failed_ids.begin()));
+
+ for (uint32_t id : failed_ids) {
+ msg("mariabackup: Failed to read undo log "
+ "tablespace space id %d and there is no undo "
+ "tablespace truncation redo record.",
+ id);
+ }
+
+ if (failed_ids.size() > 0) {
+ return false;
+ }
+
+ if (!xtrabackup_incremental) {
+ safe_strcpy(metadata_type, sizeof(metadata_type),
+ "full-backuped");
+ metadata_from_lsn = 0;
+ } else {
+ safe_strcpy(metadata_type, sizeof(metadata_type),
+ "incremental");
+ metadata_from_lsn = incremental_lsn;
+ }
+ metadata_last_lsn = recv_sys.lsn;
+
+ if (!xtrabackup_stream_metadata(m_meta)) {
+ msg("Error: failed to stream metadata.");
+ return false;
+ }
+ if (xtrabackup_extra_lsndir) {
+ char filename[FN_REFLEN];
+
+ sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
+ XTRABACKUP_METADATA_FILENAME);
+ if (!xtrabackup_write_metadata(filename)) {
+ msg("Error: failed to write metadata "
+ "to '%s'.", filename);
+ return false;
+ }
+ sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
+ XTRABACKUP_INFO);
+ if (!write_xtrabackup_info(m_data,
+ mysql_connection, filename, false, false)) {
+ msg("Error: failed to write info "
+ "to '%s'.", filename);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+/** Implement --backup
+@return whether the operation succeeded */
+static bool xtrabackup_backup_func()
+{
+ MY_STAT stat_info;
+ uint i;
+ uint count;
+ pthread_mutex_t count_mutex;
+ CorruptedPages corrupted_pages;
+ data_thread_ctxt_t *data_threads;
+ Backup_datasinks backup_datasinks;
+ pthread_cond_init(&scanned_lsn_cond, NULL);
+
+#ifdef USE_POSIX_FADVISE
+ msg("uses posix_fadvise().");
+#endif
+
+ /* cd to datadir */
+
+ if (my_setwd(mysql_real_data_home,MYF(MY_WME)))
+ {
+ msg("my_setwd() failed , %s", mysql_real_data_home);
+ return(false);
+ }
+ msg("cd to %s", mysql_real_data_home);
+ xb_plugin_backup_init(mysql_connection);
+ msg("open files limit requested %lu, set to %lu",
+ xb_open_files_limit,
+ xb_set_max_open_files(xb_open_files_limit));
+
+ mysql_data_home= mysql_data_home_buff;
+ mysql_data_home[0]=FN_CURLIB; // all paths are relative from here
+ mysql_data_home[1]=0;
+
+ srv_n_purge_threads = 1;
+ srv_read_only_mode = TRUE;
+
+ srv_operation = SRV_OPERATION_BACKUP;
+ log_file_op = backup_file_op;
+ undo_space_trunc = backup_undo_trunc;
+ first_page_init = backup_first_page_op;
+ metadata_to_lsn = 0;
+
+ /* initialize components */
+ if(innodb_init_param()) {
+fail:
+ if (log_copying_running) {
+ mysql_mutex_lock(&recv_sys.mutex);
+ metadata_to_lsn = 1;
+ stop_backup_threads();
+ mysql_mutex_unlock(&recv_sys.mutex);
+ }
+
+ log_file_op = NULL;
+ undo_space_trunc = NULL;
+ first_page_init = NULL;
+ if (dst_log_file) {
+ ds_close(dst_log_file);
+ dst_log_file = NULL;
+ }
+ if (fil_system.is_initialised()) {
+ innodb_shutdown();
+ }
+ return(false);
+ }
+
+ if (srv_buf_pool_size >= 1000 * 1024 * 1024) {
+ /* Here we still have srv_pool_size counted
+ in kilobytes (in 4.0 this was in bytes)
+ srv_boot() converts the value to
+ pages; if buffer pool is less than 1000 MB,
+ assume fewer threads. */
+ srv_max_n_threads = 50000;
+
+ } else if (srv_buf_pool_size >= 8 * 1024 * 1024) {
+
+ srv_max_n_threads = 10000;
+ } else {
+ srv_max_n_threads = 1000; /* saves several MB of memory,
+ especially in 64-bit
+ computers */
+ }
+ srv_thread_pool_init();
+ /* Reset the system variables in the recovery module. */
+ trx_pool_init();
+ recv_sys.create();
+
+ xb_filters_init();
+
+ xb_fil_io_init();
+
+ if (os_aio_init()) {
+ msg("Error: cannot initialize AIO subsystem");
+ goto fail;
+ }
+
+ if (!log_sys.create()) {
+ goto fail;
+ }
+ /* get current checkpoint_lsn */
+ {
+ mysql_mutex_lock(&recv_sys.mutex);
+
+ dberr_t err = recv_sys.find_checkpoint();
+
+ if (err != DB_SUCCESS) {
+ msg("Error: cannot read redo log header");
+ } else if (!log_sys.is_latest()) {
+ msg("Error: cannot process redo log before "
+ "MariaDB 10.8");
+ err = DB_ERROR;
+ } else {
+ recv_needed_recovery = true;
+ }
+ mysql_mutex_unlock(&recv_sys.mutex);
+
+ if (err != DB_SUCCESS) {
+ goto fail;
+ }
+ }
+
+ /* create extra LSN dir if it does not exist. */
+ if (xtrabackup_extra_lsndir
+ &&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0))
+ && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) {
+ msg("Error: cannot mkdir %d: %s\n",
+ my_errno, xtrabackup_extra_lsndir);
+ goto fail;
+ }
+
+ /* create target dir if not exist */
+ if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0))
+ && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){
+ msg("Error: cannot mkdir %d: %s\n",
+ my_errno, xtrabackup_target_dir);
+ goto fail;
+ }
+
+ backup_datasinks.init();
+
+ if (!select_history()) {
+ goto fail;
+ }
+
+ /* open the log file */
+ memset(&stat_info, 0, sizeof(MY_STAT));
+ dst_log_file = ds_open(backup_datasinks.m_redo, LOG_FILE_NAME, &stat_info);
+ if (dst_log_file == NULL) {
+ msg("Error: failed to open the target stream for '%s'.",
+ LOG_FILE_NAME);
+ goto fail;
+ }
+
+ /* label it */
+ recv_sys.file_checkpoint = log_sys.next_checkpoint_lsn;
+ log_hdr_init();
+ /* Write log header*/
+ if (ds_write(dst_log_file, log_hdr_buf, 12288)) {
+ msg("error: write to logfile failed");
+ goto fail;
+ }
+ log_copying_running = true;
+
+ mysql_cond_init(0, &log_copying_stop, nullptr);
+
+ /* start io throttle */
+ if (xtrabackup_throttle) {
+ io_ticket = xtrabackup_throttle;
+ have_io_watching_thread = true;
+ mysql_cond_init(0, &wait_throttle, nullptr);
+ std::thread(io_watching_thread).detach();
+ }
+
+ /* Populate fil_system with tablespaces to copy */
+ if (dberr_t err = xb_load_tablespaces()) {
+ msg("merror: xb_load_tablespaces() failed with"
+ " error %s.", ut_strerr(err));
+ log_copying_running = false;
+ goto fail;
+ }
+
+ /* copy log file by current position */
+
+ mysql_mutex_lock(&recv_sys.mutex);
+ recv_sys.lsn = log_sys.next_checkpoint_lsn;
+
+ const bool log_copy_failed = xtrabackup_copy_logfile();
+
+ mysql_mutex_unlock(&recv_sys.mutex);
+
+ if (log_copy_failed) {
+ log_copying_running = false;
+ goto fail;
+ }
+
+ DBUG_MARIABACKUP_EVENT("before_innodb_log_copy_thread_started", {});
+
+ std::thread(log_copying_thread).detach();
+
+ /* FLUSH CHANGED_PAGE_BITMAPS call */
+ if (!flush_changed_page_bitmaps()) {
+ goto fail;
+ }
+
+ ut_a(xtrabackup_parallel > 0);
+
+ if (xtrabackup_parallel > 1) {
+ msg("mariabackup: Starting %u threads for parallel data "
+ "files transfer", xtrabackup_parallel);
+ }
+
+ if (opt_lock_ddl_per_table) {
+ mdl_lock_all();
+
+ DBUG_EXECUTE_IF("check_mdl_lock_works",
+ dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int",
+ "Waiting for table metadata lock", 0, 0););
+ }
+
+ datafiles_iter_t it;
+
+ /* Create data copying threads */
+ data_threads = (data_thread_ctxt_t *)
+ malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel);
+ count = xtrabackup_parallel;
+ pthread_mutex_init(&count_mutex, NULL);
+
+ for (i = 0; i < (uint) xtrabackup_parallel; i++) {
+ data_threads[i].it = &it;
+ data_threads[i].num = i+1;
+ data_threads[i].count = &count;
+ data_threads[i].count_mutex = &count_mutex;
+ data_threads[i].corrupted_pages = &corrupted_pages;
+ data_threads[i].datasinks= &backup_datasinks;
+ std::thread(data_copy_thread_func, data_threads + i).detach();
+ }
+
+ /* Wait for threads to exit */
+ while (1) {
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ pthread_mutex_lock(&count_mutex);
+ bool stop = count == 0;
+ pthread_mutex_unlock(&count_mutex);
+ if (stop) {
+ break;
+ }
+ }
+
+ pthread_mutex_destroy(&count_mutex);
+ free(data_threads);
+
+ DBUG_ASSERT(backup_datasinks.m_data);
+ DBUG_ASSERT(backup_datasinks.m_meta);
+ bool ok = backup_start(backup_datasinks.m_data,
+ backup_datasinks.m_meta, corrupted_pages);
+
+ if (ok) {
+ ok = backup_datasinks.backup_low();
+
+ backup_release();
+
+ DBUG_EXECUTE_IF("check_mdl_lock_works",
+ pthread_join(dbug_alter_thread, nullptr););
+
+ if (ok) {
+ backup_finish(backup_datasinks.m_data);
+ }
+ }
+
+ if (opt_log_innodb_page_corruption)
+ ok = corrupted_pages.print_to_file(backup_datasinks.m_data,
+ MB_CORRUPTED_PAGES_FILE);
+
+ if (!ok) {
+ goto fail;
+ }
+
+ if (changed_page_bitmap) {
+ xb_page_bitmap_deinit(changed_page_bitmap);
+ }
+ backup_datasinks.destroy();
+
+ msg("Redo log (from LSN " LSN_PF " to " LSN_PF ") was copied.",
+ log_sys.next_checkpoint_lsn, recv_sys.lsn);
+ xb_filters_free();
+
+ xb_data_files_close();
+
+ /* Make sure that the latest checkpoint was included */
+ if (metadata_to_lsn > recv_sys.lsn) {
+ msg("Error: failed to copy enough redo log ("
+ "LSN=" LSN_PF "; checkpoint LSN=" LSN_PF ").",
+ recv_sys.lsn, metadata_to_lsn);
+ goto fail;
+ }
+
+ innodb_shutdown();
+ log_file_op = NULL;
+ undo_space_trunc = NULL;
+ first_page_init = NULL;
+ pthread_cond_destroy(&scanned_lsn_cond);
+ if (!corrupted_pages.empty()) {
+ ut_ad(opt_log_innodb_page_corruption);
+ msg("Error: corrupted innodb pages are found and logged to "
+ MB_CORRUPTED_PAGES_FILE " file");
+ }
+ return(true);
+}
+
+
+/**
+This function handles DDL changes at the end of backup, under protection of
+FTWRL. This ensures consistent backup in presence of DDL.
+
+- New tables, that were created during backup, are now copied into backup.
+ Also, tablespaces with optimized (no redo loggin DDL) are re-copied into
+ backup. This tablespaces will get the extension ".new" in the backup
+
+- Tables that were renamed during backup, are marked as renamed
+ For these, file <old_name>.ren will be created.
+ The content of the file is the new tablespace name.
+
+- Tables that were deleted during backup, are marked as deleted
+ For these , an empty file <name>.del will be created
+
+ It is the responsibility of the prepare phase to deal with .new, .ren, and .del
+ files.
+*/
+void CorruptedPages::backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta)
+{
+ std::set<std::string> dropped_tables;
+ std::map<std::string, std::string> renamed_tables;
+ space_id_to_name_t new_tables;
+
+ /* Disable further DDL on backed up tables (only needed for --no-lock).*/
+ mysql_mutex_lock(&recv_sys.mutex);
+ log_file_op = backup_file_op_fail;
+ mysql_mutex_unlock(&recv_sys.mutex);
+
+ DBUG_MARIABACKUP_EVENT("backup_fix_ddl", {});
+
+ for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin();
+ iter != ddl_tracker.tables_in_backup.end();
+ iter++) {
+
+ const std::string name = iter->second;
+ uint32_t id = iter->first;
+
+ if (ddl_tracker.drops.find(id) != ddl_tracker.drops.end()) {
+ dropped_tables.insert(name);
+ drop_space(id);
+ continue;
+ }
+
+ if (ddl_tracker.id_to_name.find(id) == ddl_tracker.id_to_name.end()) {
+ continue;
+ }
+
+ /* tablespace was affected by DDL. */
+ const std::string new_name = ddl_tracker.id_to_name[id];
+ if (new_name != name) {
+ renamed_tables[name] = new_name;
+ if (opt_log_innodb_page_corruption)
+ rename_space(id, new_name);
+ }
+ }
+
+ /* Find tables that were created during backup (and not removed).*/
+ for(space_id_to_name_t::iterator iter = ddl_tracker.id_to_name.begin();
+ iter != ddl_tracker.id_to_name.end();
+ iter++) {
+
+ uint32_t id = iter->first;
+ std::string name = iter->second;
+
+ if (ddl_tracker.tables_in_backup.find(id) != ddl_tracker.tables_in_backup.end()) {
+ /* already processed above */
+ continue;
+ }
+
+ if (ddl_tracker.drops.find(id) == ddl_tracker.drops.end()
+ && ddl_tracker.deferred_tables.find(id)
+ == ddl_tracker.deferred_tables.end()) {
+ dropped_tables.erase(name);
+ new_tables[id] = name;
+ if (opt_log_innodb_page_corruption)
+ drop_space(id);
+ }
+ }
+
+ // Mark tablespaces for rename
+ for (std::map<std::string, std::string>::iterator iter = renamed_tables.begin();
+ iter != renamed_tables.end(); ++iter) {
+ const std::string old_name = iter->first;
+ std::string new_name = iter->second;
+ DBUG_ASSERT(ds_data);
+ ds_data->backup_file_printf((old_name + ".ren").c_str(), "%s", new_name.c_str());
+ }
+
+ // Mark tablespaces for drop
+ for (std::set<std::string>::iterator iter = dropped_tables.begin();
+ iter != dropped_tables.end();
+ iter++) {
+ const std::string name(*iter);
+ ds_data->backup_file_printf((name + ".del").c_str(), "%s", "");
+ }
+
+ // Load and copy new tables.
+ // Close all datanodes first, reload only new tables.
+ std::vector<fil_node_t *> all_nodes;
+ datafiles_iter_t it;
+ while (fil_node_t *node = datafiles_iter_next(&it)) {
+ all_nodes.push_back(node);
+ }
+ for (size_t i = 0; i < all_nodes.size(); i++) {
+ fil_node_t *n = all_nodes[i];
+ if (n->space->id == 0)
+ continue;
+ if (n->is_open()) {
+ mysql_mutex_lock(&fil_system.mutex);
+ n->close();
+ mysql_mutex_unlock(&fil_system.mutex);
+ }
+ fil_space_free(n->space->id, false);
+ }
+
+ DBUG_EXECUTE_IF("check_mdl_lock_works", DBUG_ASSERT(new_tables.size() == 0););
+
+ srv_operation = SRV_OPERATION_BACKUP_NO_DEFER;
+
+ /* Mariabackup detected the FILE_MODIFY or FILE_RENAME
+ for the deferred tablespace. So it needs to read the
+ tablespace again if innodb doesn't have page0 initialization
+ redo log for it */
+ for (space_id_to_name_t::iterator iter =
+ ddl_tracker.deferred_tables.begin();
+ iter != ddl_tracker.deferred_tables.end();
+ iter++) {
+ if (check_if_skip_table(iter->second.c_str())) {
+ continue;
+ }
+
+ if (first_page_init_ids.find(iter->first)
+ != first_page_init_ids.end()) {
+ new_tables[iter->first] = iter->second.c_str();
+ continue;
+ }
+
+ xb_load_single_table_tablespace(iter->second, false);
+ }
+
+ /* Mariabackup doesn't detect any FILE_OP for the deferred
+ tablespace. There is a possiblity that page0 could've
+ been corrupted persistently in the disk */
+ for (auto space_name: defer_space_names) {
+ if (!check_if_skip_table(space_name.c_str())) {
+ xb_load_single_table_tablespace(
+ space_name, false);
+ }
+ }
+
+ srv_operation = SRV_OPERATION_BACKUP;
+
+ for (const auto &t : new_tables) {
+ if (!check_if_skip_table(t.second.c_str())) {
+ xb_load_single_table_tablespace(t.second, false,
+ t.first);
+ }
+ }
+
+ datafiles_iter_t it2;
+
+ while (fil_node_t *node = datafiles_iter_next(&it2)) {
+ if (!fil_is_user_tablespace_id(node->space->id))
+ continue;
+ std::string dest_name= filename_to_spacename(
+ node->name, strlen(node->name));
+ dest_name.append(".new");
+
+ xtrabackup_copy_datafile(ds_data, ds_meta,
+ node, 0, dest_name.c_str(),
+ wf_write_through, *this);
+ }
+}
+
+/* ================= prepare ================= */
+
+/***********************************************************************
+Generates path to the meta file path from a given path to an incremental .delta
+by replacing trailing ".delta" with ".meta", or returns error if 'delta_path'
+does not end with the ".delta" character sequence.
+@return TRUE on success, FALSE on error. */
+static
+ibool
+get_meta_path(
+ const char *delta_path, /* in: path to a .delta file */
+ char *meta_path) /* out: path to the corresponding .meta
+ file */
+{
+ size_t len = strlen(delta_path);
+
+ if (len <= 6 || strcmp(delta_path + len - 6, ".delta")) {
+ return FALSE;
+ }
+ memcpy(meta_path, delta_path, len - 6);
+ strcpy(meta_path + len - 6, XB_DELTA_INFO_SUFFIX);
+
+ return TRUE;
+}
+
+/****************************************************************//**
+Create a new tablespace on disk and return the handle to its opened
+file. Code adopted from fil_create_new_single_table_tablespace with
+the main difference that only disk file is created without updating
+the InnoDB in-memory dictionary data structures.
+
+@return true on success, false on error. */
+static
+bool
+xb_space_create_file(
+/*==================*/
+ const char* path, /*!<in: path to tablespace */
+ uint32_t space_id, /*!<in: space id */
+ uint32_t flags, /*!<in: tablespace flags */
+ pfs_os_file_t* file) /*!<out: file handle */
+{
+ bool ret;
+
+ *file = os_file_create_simple_no_error_handling(
+ 0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, false, &ret);
+ if (!ret) {
+ msg("Can't create file %s", path);
+ return ret;
+ }
+
+ ret = os_file_set_size(path, *file,
+ FIL_IBD_FILE_INITIAL_SIZE
+ << srv_page_size_shift);
+ if (!ret) {
+ msg("mariabackup: cannot set size for file %s", path);
+ os_file_close(*file);
+ os_file_delete(0, path);
+ return ret;
+ }
+
+ return TRUE;
+}
+
+static fil_space_t* fil_space_get_by_name(const char* name)
+{
+ mysql_mutex_assert_owner(&fil_system.mutex);
+ for (fil_space_t &space : fil_system.space_list)
+ if (space.chain.start)
+ if (const char *str= strstr(space.chain.start->name, name))
+ if (!strcmp(str + strlen(name), ".ibd") &&
+ (str == space.chain.start->name ||
+ IF_WIN(str[-1] == '\\' ||,) str[-1] == '/'))
+ return &space;
+ return nullptr;
+}
+
+/***********************************************************************
+Searches for matching tablespace file for given .delta file and space_id
+in given directory. When matching tablespace found, renames it to match the
+name of .delta file. If there was a tablespace with matching name and
+mismatching ID, renames it to xtrabackup_tmp_#ID.ibd. If there was no
+matching file, creates a new tablespace.
+@return file handle of matched or created file */
+static
+pfs_os_file_t
+xb_delta_open_matching_space(
+ const char* dbname, /* in: path to destination database dir */
+ const char* name, /* in: name of delta file (without .delta) */
+ const xb_delta_info_t& info,
+ char* real_name, /* out: full path of destination file */
+ size_t real_name_len, /* out: buffer size for real_name */
+ bool* success) /* out: indicates error. true = success */
+{
+ char dest_dir[FN_REFLEN];
+ char dest_space_name[FN_REFLEN];
+ fil_space_t* fil_space;
+ pfs_os_file_t file;
+ xb_filter_entry_t* table;
+
+ ut_a(dbname != NULL ||
+ !fil_is_user_tablespace_id(info.space_id) ||
+ info.space_id == UINT32_MAX);
+
+ *success = false;
+
+ if (dbname) {
+ snprintf(dest_dir, FN_REFLEN, "%s/%s",
+ xtrabackup_target_dir, dbname);
+ snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name);
+ } else {
+ snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir);
+ snprintf(dest_space_name, FN_REFLEN, "%s", name);
+ }
+
+ snprintf(real_name, real_name_len,
+ "%s/%s",
+ xtrabackup_target_dir, dest_space_name);
+ /* Truncate ".ibd" */
+ dest_space_name[strlen(dest_space_name) - 4] = '\0';
+
+ /* Create the database directory if it doesn't exist yet */
+ if (!os_file_create_directory(dest_dir, FALSE)) {
+ msg("mariabackup: error: cannot create dir %s", dest_dir);
+ return file;
+ }
+
+ if (!info.space_id && fil_system.sys_space) {
+ fil_node_t *node
+ = UT_LIST_GET_FIRST(fil_system.sys_space->chain);
+ for (; node; node = UT_LIST_GET_NEXT(chain, node)) {
+ if (!strcmp(node->name, real_name)) {
+ break;
+ }
+ }
+ if (node && node->handle != OS_FILE_CLOSED) {
+ *success = true;
+ return node->handle;
+ }
+ msg("mariabackup: Cannot find file %s\n", real_name);
+ return OS_FILE_CLOSED;
+ }
+
+ mysql_mutex_lock(&recv_sys.mutex);
+ if (!fil_is_user_tablespace_id(info.space_id)) {
+found:
+ /* open the file and return its handle */
+
+ file = os_file_create_simple_no_error_handling(
+ 0, real_name,
+ OS_FILE_OPEN, OS_FILE_READ_WRITE, false, success);
+
+ if (!*success) {
+ msg("mariabackup: Cannot open file %s\n", real_name);
+ }
+exit:
+ mysql_mutex_unlock(&recv_sys.mutex);
+ return file;
+ }
+
+ const size_t len = strlen(dest_space_name);
+ /* remember space name for further reference */
+ table = static_cast<xb_filter_entry_t *>
+ (malloc(sizeof(xb_filter_entry_t) +
+ len + 1));
+
+ table->name = ((char*)table) + sizeof(xb_filter_entry_t);
+ memcpy(table->name, dest_space_name, len + 1);
+ const ulint fold = my_crc32c(0, dest_space_name, len);
+ HASH_INSERT(xb_filter_entry_t, name_hash, &inc_dir_tables_hash,
+ fold, table);
+
+ mysql_mutex_lock(&fil_system.mutex);
+ fil_space = fil_space_get_by_name(dest_space_name);
+ mysql_mutex_unlock(&fil_system.mutex);
+
+ if (fil_space != NULL) {
+ if (fil_space->id == info.space_id
+ || info.space_id == UINT32_MAX) {
+ /* we found matching space */
+ goto found;
+ } else {
+
+ char tmpname[FN_REFLEN];
+
+ snprintf(tmpname, FN_REFLEN, "%s/xtrabackup_tmp_#%u",
+ dbname, fil_space->id);
+
+ msg("mariabackup: Renaming %s to %s.ibd",
+ fil_space->chain.start->name, tmpname);
+
+ if (fil_space->rename(tmpname, false) != DB_SUCCESS) {
+ msg("mariabackup: Cannot rename %s to %s",
+ fil_space->chain.start->name, tmpname);
+ goto exit;
+ }
+ }
+ }
+
+ if (info.space_id == UINT32_MAX)
+ {
+ die("Can't handle DDL operation on tablespace "
+ "%s\n", dest_space_name);
+ }
+ mysql_mutex_lock(&fil_system.mutex);
+ fil_space = fil_space_get_by_id(info.space_id);
+ mysql_mutex_unlock(&fil_system.mutex);
+ if (fil_space != NULL) {
+ char tmpname[FN_REFLEN];
+
+ snprintf(tmpname, sizeof tmpname, "%s.ibd", dest_space_name);
+
+ msg("mariabackup: Renaming %s to %s",
+ fil_space->chain.start->name, tmpname);
+
+ if (fil_space->rename(tmpname, false) != DB_SUCCESS) {
+ msg("mariabackup: Cannot rename %s to %s",
+ fil_space->chain.start->name, tmpname);
+ goto exit;
+ }
+
+ goto found;
+ }
+
+ /* No matching space found. create the new one. */
+ const uint32_t flags = info.zip_size
+ ? get_bit_shift(info.page_size
+ >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
+ << FSP_FLAGS_POS_ZIP_SSIZE
+ | FSP_FLAGS_MASK_POST_ANTELOPE
+ | FSP_FLAGS_MASK_ATOMIC_BLOBS
+ | (srv_page_size == UNIV_PAGE_SIZE_ORIG
+ ? 0
+ : get_bit_shift(srv_page_size
+ >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
+ << FSP_FLAGS_POS_PAGE_SSIZE)
+ : FSP_FLAGS_PAGE_SSIZE();
+ ut_ad(fil_space_t::zip_size(flags) == info.zip_size);
+ ut_ad(fil_space_t::physical_size(flags) == info.page_size);
+
+ mysql_mutex_lock(&fil_system.mutex);
+ fil_space_t* space = fil_space_t::create(info.space_id, flags,
+ FIL_TYPE_TABLESPACE, 0,
+ FIL_ENCRYPTION_DEFAULT, true);
+ mysql_mutex_unlock(&fil_system.mutex);
+ if (space) {
+ *success = xb_space_create_file(real_name, info.space_id,
+ flags, &file);
+ } else {
+ msg("Can't create tablespace %s\n", dest_space_name);
+ }
+
+ goto exit;
+}
+
+/************************************************************************
+Applies a given .delta file to the corresponding data file.
+@return TRUE on success */
+static
+ibool
+xtrabackup_apply_delta(
+ const char* dirname, /* in: dir name of incremental */
+ const char* dbname, /* in: database name (ibdata: NULL) */
+ const char* filename, /* in: file name (not a path),
+ including the .delta extension */
+ void* /*data*/)
+{
+ pfs_os_file_t src_file;
+ pfs_os_file_t dst_file;
+ char src_path[FN_REFLEN];
+ char dst_path[FN_REFLEN];
+ char meta_path[FN_REFLEN];
+ char space_name[FN_REFLEN];
+ bool success;
+
+ ibool last_buffer = FALSE;
+ ulint page_in_buffer;
+ ulint incremental_buffers = 0;
+
+ xb_delta_info_t info(srv_page_size, 0, SRV_TMP_SPACE_ID);
+ ulint page_size;
+ ulint page_size_shift;
+ byte* incremental_buffer = NULL;
+
+ size_t offset;
+
+ ut_a(xtrabackup_incremental);
+
+ if (dbname) {
+ snprintf(src_path, sizeof(src_path), "%s/%s/%s",
+ dirname, dbname, filename);
+ snprintf(dst_path, sizeof(dst_path), "%s/%s/%s",
+ xtrabackup_real_target_dir, dbname, filename);
+ } else {
+ snprintf(src_path, sizeof(src_path), "%s/%s",
+ dirname, filename);
+ snprintf(dst_path, sizeof(dst_path), "%s/%s",
+ xtrabackup_real_target_dir, filename);
+ }
+ dst_path[strlen(dst_path) - 6] = '\0';
+
+ strncpy(space_name, filename, FN_REFLEN - 1);
+ space_name[FN_REFLEN - 1] = '\0';
+ space_name[strlen(space_name) - 6] = 0;
+
+ if (!get_meta_path(src_path, meta_path)) {
+ goto error;
+ }
+
+ if (!xb_read_delta_metadata(meta_path, &info)) {
+ goto error;
+ }
+
+ page_size = info.page_size;
+ page_size_shift = get_bit_shift(page_size);
+ msg("page size for %s is %zu bytes",
+ src_path, page_size);
+ if (page_size_shift < 10 ||
+ page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) {
+ msg("error: invalid value of page_size "
+ "(%zu bytes) read from %s", page_size, meta_path);
+ goto error;
+ }
+
+ src_file = os_file_create_simple_no_error_handling(
+ 0, src_path,
+ OS_FILE_OPEN, OS_FILE_READ_WRITE, false, &success);
+ if (!success) {
+ os_file_get_last_error(TRUE);
+ msg("error: can't open %s", src_path);
+ goto error;
+ }
+
+ posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL);
+
+ dst_file = xb_delta_open_matching_space(
+ dbname, space_name, info,
+ dst_path, sizeof(dst_path), &success);
+ if (!success) {
+ msg("error: can't open %s", dst_path);
+ goto error;
+ }
+
+ posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
+
+ /* allocate buffer for incremental backup (4096 pages) */
+ incremental_buffer = static_cast<byte *>
+ (aligned_malloc(page_size / 4 * page_size, page_size));
+
+ msg("Applying %s to %s...", src_path, dst_path);
+
+ while (!last_buffer) {
+ ulint cluster_header;
+
+ /* read to buffer */
+ /* first block of block cluster */
+ offset = ((incremental_buffers * (page_size / 4))
+ << page_size_shift);
+ if (os_file_read(IORequestRead, src_file,
+ incremental_buffer, offset, page_size,
+ nullptr)
+ != DB_SUCCESS) {
+ goto error;
+ }
+
+ cluster_header = mach_read_from_4(incremental_buffer);
+ switch(cluster_header) {
+ case 0x78747261UL: /*"xtra"*/
+ break;
+ case 0x58545241UL: /*"XTRA"*/
+ last_buffer = TRUE;
+ break;
+ default:
+ msg("error: %s seems not "
+ ".delta file.", src_path);
+ goto error;
+ }
+
+ /* FIXME: If the .delta modifies FSP_SIZE on page 0,
+ extend the file to that size. */
+
+ for (page_in_buffer = 1; page_in_buffer < page_size / 4;
+ page_in_buffer++) {
+ if (mach_read_from_4(incremental_buffer + page_in_buffer * 4)
+ == 0xFFFFFFFFUL)
+ break;
+ }
+
+ ut_a(last_buffer || page_in_buffer == page_size / 4);
+
+ /* read whole of the cluster */
+ if (os_file_read(IORequestRead, src_file,
+ incremental_buffer,
+ offset, page_in_buffer * page_size, nullptr)
+ != DB_SUCCESS) {
+ goto error;
+ }
+
+ posix_fadvise(src_file, offset, page_in_buffer * page_size,
+ POSIX_FADV_DONTNEED);
+
+ for (page_in_buffer = 1; page_in_buffer < page_size / 4;
+ page_in_buffer++) {
+ ulint offset_on_page;
+
+ offset_on_page = mach_read_from_4(incremental_buffer + page_in_buffer * 4);
+
+ if (offset_on_page == 0xFFFFFFFFUL)
+ break;
+
+ uchar *buf = incremental_buffer + page_in_buffer * page_size;
+ const os_offset_t off = os_offset_t(offset_on_page)*page_size;
+
+ if (off == 0) {
+ /* Read tablespace size from page 0,
+ and extend the file to specified size.*/
+ os_offset_t n_pages = mach_read_from_4(
+ buf + FSP_HEADER_OFFSET + FSP_SIZE);
+ if (mach_read_from_4(buf
+ + FIL_PAGE_SPACE_ID)) {
+ if (!os_file_set_size(
+ dst_path, dst_file,
+ n_pages * page_size))
+ goto error;
+ } else if (fil_space_t* space
+ = fil_system.sys_space) {
+ /* The system tablespace can
+ consist of multiple files. The
+ first one has full tablespace
+ size in page 0, but only the last
+ file should be extended. */
+ fil_node_t* n = UT_LIST_GET_FIRST(
+ space->chain);
+ bool fail = !strcmp(n->name, dst_path)
+ && !fil_space_extend(
+ space, uint32_t(n_pages));
+ if (fail) goto error;
+ }
+ }
+
+ if (os_file_write(IORequestWrite,
+ dst_path, dst_file, buf, off,
+ page_size) != DB_SUCCESS) {
+ goto error;
+ }
+ }
+
+ /* Free file system buffer cache after the batch was written. */
+#ifdef __linux__
+ os_file_flush_func(dst_file);
+#endif
+ posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
+
+
+ incremental_buffers++;
+ }
+
+ aligned_free(incremental_buffer);
+ if (src_file != OS_FILE_CLOSED) {
+ os_file_close(src_file);
+ os_file_delete(0,src_path);
+ }
+ if (dst_file != OS_FILE_CLOSED && info.space_id)
+ os_file_close(dst_file);
+ return TRUE;
+
+error:
+ aligned_free(incremental_buffer);
+ if (src_file != OS_FILE_CLOSED)
+ os_file_close(src_file);
+ if (dst_file != OS_FILE_CLOSED && info.space_id)
+ os_file_close(dst_file);
+ msg("Error: xtrabackup_apply_delta(): "
+ "failed to apply %s to %s.\n", src_path, dst_path);
+ return FALSE;
+}
+
+
+std::string change_extension(std::string filename, std::string new_ext) {
+ DBUG_ASSERT(new_ext.size() == 3);
+ std::string new_name(filename);
+ new_name.resize(new_name.size() - new_ext.size());
+ new_name.append(new_ext);
+ return new_name;
+}
+
+
+static void rename_file(const char *from,const char *to) {
+ msg("Renaming %s to %s\n", from, to);
+ if (my_rename(from, to, MY_WME)) {
+ die("Can't rename %s to %s errno %d", from, to, errno);
+ }
+}
+
+static void rename_file(const std::string& from, const std::string &to) {
+ rename_file(from.c_str(), to.c_str());
+}
+/************************************************************************
+Callback to handle datadir entry. Function of this type will be called
+for each entry which matches the mask by xb_process_datadir.
+@return should return TRUE on success */
+typedef ibool (*handle_datadir_entry_func_t)(
+/*=========================================*/
+ const char* data_home_dir, /*!<in: path to datadir */
+ const char* db_name, /*!<in: database name */
+ const char* file_name, /*!<in: file name with suffix */
+ void* arg); /*!<in: caller-provided data */
+
+/** Rename, and replace destination file, if exists */
+static void rename_force(const char *from, const char *to) {
+ if (access(to, R_OK) == 0) {
+ msg("Removing %s", to);
+ if (my_delete(to, MYF(MY_WME))) {
+ msg("Can't remove %s, errno %d", to, errno);
+ exit(EXIT_FAILURE);
+ }
+ }
+ rename_file(from,to);
+}
+
+
+/** During prepare phase, rename ".new" files, that were created in
+backup_fix_ddl() and backup_optimized_ddl_op(), to ".ibd". In the case of
+incremental backup, i.e. of arg argument is set, move ".new" files to
+destination directory and rename them to ".ibd", remove existing ".ibd.delta"
+and ".idb.meta" files in incremental directory to avoid applying delta to
+".ibd" file.
+
+@param[in] data_home_dir path to datadir
+@param[in] db_name database name
+@param[in] file_name file name with suffix
+@param[in] arg destination path, used in incremental backup to notify, that
+*.new file must be moved to destibation directory
+
+@return true */
+static ibool prepare_handle_new_files(const char *data_home_dir,
+ const char *db_name,
+ const char *file_name, void *arg)
+{
+ const char *dest_dir = static_cast<const char *>(arg);
+ std::string src_path = std::string(data_home_dir) + '/' + std::string(db_name) + '/';
+ /* Copy "*.new" files from incremental to base dir for incremental backup */
+ std::string dest_path=
+ dest_dir ? std::string(dest_dir) + '/' + std::string(db_name) +
+ '/' : src_path;
+
+ /*
+ A CREATE DATABASE could have happened during the base mariabackup run.
+ In case if the current table file (e.g. `t1.new`) is from such
+ a new database, the database directory may not exist yet in
+ the base backup directory. Let's make sure to check if the directory
+ exists (and create if needed).
+ */
+ if (!directory_exists(dest_path.c_str(), true/*create if not exists*/))
+ return FALSE;
+ src_path+= file_name;
+ dest_path+= file_name;
+
+ size_t index = dest_path.find(".new");
+ DBUG_ASSERT(index != std::string::npos);
+ dest_path.replace(index, strlen(".ibd"), ".ibd");
+ rename_force(src_path.c_str(),dest_path.c_str());
+
+ if (dest_dir) {
+ /* remove delta and meta files to avoid delta applying for new file */
+ index = src_path.find(".new");
+ DBUG_ASSERT(index != std::string::npos);
+ src_path.replace(index, std::string::npos, ".ibd.delta");
+ if (access(src_path.c_str(), R_OK) == 0) {
+ msg("Removing %s", src_path.c_str());
+ if (my_delete(src_path.c_str(), MYF(MY_WME)))
+ die("Can't remove %s, errno %d", src_path.c_str(), errno);
+ }
+ src_path.replace(index, std::string::npos, ".ibd.meta");
+ if (access(src_path.c_str(), R_OK) == 0) {
+ msg("Removing %s", src_path.c_str());
+ if (my_delete(src_path.c_str(), MYF(MY_WME)))
+ die("Can't remove %s, errno %d", src_path.c_str(), errno);
+ }
+
+ /* add table name to the container to avoid it's deletion at the end of
+ prepare */
+ std::string table_name = std::string(db_name) + '/'
+ + std::string(file_name, file_name + strlen(file_name) - strlen(".new"));
+ xb_filter_entry_t *table = static_cast<xb_filter_entry_t *>
+ (malloc(sizeof(xb_filter_entry_t) + table_name.size() + 1));
+ table->name = ((char*)table) + sizeof(xb_filter_entry_t);
+ strcpy(table->name, table_name.c_str());
+ const ulint fold = my_crc32c(0, table->name,
+ table_name.size());
+ HASH_INSERT(xb_filter_entry_t, name_hash, &inc_dir_tables_hash,
+ fold, table);
+ }
+
+ return TRUE;
+}
+
+/************************************************************************
+Callback to handle datadir entry. Deletes entry if it has no matching
+fil_space in fil_system directory.
+@return FALSE if delete attempt was unsuccessful */
+static
+ibool
+rm_if_not_found(
+ const char* data_home_dir, /*!<in: path to datadir */
+ const char* db_name, /*!<in: database name */
+ const char* file_name, /*!<in: file name with suffix */
+ void* arg __attribute__((unused)))
+{
+ char name[FN_REFLEN];
+ xb_filter_entry_t* table;
+
+ snprintf(name, FN_REFLEN, "%s/%s", db_name, file_name);
+ /* Truncate ".ibd" */
+ const size_t len = strlen(name) - 4;
+ name[len] = '\0';
+ const ulint fold = my_crc32c(0, name, len);
+
+ HASH_SEARCH(name_hash, &inc_dir_tables_hash, fold,
+ xb_filter_entry_t*,
+ table, (void) 0,
+ !strcmp(table->name, name));
+
+ if (!table) {
+ snprintf(name, FN_REFLEN, "%s/%s/%s", data_home_dir,
+ db_name, file_name);
+ return os_file_delete(0, name);
+ }
+
+ return(TRUE);
+}
+
+/** Function enumerates files in datadir (provided by path) which are matched
+by provided suffix. For each entry callback is called.
+
+@param[in] path datadir path
+@param[in] suffix suffix to match against
+@param[in] func callback
+@param[in] func_arg arguments for the above callback
+
+@return FALSE if callback for some entry returned FALSE */
+static ibool xb_process_datadir(const char *path, const char *suffix,
+ handle_datadir_entry_func_t func,
+ void *func_arg = NULL)
+{
+ ulint ret;
+ char dbpath[OS_FILE_MAX_PATH+2];
+ os_file_dir_t dir;
+ os_file_dir_t dbdir;
+ os_file_stat_t dbinfo;
+ os_file_stat_t fileinfo;
+ ulint suffix_len;
+ dberr_t err = DB_SUCCESS;
+ static char current_dir[2];
+
+ current_dir[0] = FN_CURLIB;
+ current_dir[1] = 0;
+ srv_data_home = current_dir;
+
+ suffix_len = strlen(suffix);
+
+ /* datafile */
+ dbdir = os_file_opendir(path);
+ if (UNIV_UNLIKELY(dbdir != IF_WIN(INVALID_HANDLE_VALUE, nullptr))) {
+ ret = fil_file_readdir_next_file(&err, path, dbdir, &fileinfo);
+ while (ret == 0) {
+ if (fileinfo.type == OS_FILE_TYPE_DIR) {
+ goto next_file_item_1;
+ }
+
+ if (strlen(fileinfo.name) > suffix_len
+ && 0 == strcmp(fileinfo.name +
+ strlen(fileinfo.name) - suffix_len,
+ suffix)) {
+ if (!func(
+ path, NULL,
+ fileinfo.name, func_arg))
+ {
+ os_file_closedir(dbdir);
+ return(FALSE);
+ }
+ }
+next_file_item_1:
+ ret = fil_file_readdir_next_file(&err,
+ path, dbdir,
+ &fileinfo);
+ }
+
+ os_file_closedir(dbdir);
+ } else {
+ msg("Can't open dir %s", path);
+ }
+
+ /* single table tablespaces */
+ dir = os_file_opendir(path);
+
+ if (UNIV_UNLIKELY(dbdir == IF_WIN(INVALID_HANDLE_VALUE, nullptr))) {
+ msg("Can't open dir %s", path);
+ return TRUE;
+ }
+
+ ret = fil_file_readdir_next_file(&err, path, dir, &dbinfo);
+ while (ret == 0) {
+ if (dbinfo.type == OS_FILE_TYPE_FILE
+ || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
+
+ goto next_datadir_item;
+ }
+
+ snprintf(dbpath, sizeof(dbpath), "%.*s/%.*s",
+ OS_FILE_MAX_PATH/2-1,
+ path,
+ OS_FILE_MAX_PATH/2-1,
+ dbinfo.name);
+
+ dbdir = os_file_opendir(dbpath);
+
+ if (dbdir != IF_WIN(INVALID_HANDLE_VALUE, nullptr)) {
+ ret = fil_file_readdir_next_file(&err, dbpath, dbdir,
+ &fileinfo);
+ while (ret == 0) {
+
+ if (fileinfo.type == OS_FILE_TYPE_DIR) {
+
+ goto next_file_item_2;
+ }
+
+ if (strlen(fileinfo.name) > suffix_len
+ && 0 == strcmp(fileinfo.name +
+ strlen(fileinfo.name) -
+ suffix_len,
+ suffix)) {
+ /* The name ends in suffix; process
+ the file */
+ if (!func(
+ path,
+ dbinfo.name,
+ fileinfo.name, func_arg))
+ {
+ os_file_closedir(dbdir);
+ os_file_closedir(dir);
+ return(FALSE);
+ }
+ }
+next_file_item_2:
+ ret = fil_file_readdir_next_file(&err,
+ dbpath, dbdir,
+ &fileinfo);
+ }
+
+ os_file_closedir(dbdir);
+ }
+next_datadir_item:
+ ret = fil_file_readdir_next_file(&err,
+ path,
+ dir, &dbinfo);
+ }
+
+ os_file_closedir(dir);
+
+ return(TRUE);
+}
+
+/************************************************************************
+Applies all .delta files from incremental_dir to the full backup.
+@return TRUE on success. */
+static
+ibool
+xtrabackup_apply_deltas()
+{
+ return xb_process_datadir(xtrabackup_incremental_dir, ".delta",
+ xtrabackup_apply_delta);
+}
+
+
+static
+void
+innodb_free_param()
+{
+ srv_sys_space.shutdown();
+ free_tmpdir(&mysql_tmpdir_list);
+}
+
+
+/** Check if file exists*/
+static bool file_exists(std::string name)
+{
+ return access(name.c_str(), R_OK) == 0 ;
+}
+
+/** Read file content into STL string */
+static std::string read_file_as_string(const std::string file) {
+ char content[FN_REFLEN];
+ FILE *f = fopen(file.c_str(), "r");
+ if (!f) {
+ msg("Can not open %s", file.c_str());
+ }
+ size_t len = fread(content, 1, FN_REFLEN, f);
+ fclose(f);
+ return std::string(content, len);
+}
+
+/** Delete file- Provide verbose diagnostics and exit, if operation fails. */
+static void delete_file(const std::string& file, bool if_exists = false) {
+ if (if_exists && !file_exists(file))
+ return;
+ if (my_delete(file.c_str(), MYF(MY_WME))) {
+ die("Can't remove %s, errno %d", file.c_str(), errno);
+ }
+}
+
+/**
+Rename tablespace during prepare.
+Backup in its end phase may generate some .ren files, recording
+tablespaces that should be renamed in --prepare.
+*/
+static void rename_table_in_prepare(const std::string &datadir, const std::string& from , const std::string& to,
+ const char *extension=0) {
+ if (!extension) {
+ static const char *extensions_nonincremental[] = { ".ibd", 0 };
+ static const char *extensions_incremental[] = { ".ibd.delta", ".ibd.meta", 0 };
+ const char **extensions = xtrabackup_incremental_dir ?
+ extensions_incremental : extensions_nonincremental;
+ for (size_t i = 0; extensions[i]; i++) {
+ rename_table_in_prepare(datadir, from, to, extensions[i]);
+ }
+ return;
+ }
+ std::string src = std::string(datadir) + "/" + from + extension;
+ std::string dest = std::string(datadir) + "/" + to + extension;
+ std::string ren2, tmp;
+ if (file_exists(dest)) {
+ ren2= std::string(datadir) + "/" + to + ".ren";
+ if (!file_exists(ren2)) {
+ die("ERROR : File %s was not found, but expected during rename processing\n", ren2.c_str());
+ }
+ tmp = to + "#";
+ rename_table_in_prepare(datadir, to, tmp);
+ }
+ rename_file(src, dest);
+ if (ren2.size()) {
+ // Make sure the temp. renamed file is processed.
+ std::string to2 = read_file_as_string(ren2);
+ rename_table_in_prepare(datadir, tmp, to2);
+ delete_file(ren2);
+ }
+}
+
+static ibool prepare_handle_ren_files(const char *datadir, const char *db, const char *filename, void *) {
+
+ std::string ren_file = std::string(datadir) + "/" + db + "/" + filename;
+ if (!file_exists(ren_file))
+ return TRUE;
+
+ std::string to = read_file_as_string(ren_file);
+ std::string source_space_name = std::string(db) + "/" + filename;
+ source_space_name.resize(source_space_name.size() - 4); // remove extension
+
+ rename_table_in_prepare(datadir, source_space_name.c_str(), to.c_str());
+ delete_file(ren_file);
+ return TRUE;
+}
+
+/* Remove tablespaces during backup, based on */
+static ibool prepare_handle_del_files(const char *datadir, const char *db, const char *filename, void *) {
+ std::string del_file = std::string(datadir) + "/" + db + "/" + filename;
+ std::string path(del_file);
+ path.resize(path.size() - 4); // remove extension;
+ if (xtrabackup_incremental) {
+ delete_file(path + ".ibd.delta", true);
+ delete_file(path + ".ibd.meta", true);
+ }
+ else {
+ delete_file(path + ".ibd", true);
+ }
+ delete_file(del_file);
+ return TRUE;
+}
+
+/** Implement --prepare
+@return whether the operation succeeded */
+static bool xtrabackup_prepare_func(char** argv)
+{
+ CorruptedPages corrupted_pages;
+ char metadata_path[FN_REFLEN];
+
+ /* cd to target-dir */
+
+ if (my_setwd(xtrabackup_real_target_dir,MYF(MY_WME)))
+ {
+ msg("can't my_setwd %s", xtrabackup_real_target_dir);
+ return(false);
+ }
+ msg("cd to %s", xtrabackup_real_target_dir);
+
+ fil_path_to_mysql_datadir = ".";
+
+ ut_ad(xtrabackup_incremental == xtrabackup_incremental_dir);
+ if (xtrabackup_incremental)
+ inc_dir_tables_hash.create(1000);
+
+ msg("open files limit requested %u, set to %lu",
+ (uint) xb_open_files_limit,
+ xb_set_max_open_files(xb_open_files_limit));
+
+ /* Fix DDL for prepare. Process .del,.ren, and .new files.
+ The order in which files are processed, is important
+ (see MDEV-18185, MDEV-18201)
+ */
+ xb_process_datadir(xtrabackup_incremental_dir ? xtrabackup_incremental_dir : ".",
+ ".del", prepare_handle_del_files);
+ xb_process_datadir(xtrabackup_incremental_dir? xtrabackup_incremental_dir:".",
+ ".ren", prepare_handle_ren_files);
+ if (xtrabackup_incremental_dir) {
+ xb_process_datadir(xtrabackup_incremental_dir, ".new.meta", prepare_handle_new_files);
+ xb_process_datadir(xtrabackup_incremental_dir, ".new.delta", prepare_handle_new_files);
+ xb_process_datadir(xtrabackup_incremental_dir, ".new",
+ prepare_handle_new_files, (void *)".");
+ }
+ else {
+ xb_process_datadir(".", ".new", prepare_handle_new_files);
+ }
+
+ int argc; for (argc = 0; argv[argc]; argc++) {}
+ xb_plugin_prepare_init(argc, argv, xtrabackup_incremental_dir);
+
+ xtrabackup_target_dir= mysql_data_home_buff;
+ xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here
+ xtrabackup_target_dir[1]=0;
+ const lsn_t target_lsn = xtrabackup_incremental
+ ? incremental_to_lsn : metadata_to_lsn;
+
+ /*
+ read metadata of target
+ */
+ sprintf(metadata_path, "%s/%s", xtrabackup_target_dir,
+ XTRABACKUP_METADATA_FILENAME);
+
+ if (!xtrabackup_read_metadata(metadata_path)) {
+ msg("Error: failed to read metadata from '%s'\n",
+ metadata_path);
+ return(false);
+ }
+
+ if (!strcmp(metadata_type, "full-backuped")) {
+ if (xtrabackup_incremental) {
+ msg("error: applying incremental backup "
+ "needs a prepared target.");
+ return(false);
+ }
+ msg("This target seems to be not prepared yet.");
+ } else if (!strcmp(metadata_type, "log-applied")) {
+ msg("This target seems to be already prepared.");
+ } else {
+ msg("This target does not have correct metadata.");
+ return(false);
+ }
+
+ bool ok = !xtrabackup_incremental
+ || metadata_to_lsn == incremental_lsn;
+ if (!ok) {
+ msg("error: This incremental backup seems "
+ "not to be proper for the target. Check 'to_lsn' of the target and "
+ "'from_lsn' of the incremental.");
+ return(false);
+ }
+
+ srv_max_n_threads = 1000;
+ srv_n_purge_threads = 1;
+
+ xb_filters_init();
+
+ srv_log_group_home_dir = NULL;
+
+ if (xtrabackup_incremental) {
+ srv_operation = SRV_OPERATION_RESTORE_DELTA;
+
+ if (innodb_init_param()) {
+error:
+ ok = false;
+ goto cleanup;
+ }
+
+ recv_sys.create();
+ if (!log_sys.create()) {
+ goto error;
+ }
+ recv_sys.recovery_on = true;
+
+ xb_fil_io_init();
+ if (dberr_t err = xb_load_tablespaces()) {
+ msg("mariabackup: error: xb_data_files_init() failed "
+ "with error %s\n", ut_strerr(err));
+ goto error;
+ }
+
+ ok = fil_system.sys_space->open(false)
+ && xtrabackup_apply_deltas();
+
+ xb_data_files_close();
+
+ if (ok) {
+ /* Cleanup datadir from tablespaces deleted
+ between full and incremental backups */
+
+ xb_process_datadir("./", ".ibd", rm_if_not_found);
+ }
+
+ xb_filter_hash_free(&inc_dir_tables_hash);
+
+ fil_system.close();
+ innodb_free_param();
+ log_sys.close();
+ if (!ok) goto cleanup;
+ }
+
+ srv_operation = xtrabackup_export
+ ? SRV_OPERATION_RESTORE_EXPORT : SRV_OPERATION_RESTORE;
+
+ if (innodb_init_param()) {
+ goto error;
+ }
+
+ fil_system.freeze_space_list = 0;
+
+ msg("Starting InnoDB instance for recovery.");
+
+ msg("mariabackup: Using %lld bytes for buffer pool "
+ "(set by --use-memory parameter)", xtrabackup_use_memory);
+
+ srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct;
+
+ if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) {
+ srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct;
+ }
+
+ if (innodb_init()) {
+ goto error;
+ }
+
+ ut_ad(!fil_system.freeze_space_list);
+
+ corrupted_pages.read_from_file(MB_CORRUPTED_PAGES_FILE);
+ if (xtrabackup_incremental)
+ {
+ char inc_filename[FN_REFLEN];
+ sprintf(inc_filename, "%s/%s", xtrabackup_incremental_dir,
+ MB_CORRUPTED_PAGES_FILE);
+ corrupted_pages.read_from_file(inc_filename);
+ }
+ if (!corrupted_pages.empty())
+ corrupted_pages.zero_out_free_pages();
+ if (corrupted_pages.empty())
+ {
+ if (!xtrabackup_incremental && unlink(MB_CORRUPTED_PAGES_FILE) &&
+ errno != ENOENT)
+ {
+ char errbuf[MYSYS_STRERROR_SIZE];
+ my_strerror(errbuf, sizeof(errbuf), errno);
+ die("Error: unlink %s failed: %s", MB_CORRUPTED_PAGES_FILE,
+ errbuf);
+ }
+ }
+ else
+ corrupted_pages.print_to_file(NULL, MB_CORRUPTED_PAGES_FILE);
+
+ if (ok) {
+ msg("Last binlog file %s, position %lld",
+ trx_sys.recovered_binlog_filename,
+ longlong(trx_sys.recovered_binlog_offset));
+ }
+
+ /* Check whether the log is applied enough or not. */
+ if (recv_sys.lsn && recv_sys.lsn < target_lsn) {
+ msg("mariabackup: error: "
+ "The log was only applied up to LSN " LSN_PF
+ ", instead of " LSN_PF, recv_sys.lsn, target_lsn);
+ ok = false;
+ }
+#ifdef WITH_WSREP
+ else if (ok) xb_write_galera_info(xtrabackup_incremental);
+#endif
+
+ innodb_shutdown();
+
+ innodb_free_param();
+
+ /* output to metadata file */
+ if (ok) {
+ char filename[FN_REFLEN];
+
+ safe_strcpy(metadata_type, sizeof(metadata_type),
+ "log-applied");
+
+ if(xtrabackup_incremental
+ && metadata_to_lsn < incremental_to_lsn)
+ {
+ metadata_to_lsn = incremental_to_lsn;
+ metadata_last_lsn = incremental_last_lsn;
+ }
+
+ sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME);
+ if (!xtrabackup_write_metadata(filename)) {
+
+ msg("mariabackup: Error: failed to write metadata "
+ "to '%s'", filename);
+ ok = false;
+ } else if (xtrabackup_extra_lsndir) {
+ sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME);
+ if (!xtrabackup_write_metadata(filename)) {
+ msg("mariabackup: Error: failed to write "
+ "metadata to '%s'", filename);
+ ok = false;
+ }
+ }
+ }
+
+ if (ok) ok = apply_log_finish();
+
+ if (ok && xtrabackup_export)
+ ok= (prepare_export() == 0);
+
+cleanup:
+ xb_filters_free();
+ return ok && !ib::error::was_logged() && corrupted_pages.empty();
+}
+
+/**************************************************************************
+Append group name to xb_load_default_groups list. */
+static
+void
+append_defaults_group(const char *group, const char *default_groups[],
+ size_t default_groups_size)
+{
+ uint i;
+ bool appended = false;
+ for (i = 0; i < default_groups_size - 1; i++) {
+ if (default_groups[i] == NULL) {
+ default_groups[i] = group;
+ appended = true;
+ break;
+ }
+ }
+ ut_a(appended);
+}
+
+static const char*
+normalize_privilege_target_name(const char* name)
+{
+ if (strcmp(name, "*") == 0) {
+ return "\\*";
+ }
+ else {
+ /* should have no regex special characters. */
+ ut_ad(strpbrk(name, ".()[]*+?") == 0);
+ }
+ return name;
+}
+
+/******************************************************************//**
+Check if specific privilege is granted.
+Uses regexp magic to check if requested privilege is granted for given
+database.table or database.* or *.*
+or if user has 'ALL PRIVILEGES' granted.
+@return true if requested privilege is granted, false otherwise. */
+static bool
+has_privilege(const std::list<std::string> &granted,
+ const char* required,
+ const char* db_name,
+ const char* table_name)
+{
+ char buffer[1000];
+ regex_t priv_re;
+ regmatch_t tables_regmatch[1];
+ bool result = false;
+
+ db_name = normalize_privilege_target_name(db_name);
+ table_name = normalize_privilege_target_name(table_name);
+
+ int written = snprintf(buffer, sizeof(buffer),
+ "GRANT .*(%s)|(ALL PRIVILEGES).* ON (\\*|`%s`)\\.(\\*|`%s`)",
+ required, db_name, table_name);
+ if (written < 0 || written == sizeof(buffer)
+ || regcomp(&priv_re, buffer, REG_EXTENDED)) {
+ die("regcomp() failed for '%s'", buffer);
+ }
+
+ typedef std::list<std::string>::const_iterator string_iter;
+ for (string_iter i = granted.begin(), e = granted.end(); i != e; ++i) {
+ int res = regexec(&priv_re, i->c_str(),
+ 1, tables_regmatch, 0);
+
+ if (res != REG_NOMATCH) {
+ result = true;
+ break;
+ }
+ }
+
+ xb_regfree(&priv_re);
+ return result;
+}
+
+enum {
+ PRIVILEGE_OK = 0,
+ PRIVILEGE_WARNING = 1,
+ PRIVILEGE_ERROR = 2,
+};
+
+/******************************************************************//**
+Check if specific privilege is granted.
+Prints error message if required privilege is missing.
+@return PRIVILEGE_OK if requested privilege is granted, error otherwise. */
+static
+int check_privilege(
+ const std::list<std::string> &granted_priv, /* in: list of
+ granted privileges*/
+ const char* required, /* in: required privilege name */
+ const char* target_database, /* in: required privilege target
+ database name */
+ const char* target_table, /* in: required privilege target
+ table name */
+ int error = PRIVILEGE_ERROR) /* in: return value if privilege
+ is not granted */
+{
+ if (!has_privilege(granted_priv,
+ required, target_database, target_table)) {
+ msg("%s: missing required privilege %s on %s.%s",
+ (error == PRIVILEGE_ERROR ? "Error" : "Warning"),
+ required, target_database, target_table);
+ return error;
+ }
+ return PRIVILEGE_OK;
+}
+
+
+/**
+Check DB user privileges according to the intended actions.
+
+Fetches DB user privileges, determines intended actions based on
+command-line arguments and prints missing privileges.
+@return whether all the necessary privileges are granted */
+static bool check_all_privileges()
+{
+ if (!mysql_connection) {
+ /* Not connected, no queries is going to be executed. */
+ return true;
+ }
+
+ /* Fetch effective privileges. */
+ std::list<std::string> granted_privileges;
+ MYSQL_RES* result = xb_mysql_query(mysql_connection, "SHOW GRANTS",
+ true);
+ while (MYSQL_ROW row = mysql_fetch_row(result)) {
+ granted_privileges.push_back(*row);
+ }
+ mysql_free_result(result);
+
+ int check_result = PRIVILEGE_OK;
+
+ /* FLUSH TABLES WITH READ LOCK */
+ if (!opt_no_lock)
+ {
+ check_result |= check_privilege(
+ granted_privileges,
+ "RELOAD", "*", "*");
+ check_result |= check_privilege(
+ granted_privileges,
+ "PROCESS", "*", "*");
+ }
+
+ /* KILL ... */
+ if (!opt_no_lock && (opt_kill_long_queries_timeout || opt_kill_long_query_type)) {
+ check_result |= check_privilege(
+ granted_privileges,
+ "CONNECTION ADMIN", "*", "*",
+ PRIVILEGE_WARNING);
+ }
+
+ /* START SLAVE SQL_THREAD */
+ /* STOP SLAVE SQL_THREAD */
+ if (opt_safe_slave_backup) {
+ check_result |= check_privilege(
+ granted_privileges,
+ "REPLICATION SLAVE ADMIN", "*", "*",
+ PRIVILEGE_WARNING);
+ }
+
+ /* SHOW MASTER STATUS */
+ /* SHOW SLAVE STATUS */
+ if (opt_galera_info || opt_slave_info
+ || opt_safe_slave_backup) {
+ check_result |= check_privilege(granted_privileges,
+ "SLAVE MONITOR", "*", "*",
+ PRIVILEGE_WARNING);
+ }
+
+ if (check_result & PRIVILEGE_ERROR) {
+ msg("Current privileges, as reported by 'SHOW GRANTS': ");
+ int n=1;
+ for (std::list<std::string>::const_iterator it = granted_privileges.begin();
+ it != granted_privileges.end();
+ it++,n++) {
+ msg(" %d.%s", n, it->c_str());
+ }
+ return false;
+ }
+
+ return true;
+}
+
+bool
+xb_init()
+{
+ const char *mixed_options[4] = {NULL, NULL, NULL, NULL};
+ int n_mixed_options;
+
+ /* sanity checks */
+
+ if (opt_slave_info
+ && opt_no_lock
+ && !opt_safe_slave_backup) {
+ msg("Error: --slave-info is used with --no-lock but "
+ "without --safe-slave-backup. The binlog position "
+ "cannot be consistent with the backup data.");
+ return(false);
+ }
+
+ if (xtrabackup_backup && opt_rsync)
+ {
+ if (xtrabackup_stream_fmt)
+ {
+ msg("Error: --rsync doesn't work with --stream\n");
+ return(false);
+ }
+ bool have_rsync = IF_WIN(false, (system("rsync --version > /dev/null 2>&1") == 0));
+ if (!have_rsync)
+ {
+ msg("Error: rsync executable not found, cannot run backup with --rsync\n");
+ return false;
+ }
+ }
+
+ n_mixed_options = 0;
+
+ if (opt_decompress) {
+ mixed_options[n_mixed_options++] = "--decompress";
+ }
+
+ if (xtrabackup_copy_back) {
+ mixed_options[n_mixed_options++] = "--copy-back";
+ }
+
+ if (xtrabackup_move_back) {
+ mixed_options[n_mixed_options++] = "--move-back";
+ }
+
+ if (xtrabackup_prepare) {
+ mixed_options[n_mixed_options++] = "--apply-log";
+ }
+
+ if (n_mixed_options > 1) {
+ msg("Error: %s and %s are mutually exclusive\n",
+ mixed_options[0], mixed_options[1]);
+ return(false);
+ }
+
+ if (xtrabackup_backup) {
+ if ((mysql_connection = xb_mysql_connect()) == NULL) {
+ return(false);
+ }
+
+ if (!get_mysql_vars(mysql_connection)) {
+ return(false);
+ }
+
+ if (opt_check_privileges && !check_all_privileges()) {
+ return(false);
+ }
+ history_start_time = time(NULL);
+ }
+
+ return(true);
+}
+
+
+extern void init_signals(void);
+
+#include <sql_locale.h>
+
+
+void setup_error_messages()
+{
+ my_default_lc_messages = &my_locale_en_US;
+ if (init_errmessage())
+ die("could not initialize error messages");
+}
+
+/** Handle mariabackup options. The options are handled with the following
+order:
+
+1) Load server groups and process server options, ignore unknown options
+2) Load client groups and process client options, ignore unknown options
+3) Load backup groups and process client-server options, exit on unknown option
+4) Process --mysqld-args options, ignore unknown options
+
+@param[in] argc arguments count
+@param[in] argv arguments array
+@param[out] argv_server server options including loaded from server groups
+@param[out] argv_client client options including loaded from client groups
+@param[out] argv_backup backup options including loaded from backup groups */
+void handle_options(int argc, char **argv, char ***argv_server,
+ char ***argv_client, char ***argv_backup)
+{
+ /* Setup some variables for Innodb.*/
+ srv_operation = SRV_OPERATION_RESTORE;
+
+ files_charset_info = &my_charset_utf8mb3_general_ci;
+
+
+ setup_error_messages();
+ sys_var_init();
+ plugin_mutex_init();
+ mysql_prlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash);
+ opt_stack_trace = 1;
+ test_flags |= TEST_SIGINT;
+ init_signals();
+#ifndef _WIN32
+ /* Exit process on SIGINT. */
+ my_sigset(SIGINT, SIG_DFL);
+#endif
+
+ sf_leaking_memory = 1; /* don't report memory leaks on early exist */
+
+ int i;
+ int ho_error;
+
+ char* target_dir = NULL;
+ bool prepare = false;
+
+ char conf_file[FN_REFLEN];
+
+ // array_elements() will not work for load_defaults, as it is defined
+ // as external symbol, so let's use dynamic array to have ability to
+ // add new server default groups
+ std::vector<const char *> server_default_groups;
+
+ for (const char **default_group= load_default_groups; *default_group;
+ ++default_group)
+ server_default_groups.push_back(*default_group);
+
+ std::vector<char *> mysqld_args;
+ std::vector<char *> mariabackup_args;
+ mysqld_args.push_back(argv[0]);
+ mariabackup_args.push_back(argv[0]);
+
+ /* scan options for group and config file to load defaults from */
+ for (i= 1; i < argc; i++)
+ {
+ char *optend= strcend(argv[i], '=');
+ if (mysqld_args.size() > 1 ||
+ strncmp(argv[i], "--mysqld-args", optend - argv[i]) == 0)
+ {
+ mysqld_args.push_back(argv[i]);
+ continue;
+ }
+ else
+ mariabackup_args.push_back(argv[i]);
+
+ if (strncmp(argv[i], "--defaults-group", optend - argv[i]) == 0)
+ {
+ defaults_group= optend + 1;
+ server_default_groups.push_back(defaults_group);
+ }
+ else if (strncmp(argv[i], "--login-path", optend - argv[i]) == 0)
+ {
+ append_defaults_group(optend + 1, xb_client_default_groups,
+ array_elements(xb_client_default_groups));
+ }
+ else if (!strncmp(argv[i], "--prepare", optend - argv[i]))
+ {
+ prepare= true;
+ }
+ else if (!strncmp(argv[i], "--apply-log", optend - argv[i]))
+ {
+ prepare= true;
+ }
+ else if (!strncmp(argv[i], "--incremental-dir", optend - argv[i]) &&
+ *optend)
+ {
+ target_dir= optend + 1;
+ }
+ else if (!strncmp(argv[i], "--target-dir", optend - argv[i]) &&
+ *optend && !target_dir)
+ {
+ target_dir= optend + 1;
+ }
+ else if (!*optend && argv[i][0] != '-' && !target_dir)
+ {
+ target_dir= argv[i];
+ }
+ }
+
+ server_default_groups.push_back(NULL);
+ snprintf(conf_file, sizeof(conf_file), "my");
+
+ if (prepare && target_dir) {
+ snprintf(conf_file, sizeof(conf_file),
+ "%s/backup-my.cnf", target_dir);
+ if (!strncmp(argv[1], "--defaults-file=", 16)) {
+ /* Remove defaults-file*/
+ for (int i = 2; ; i++) {
+ if ((argv[i-1]= argv[i]) == 0)
+ break;
+ }
+ argc--;
+ }
+ }
+
+ mariabackup_args.push_back(nullptr);
+ *argv_client= *argv_server= *argv_backup= &mariabackup_args[0];
+ int argc_backup= static_cast<int>(mariabackup_args.size() - 1);
+ int argc_client= argc_backup;
+ int argc_server= argc_backup;
+
+ /* 1) Load server groups and process server options, ignore unknown
+ options */
+
+ load_defaults_or_exit(conf_file, &server_default_groups[0],
+ &argc_server, argv_server);
+
+ int n;
+ for (n = 0; (*argv_server)[n]; n++) {};
+ argc_server = n;
+
+ print_param_str <<
+ "# This MySQL options file was generated by XtraBackup.\n"
+ "[" << defaults_group << "]\n";
+
+ /* We want xtrabackup to ignore unknown options, because it only
+ recognizes a small subset of server variables */
+ my_getopt_skip_unknown = TRUE;
+
+ /* Reset u_max_value for all options, as we don't want the
+ --maximum-... modifier to set the actual option values */
+ for (my_option *optp= xb_server_options; optp->name; optp++) {
+ optp->u_max_value = (G_PTR *) &global_max_value;
+ }
+
+ /* Throw a descriptive error if --defaults-file or --defaults-extra-file
+ is not the first command line argument */
+ for (int i = 2 ; i < argc ; i++) {
+ char *optend = strcend((argv)[i], '=');
+
+ if (optend - argv[i] == 15 &&
+ !strncmp(argv[i], "--defaults-file", optend - argv[i])) {
+ die("--defaults-file must be specified first on the command line");
+ }
+ if (optend - argv[i] == 21 &&
+ !strncmp(argv[i], "--defaults-extra-file",
+ optend - argv[i])) {
+ die("--defaults-extra-file must be specified first on the command line");
+ }
+ }
+
+ if (argc_server > 0
+ && (ho_error=handle_options(&argc_server, argv_server,
+ xb_server_options, xb_get_one_option)))
+ exit(ho_error);
+
+ /* 2) Load client groups and process client options, ignore unknown
+ options */
+
+ load_defaults_or_exit(conf_file, xb_client_default_groups,
+ &argc_client, argv_client);
+
+ for (n = 0; (*argv_client)[n]; n++) {};
+ argc_client = n;
+
+ if (innobackupex_mode && argc_client > 0) {
+ if (!ibx_handle_options(&argc_client, argv_client)) {
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ if (argc_client > 0
+ && (ho_error=handle_options(&argc_client, argv_client,
+ xb_client_options, xb_get_one_option)))
+ exit(ho_error);
+
+ /* 3) Load backup groups and process client-server options, exit on
+ unknown option */
+
+ load_defaults_or_exit(conf_file, backup_default_groups, &argc_backup,
+ argv_backup);
+ for (n= 0; (*argv_backup)[n]; n++)
+ {
+ };
+ argc_backup= n;
+
+ my_handle_options_init_variables = FALSE;
+
+ if (argc_backup > 0 &&
+ (ho_error= handle_options(&argc_backup, argv_backup,
+ xb_server_options, xb_get_one_option)))
+ exit(ho_error);
+
+ /* Add back the program name handle_options removes */
+ ++argc_backup;
+ --(*argv_backup);
+
+ if (innobackupex_mode && argc_backup > 0 &&
+ !ibx_handle_options(&argc_backup, argv_backup))
+ exit(EXIT_FAILURE);
+
+ my_getopt_skip_unknown = FALSE;
+
+ if (argc_backup > 0 &&
+ (ho_error= handle_options(&argc_backup, argv_backup,
+ xb_client_options, xb_get_one_option)))
+ exit(ho_error);
+
+ if (opt_password)
+ {
+ char *argument= (char*) opt_password;
+ char *start= (char*) opt_password;
+ opt_password= my_strdup(PSI_NOT_INSTRUMENTED, opt_password,
+ MYF(MY_FAE));
+ while (*argument)
+ *argument++= 'x'; // Destroy argument
+ if (*start)
+ start[1]= 0;
+ }
+
+ /* 4) Process --mysqld-args options, ignore unknown options */
+
+ my_getopt_skip_unknown = TRUE;
+
+ int argc_mysqld = static_cast<int>(mysqld_args.size());
+ if (argc_mysqld > 1)
+ {
+ char **argv_mysqld= &mysqld_args[0];
+ if ((ho_error= handle_options(&argc_mysqld, &argv_mysqld,
+ xb_server_options, xb_get_one_option)))
+ exit(ho_error);
+ }
+
+ my_handle_options_init_variables = TRUE;
+
+ /* Reject command line arguments that don't look like options, i.e. are
+ not of the form '-X' (single-character options) or '--option' (long
+ options) */
+ for (int i = 0 ; i < argc_backup ; i++) {
+ const char * const opt = (*argv_backup)[i];
+
+ if (strncmp(opt, "--", 2) &&
+ !(strlen(opt) == 2 && opt[0] == '-')) {
+ bool server_option = true;
+
+ for (int j = 0; j < argc_backup; j++) {
+ if (opt == (*argv_backup)[j]) {
+ server_option = false;
+ break;
+ }
+ }
+
+ if (!server_option) {
+ msg("mariabackup: Error:"
+ " unknown argument: '%s'", opt);
+ exit(EXIT_FAILURE);
+ }
+ }
+ }
+}
+
+static int main_low(char** argv);
+static int get_exepath(char *buf, size_t size, const char *argv0);
+
+/* ================= main =================== */
+int main(int argc, char **argv)
+{
+ char **server_defaults;
+ char **client_defaults;
+ char **backup_defaults;
+
+ my_getopt_prefix_matching= 0;
+
+ if (get_exepath(mariabackup_exe,FN_REFLEN, argv[0]))
+ strncpy(mariabackup_exe,argv[0], FN_REFLEN-1);
+
+
+ if (argc > 1 )
+ {
+ /* In "prepare export", we need to start mysqld
+ Since it is not always be installed on the machine,
+ we start "mariabackup --mysqld", which acts as mysqld
+ */
+ if (strcmp(argv[1], "--mysqld") == 0)
+ {
+ srv_operation= SRV_OPERATION_EXPORT_RESTORED;
+ extern int mysqld_main(int argc, char **argv);
+ argc--;
+ argv++;
+ argv[0]+=2;
+ return mysqld_main(argc, argv);
+ }
+ if(strcmp(argv[1], "--innobackupex") == 0)
+ {
+ argv++;
+ argc--;
+ innobackupex_mode = true;
+ }
+ }
+
+ if (argc > 1)
+ strncpy(orig_argv1,argv[1],sizeof(orig_argv1) -1);
+
+ init_signals();
+ MY_INIT(argv[0]);
+
+ xb_regex_init();
+
+ capture_tool_command(argc, argv);
+
+ if (mysql_server_init(-1, NULL, NULL))
+ {
+ die("mysql_server_init() failed");
+ }
+
+ system_charset_info = &my_charset_utf8mb3_general_ci;
+ key_map_full.set_all();
+
+ logger.init_base();
+ logger.set_handlers(LOG_NONE, LOG_NONE);
+ mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log,
+ MY_MUTEX_INIT_FAST);
+
+ handle_options(argc, argv, &server_defaults, &client_defaults,
+ &backup_defaults);
+
+#ifndef DBUG_OFF
+ if (dbug_option) {
+ DBUG_SET_INITIAL(dbug_option);
+ DBUG_SET(dbug_option);
+ }
+#endif
+ /* Main functions for library */
+ init_thr_timer(5);
+
+ int status = main_low(server_defaults);
+
+ end_thr_timer();
+ backup_cleanup();
+
+ if (innobackupex_mode) {
+ ibx_cleanup();
+ }
+
+ free_defaults(server_defaults);
+ free_defaults(client_defaults);
+ free_defaults(backup_defaults);
+
+#ifndef DBUG_OFF
+ if (dbug_option) {
+ DBUG_END();
+ }
+#endif
+
+ logger.cleanup_base();
+ cleanup_errmsgs();
+ free_error_messages();
+ mysql_mutex_destroy(&LOCK_error_log);
+
+ if (status == EXIT_SUCCESS) {
+ msg("completed OK!");
+ }
+
+ return status;
+}
+
+static int main_low(char** argv)
+{
+ if (innobackupex_mode) {
+ if (!ibx_init()) {
+ return(EXIT_FAILURE);
+ }
+ }
+
+ if (!xtrabackup_print_param && !xtrabackup_prepare
+ && !strcmp(mysql_data_home, "./")) {
+ if (!xtrabackup_print_param)
+ usage();
+ msg("mariabackup: Error: Please set parameter 'datadir'");
+ return(EXIT_FAILURE);
+ }
+
+ /* Expand target-dir, incremental-basedir, etc. */
+
+ char cwd[FN_REFLEN];
+ my_getwd(cwd, sizeof(cwd), MYF(0));
+
+ my_load_path(xtrabackup_real_target_dir,
+ xtrabackup_target_dir, cwd);
+ unpack_dirname(xtrabackup_real_target_dir,
+ xtrabackup_real_target_dir);
+ xtrabackup_target_dir= xtrabackup_real_target_dir;
+
+ if (xtrabackup_incremental_basedir) {
+ my_load_path(xtrabackup_real_incremental_basedir,
+ xtrabackup_incremental_basedir, cwd);
+ unpack_dirname(xtrabackup_real_incremental_basedir,
+ xtrabackup_real_incremental_basedir);
+ xtrabackup_incremental_basedir =
+ xtrabackup_real_incremental_basedir;
+ }
+
+ if (xtrabackup_incremental_dir) {
+ my_load_path(xtrabackup_real_incremental_dir,
+ xtrabackup_incremental_dir, cwd);
+ unpack_dirname(xtrabackup_real_incremental_dir,
+ xtrabackup_real_incremental_dir);
+ xtrabackup_incremental_dir = xtrabackup_real_incremental_dir;
+ }
+
+ if (xtrabackup_extra_lsndir) {
+ my_load_path(xtrabackup_real_extra_lsndir,
+ xtrabackup_extra_lsndir, cwd);
+ unpack_dirname(xtrabackup_real_extra_lsndir,
+ xtrabackup_real_extra_lsndir);
+ xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir;
+ }
+
+ /* get default temporary directory */
+ if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
+ opt_mysql_tmpdir = getenv("TMPDIR");
+#if defined(_WIN32)
+ if (!opt_mysql_tmpdir) {
+ opt_mysql_tmpdir = getenv("TEMP");
+ }
+ if (!opt_mysql_tmpdir) {
+ opt_mysql_tmpdir = getenv("TMP");
+ }
+#endif
+ if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
+ opt_mysql_tmpdir = const_cast<char*>(DEFAULT_TMPDIR);
+ }
+ }
+
+ /* temporary setting of enough size */
+ srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_MAX;
+ srv_page_size = UNIV_PAGE_SIZE_MAX;
+ if (xtrabackup_backup && xtrabackup_incremental) {
+ /* direct specification is only for --backup */
+ /* and the lsn is prior to the other option */
+
+ char* endchar;
+ int error = 0;
+ incremental_lsn = strtoll(xtrabackup_incremental, &endchar, 10);
+ if (*endchar != '\0')
+ error = 1;
+
+ if (error) {
+ msg("mariabackup: value '%s' may be wrong format for "
+ "incremental option.", xtrabackup_incremental);
+ return(EXIT_FAILURE);
+ }
+ } else if (xtrabackup_backup && xtrabackup_incremental_basedir) {
+ char filename[FN_REFLEN];
+
+ sprintf(filename, "%s/%s", xtrabackup_incremental_basedir, XTRABACKUP_METADATA_FILENAME);
+
+ if (!xtrabackup_read_metadata(filename)) {
+ msg("mariabackup: error: failed to read metadata from "
+ "%s", filename);
+ return(EXIT_FAILURE);
+ }
+
+ incremental_lsn = metadata_to_lsn;
+ xtrabackup_incremental = xtrabackup_incremental_basedir; //dummy
+ } else if (xtrabackup_prepare && xtrabackup_incremental_dir) {
+ char filename[FN_REFLEN];
+
+ sprintf(filename, "%s/%s", xtrabackup_incremental_dir, XTRABACKUP_METADATA_FILENAME);
+
+ if (!xtrabackup_read_metadata(filename)) {
+ msg("mariabackup: error: failed to read metadata from "
+ "%s", filename);
+ return(EXIT_FAILURE);
+ }
+
+ incremental_lsn = metadata_from_lsn;
+ incremental_to_lsn = metadata_to_lsn;
+ incremental_last_lsn = metadata_last_lsn;
+ xtrabackup_incremental = xtrabackup_incremental_dir; //dummy
+
+ } else if (opt_incremental_history_name) {
+ xtrabackup_incremental = opt_incremental_history_name;
+ } else if (opt_incremental_history_uuid) {
+ xtrabackup_incremental = opt_incremental_history_uuid;
+ } else {
+ xtrabackup_incremental = NULL;
+ }
+
+ if (xtrabackup_stream && !xtrabackup_backup) {
+ msg("Warning: --stream parameter is ignored, it only works together with --backup.");
+ }
+
+ if (!xb_init()) {
+ return(EXIT_FAILURE);
+ }
+
+ /* --print-param */
+ if (xtrabackup_print_param) {
+ printf("%s", print_param_str.str().c_str());
+ return(EXIT_SUCCESS);
+ }
+
+ print_version();
+ if (xtrabackup_incremental) {
+ msg("incremental backup from " LSN_PF " is enabled.",
+ incremental_lsn);
+ }
+
+ if (xtrabackup_export && !srv_file_per_table) {
+ msg("mariabackup: auto-enabling --innodb-file-per-table due to "
+ "the --export option");
+ srv_file_per_table = TRUE;
+ }
+
+ /* cannot execute both for now */
+ {
+ int num = 0;
+
+ if (xtrabackup_backup) num++;
+ if (xtrabackup_prepare) num++;
+ if (xtrabackup_copy_back) num++;
+ if (xtrabackup_move_back) num++;
+ if (xtrabackup_decrypt_decompress) num++;
+ if (num != 1) { /* !XOR (for now) */
+ usage();
+ return(EXIT_FAILURE);
+ }
+ }
+
+ ut_ad(!field_ref_zero);
+ if (auto b = aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) {
+ field_ref_zero = static_cast<byte*>(
+ memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX));
+ } else {
+ msg("Can't allocate memory for field_ref_zero");
+ return EXIT_FAILURE;
+ }
+
+ auto _ = make_scope_exit([]() {
+ aligned_free(const_cast<byte*>(field_ref_zero));
+ field_ref_zero = nullptr;
+ });
+
+ /* --backup */
+ if (xtrabackup_backup && !xtrabackup_backup_func()) {
+ return(EXIT_FAILURE);
+ }
+
+ /* --prepare */
+ if (xtrabackup_prepare
+ && !xtrabackup_prepare_func(argv)) {
+ return(EXIT_FAILURE);
+ }
+
+ if (xtrabackup_copy_back || xtrabackup_move_back) {
+ if (!check_if_param_set("datadir")) {
+ mysql_data_home = get_default_datadir();
+ }
+ if (!copy_back())
+ return(EXIT_FAILURE);
+ }
+
+ if (xtrabackup_decrypt_decompress && !decrypt_decompress()) {
+ return(EXIT_FAILURE);
+ }
+
+ return(EXIT_SUCCESS);
+}
+
+
+static int get_exepath(char *buf, size_t size, const char *argv0)
+{
+#ifdef _WIN32
+ DWORD ret = GetModuleFileNameA(NULL, buf, (DWORD)size);
+ if (ret > 0)
+ return 0;
+#elif defined(__linux__)
+ ssize_t ret = readlink("/proc/self/exe", buf, size-1);
+ if(ret > 0)
+ return 0;
+#elif defined(__APPLE__)
+ size_t ret = proc_pidpath(getpid(), buf, static_cast<uint32_t>(size));
+ if (ret > 0) {
+ buf[ret] = 0;
+ return 0;
+ }
+#elif defined(__FreeBSD__)
+ int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
+ if (sysctl(mib, 4, buf, &size, NULL, 0) == 0) {
+ return 0;
+ }
+#endif
+
+ return my_realpath(buf, argv0, 0);
+}
+
+
+#if defined (__SANITIZE_ADDRESS__) && defined (__linux__)
+/* Avoid LeakSanitizer's false positives. */
+const char* __asan_default_options()
+{
+ return "detect_leaks=0";
+}
+#endif