diff options
Diffstat (limited to 'extra/mariabackup')
41 files changed, 4230 insertions, 2182 deletions
diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index f1c9dca7..63ac8cf3 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -31,6 +31,7 @@ ENDIF() INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql + ${CMAKE_SOURCE_DIR}/storage/maria ${CMAKE_CURRENT_SOURCE_DIR}/quicklz ${CMAKE_CURRENT_SOURCE_DIR} ) @@ -49,14 +50,9 @@ ADD_DEFINITIONS(-UMYSQL_SERVER) ADD_DEFINITIONS(-DPCRE_STATIC=1) ADD_DEFINITIONS(${SSL_DEFINES}) -IF(PMEM_FOUND) - ADD_COMPILE_FLAGS(xtrabackup.cc COMPILE_FLAGS "-DHAVE_PMEM") -ENDIF() - MYSQL_ADD_EXECUTABLE(mariadb-backup xtrabackup.cc innobackupex.cc - changed_page_bitmap.cc datasink.cc ds_buffer.cc ds_compress.cc @@ -72,8 +68,12 @@ MYSQL_ADD_EXECUTABLE(mariadb-backup xbstream_write.cc backup_mysql.cc backup_copy.cc - xb_plugin.cc + encryption_plugin.cc ${PROJECT_BINARY_DIR}/sql/sql_builtin.cc + aria_backup_client.cc + thread_pool.cc + ddl_log.cc + common_engine.cc ${PROJECT_SOURCE_DIR}/sql/net_serv.cc ${PROJECT_SOURCE_DIR}/libmysqld/libmysql.c COMPONENT backup @@ -82,7 +82,8 @@ MYSQL_ADD_EXECUTABLE(mariadb-backup # Export all symbols on Unix, for better crash callstacks SET_TARGET_PROPERTIES(mariadb-backup PROPERTIES ENABLE_EXPORTS TRUE) -TARGET_LINK_LIBRARIES(mariadb-backup sql sql_builtins) +TARGET_LINK_LIBRARIES(mariadb-backup sql sql_builtins aria) + IF(NOT HAVE_SYSTEM_REGEX) TARGET_LINK_LIBRARIES(mariadb-backup pcre2-posix) ENDIF() diff --git a/extra/mariabackup/aria_backup_client.cc b/extra/mariabackup/aria_backup_client.cc new file mode 100644 index 00000000..25468148 --- /dev/null +++ b/extra/mariabackup/aria_backup_client.cc @@ -0,0 +1,1016 @@ +#include <my_global.h> +#include <m_string.h> +extern "C" { +#include "maria_def.h" +} +#undef LSN_MAX +#include "aria_backup_client.h" +#include "backup_copy.h" +#include "common.h" +#include "sql_table.h" +#include "ma_checkpoint.h" +#include "ma_recovery.h" +#include "backup_debug.h" +#include "aria_backup.h" +#include <thread> +#include <string> +#include <vector> +#include <memory> +#include <limits> +#include <unordered_map> +#include <atomic> +#include <utility> +#include <sstream> +#include <iomanip> +#include <cstdlib> + +namespace aria { + +const char *log_preffix = "aria_log."; + + +static std::string log_file_name_only(size_t log_num) { + std::string log_file; + { + std::stringstream ss; + ss << std::setw(8) << std::setfill('0') << log_num; + log_file.append(log_preffix).append(ss.str()); + } + return log_file; +} + + +static std::string log_file_name(const char *datadir_path, size_t log_num) { + std::string log_file(datadir_path); + return log_file.append("/").append(log_file_name_only(log_num)); +} + + +class LogFileCollection +{ + uint32 m_first; + uint32 m_count; +public: + uint32 first() const { return m_first; } + uint32 count() const { return m_count; } + uint32 last() const + { + DBUG_ASSERT(m_count > 0); + return m_first + m_count - 1; + } + + // Initialize by checking existing log files on the disk + LogFileCollection(const char *datadir, uint32 max_log_no) + { + uint32 end= find_greatest_existing_log(datadir, max_log_no); + if (!end) + { + // No log files were found at all + m_first= 0; + m_count= 0; + } + else if (end == 1) + { + // Just the very first one log file (aria_log.00000001) was found. + m_first= 1; + m_count= 1; + } + else + { + // Multiple files were found + m_first= find_greatest_missing_log(datadir, end - 1) + 1; + m_count= 1 + end - m_first; + } + } + + /* + Skip all missing log files and find the greatest existing log file, or + Skip all existing log files and find the greatest missing log file. + + @param datadir - Search files in this directory + @param start - Start searching from this log number and go downto 1. + @param kind - true - search for an existing file + false - search for a missing file. + @returns - [1..start] - the greatest found log file + of the searched kind + - 0 - if no log files of this kind + were found in the range [1..start]. + */ + static uint32 find_greatest_existing_or_missing_log(const char *datadir, + uint32 start, + bool kind) + { + DBUG_ASSERT(start > 0); + for (uint32 i= start; i > 0; i--) + { + if (file_exists(log_file_name(datadir, i).c_str()) == kind) + return i; + } + return 0; // No log files of the searched kind were found + } + + static uint32 find_greatest_existing_log(const char *datadir, uint32 start) + { + return find_greatest_existing_or_missing_log(datadir, start, true); + } + + static uint32 find_greatest_missing_log(const char *datadir, uint32 start) + { + return find_greatest_existing_or_missing_log(datadir, start, false); + } + + /* + In some scenarios (e.g. log rotate) some new log files can appear + outside of the initially assumed [first,last] log number range. + This function adds all extra files behind "last". + */ + void find_logs_after_last(const char *datadir) + { + DBUG_ASSERT(m_count > 0); + for ( ; + file_exists(log_file_name(datadir, last() + 1).c_str()) ; + m_count++) + { } + } + + void report_found(unsigned thread_num) const + { + if (m_count) + msg(thread_num, + "Found %u aria log files, " + "minimum log number %u, " + "maximum log number %u", + m_count, m_first, last()); + } + + void die_if_missing(uint32 logno) const + { + DBUG_ASSERT(logno > 0); + if (!m_count || m_first > logno || last() < logno) + die("Aria log file %u does not exists.", logno); + } +}; + + +class Table { +public: + struct Partition { + std::string m_file_path; + File m_index_file = -1; + MY_STAT m_index_file_stat; + File m_data_file = -1; + MY_STAT m_data_file_stat; + }; + Table() = default; + Table (Table &&other) = delete; + Table & operator= (Table &&other) = delete; + Table(const Table &) = delete; + Table & operator= (const Table &) = delete; + ~Table(); + bool init(const char *data_file_path); + bool open(MYSQL *con, bool opt_no_lock, unsigned thread_num); + bool close(); + bool copy(ds_ctxt_t *ds, unsigned thread_num); + + bool is_online_backup_safe() const { + DBUG_ASSERT(is_opened()); + return m_cap.online_backup_safe; + } + bool is_stats() const { + return is_stats_table(m_db.c_str(), m_table.c_str()); + } + bool is_log() const { + return is_log_table(m_db.c_str(), m_table.c_str()); + } + bool is_opened() const { + return !m_partitions.empty() && + m_partitions[0].m_index_file >= 0 && m_partitions[0].m_data_file >= 0; + }; + std::string &get_full_name() { + return m_full_name; + } + std::string &get_db() { return m_db; } + std::string &get_table() { return m_table; } + std::string &get_version() { return m_table_version; } + bool is_partitioned() const { return m_partitioned; } + void add_partition(const Table &partition) { + DBUG_ASSERT(is_partitioned()); + m_partitions.push_back(partition.m_partitions[0]); + } +#ifndef DBUG_OFF + const std::string& get_sql_name() const { return m_sql_name; } +#endif //DBUG_OFF +private: + + bool copy(ds_ctxt_t *ds, bool is_index, unsigned thread_num); + // frm and par files will be copied under BLOCK_DDL stage in + // backup_copy_non_system() + bool copy_frm_and_par(ds_ctxt_t *ds, unsigned thread_num); + bool read_table_version_id(File file); + + std::string m_db; + std::string m_table; + std::string m_full_name; + std::string m_frm_par_path; + std::string m_table_version; +#ifndef DBUG_OFF + std::string m_sql_name; +#endif //DBUG_OFF + bool m_partitioned = false; + std::vector<Partition> m_partitions; + ARIA_TABLE_CAPABILITIES m_cap; +}; + +Table::~Table() { + (void)close(); +} + +bool Table::init(const char *data_file_path) { + DBUG_ASSERT(data_file_path); + + const char *ext_pos = strrchr(data_file_path, '.'); + if (!ext_pos) + return false; + + char db_name_orig[FN_REFLEN]; + char table_name_orig[FN_REFLEN]; + parse_db_table_from_file_path( + data_file_path, db_name_orig, table_name_orig); + if (!db_name_orig[0] || !table_name_orig[0]) + return false; + char db_name_conv[FN_REFLEN]; + char table_name_conv[FN_REFLEN]; + filename_to_tablename(db_name_orig, db_name_conv, sizeof(db_name_conv)); + filename_to_tablename( + table_name_orig, table_name_conv, sizeof(table_name_conv)); + if (!db_name_conv[0] || !table_name_conv[0]) + return false; + + if (strstr(data_file_path, "#P#")) + m_partitioned = true; + + const char *table_name_begin = strrchr(data_file_path, FN_LIBCHAR); + if (!table_name_begin) + return false; + m_frm_par_path.assign(data_file_path, table_name_begin + 1). + append(table_name_orig); + + m_db.assign(db_name_conv); + m_table.assign(table_name_conv); + // TODO: find the correct way to represent quoted table/db names + m_full_name.assign("`").append(m_db).append("`.`"). + append(m_table).append("`"); +#ifndef DBUG_OFF + m_sql_name.assign(m_db).append("/").append(m_table); +#endif // DBUG_OFF + Partition partition; + partition.m_file_path.assign(data_file_path, ext_pos - data_file_path); + m_partitions.push_back(std::move(partition)); + return true; +} + +bool Table::read_table_version_id(File file) { + m_table_version = ::read_table_version_id(file); + return m_table_version.empty(); +} + +bool Table::open(MYSQL *con, bool opt_no_lock, unsigned thread_num) { + int error= 1; + bool have_capabilities = false; + File frm_file = -1; + + if (!opt_no_lock && !backup_lock(con, m_full_name.c_str())) { + msg(thread_num, "Error on BACKUP LOCK for aria table %s", + m_full_name.c_str()); + goto exit; + } + + for (Partition &partition : m_partitions) { + std::string file_path = partition.m_file_path + ".MAI"; + if ((partition.m_index_file= my_open(file_path.c_str(), + O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_WME))) < 0) { + msg(thread_num, "Error on aria table file open %s", file_path.c_str()); + goto exit; + } + if (!my_stat(file_path.c_str(), &partition.m_index_file_stat, MYF(0))) { + msg(thread_num, "Error on aria table file stat %s", file_path.c_str()); + goto exit; + } + if (!have_capabilities) { + if ((error= aria_get_capabilities(partition.m_index_file, &m_cap))) { + msg(thread_num, "aria_get_capabilities failed: %d", error); + goto exit; + } + have_capabilities = true; + } + + file_path = partition.m_file_path + ".MAD"; + if ((partition.m_data_file= my_open(file_path.c_str(), + O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_WME))) < 0) { + msg(thread_num, "Error on aria table file open %s", file_path.c_str()); + goto exit; + } + if (!my_stat(file_path.c_str(), &partition.m_data_file_stat, MYF(0))) { + msg(thread_num, "Error on aria table file stat %s", file_path.c_str()); + goto exit; + } + } + + if ((frm_file = mysql_file_open( + key_file_frm, (m_frm_par_path + ".frm").c_str(), + O_RDONLY | O_SHARE, MYF(0))) < 0) { + msg(thread_num, "Error on aria table %s file open", + (m_frm_par_path + ".frm").c_str()); + goto exit; + } + + error = 0; + +exit: + if (!opt_no_lock && !backup_unlock(con)) { + msg(thread_num, "Error on BACKUP UNLOCK for aria table %s", + m_full_name.c_str()); + error = 1; + } + if (error) + (void)close(); + else { + (void)read_table_version_id(frm_file); + mysql_file_close(frm_file, MYF(MY_WME)); + } + return !error; +} + +bool Table::close() { + for (Partition &partition : m_partitions) { + if (partition.m_index_file >= 0) { + my_close(partition.m_index_file, MYF(MY_WME)); + partition.m_index_file = -1; + } + if (partition.m_data_file >= 0) { + my_close(partition.m_data_file, MYF(MY_WME)); + partition.m_data_file = -1; + } + } + return true; +} + +bool Table::copy(ds_ctxt_t *ds, unsigned thread_num) { + DBUG_ASSERT(is_opened()); + DBUG_MARIABACKUP_EVENT_LOCK("before_aria_table_copy", + fil_space_t::name_type(m_sql_name.data(), m_sql_name.size())); + bool result = +// copy_frm_and_par(ds, thread_num) && + copy(ds, true, thread_num) && copy(ds, false, thread_num); + return result; +} + +bool Table::copy(ds_ctxt_t *ds, bool is_index, unsigned thread_num) { + DBUG_ASSERT(ds); + const char *ext = is_index ? ".MAI" : ".MAD"; + int error= 1; + for (const Partition &partition : m_partitions) { + ds_file_t *dst_file = nullptr; + uchar *copy_buffer = nullptr; + std::string full_name = partition.m_file_path + ext; + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back) ? + full_name.c_str() : trim_dotslash(full_name.c_str()); + + dst_file = ds_open(ds, dst_path, + is_index ? &partition.m_index_file_stat : &partition.m_data_file_stat); + if (!dst_file) { + msg(thread_num, "error: cannot open the destination stream for %s", + dst_path); + goto err; + } + + copy_buffer = + reinterpret_cast<uchar *>(my_malloc(PSI_NOT_INSTRUMENTED, + m_cap.block_size, MYF(0))); + + DBUG_MARIABACKUP_EVENT_LOCK( + is_index ? + "before_aria_index_file_copy": + "before_aria_data_file_copy", + fil_space_t::name_type(m_sql_name.data(), + m_sql_name.size())); + + for (ulonglong block= 0 ; ; block++) { + size_t length = m_cap.block_size; + if (is_index) { + if ((error= aria_read_index( + partition.m_index_file, &m_cap, block, copy_buffer) == + HA_ERR_END_OF_FILE)) + break; + } else { + if ((error= aria_read_data( + partition.m_data_file, &m_cap, block, copy_buffer, &length) == + HA_ERR_END_OF_FILE)) + break; + } + if (error) { + msg(thread_num, "error: aria_read %s failed: %d", + is_index ? "index" : "data", error); + goto err; + } + xtrabackup_io_throttling(); + if ((error = ds_write(dst_file, copy_buffer, length))) { + msg(thread_num, "error: aria_write failed: %d", error); + goto err; + } + } + + DBUG_MARIABACKUP_EVENT_LOCK( + is_index ? + "after_aria_index_file_copy": + "after_aria_data_file_copy", + fil_space_t::name_type(m_sql_name.data(), + m_sql_name.size())); + + error = 0; + msg(thread_num, "aria table file %s is copied successfully.", + full_name.c_str()); + + err: + if (dst_file) + ds_close(dst_file); + if (copy_buffer) + my_free(copy_buffer); + if (error) + break; + } + return !error; +} + +class BackupImpl { +public: + BackupImpl( + const char *datadir_path, + const char *aria_log_path, + ds_ctxt_t *datasink, bool opt_no_lock, + std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool) : + m_datadir_path(datadir_path), + m_aria_log_dir_path(aria_log_path), + m_ds(datasink), m_con_pool(con_pool), + m_tasks_group(thread_pool), m_thread_pool(thread_pool) { } + ~BackupImpl() { destroy(); } + bool init(); + bool start(bool no_lock); + bool wait_for_finish(); + bool copy_offline_tables( + const std::unordered_set<table_key_t> *exclude_tables, bool no_lock, + bool copy_stats); + bool finalize(); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_table_post_copy_hook = hook; + } + bool copy_log_tail() { return copy_log_tail(0, false); } +private: + void destroy(); + void scan_job(bool no_lock, unsigned thread_num); + bool copy_log_tail(unsigned thread_num, bool finalize); + void copy_log_file_job(size_t log_num, unsigned thread_num); + void destroy_log_tail(); + void process_table_job(Table *table, bool online_only, bool copy_stats, + bool no_lock, unsigned thread_num); + + const char *m_datadir_path; + const char *m_aria_log_dir_path; + std::string aria_log_dir_path() const + { + if (!m_aria_log_dir_path || !m_aria_log_dir_path[0]) + return m_datadir_path; + if (is_absolute_path(m_aria_log_dir_path)) + return m_aria_log_dir_path; + return std::string(m_datadir_path).append("/") + .append(m_aria_log_dir_path); + } + ds_ctxt_t *m_ds; + std::vector<MYSQL *> &m_con_pool; + + TasksGroup m_tasks_group; + + std::mutex m_offline_tables_mutex; + std::vector<std::unique_ptr<Table>> m_offline_tables; + post_copy_table_hook_t m_table_post_copy_hook; + + ThreadPool &m_thread_pool; + + size_t m_last_log_num = 0; + ds_file_t* m_last_log_dst = nullptr; + File m_last_log_src = -1; +}; + +bool BackupImpl::init() { + DBUG_ASSERT(m_tasks_group.is_finished()); + return true; +}; + +void BackupImpl::destroy() { + DBUG_ASSERT(m_tasks_group.is_finished()); + destroy_log_tail(); +} + +bool BackupImpl::start(bool no_lock) { + DBUG_ASSERT(m_tasks_group.is_finished()); + m_tasks_group.push_task( + std::bind(&BackupImpl::scan_job, this, no_lock, std::placeholders::_1)); + return true; +} + +void BackupImpl::process_table_job( + Table *table_ptr, bool online_only, bool copy_stats, bool no_lock, + unsigned thread_num) { + DBUG_ASSERT(table_ptr); + DBUG_ASSERT(thread_num < m_con_pool.size()); + std::unique_ptr<Table> table(table_ptr); + bool is_online; + bool is_stats; + bool need_copy; + int result = 1; + + if (!m_tasks_group.get_result()) + goto exit; + + if (!table->open(m_con_pool[thread_num], no_lock, thread_num)) { + // if table can't be opened, it might be removed or renamed, this is not + // error for transactional tables + table->close(); // Close opened table files + goto exit; + } + + is_online = table->is_online_backup_safe(); + is_stats = table->is_stats(); + + need_copy = (!online_only || is_online) && (copy_stats || !is_stats); + + if (need_copy && !table->copy(m_ds, thread_num)) { + table->close(); + DBUG_MARIABACKUP_EVENT_LOCK("after_aria_table_copy", + fil_space_t::name_type(table->get_sql_name().data(), + table->get_sql_name().size())); + // if table is opened, it must be copied, + // the corresponding diagnostic messages must be issued in Table::copy() + result = 0; + goto exit; + } + + if (!table->close()) { + msg(thread_num, "Can't close aria table %s.\n", + table->get_full_name().c_str()); + result = 0; + goto exit; + } + + if (!need_copy) { + std::lock_guard<std::mutex> lock(m_offline_tables_mutex); + m_offline_tables.push_back(std::move(table)); + } + else { + DBUG_MARIABACKUP_EVENT_LOCK("after_aria_table_copy", + fil_space_t::name_type(table->get_sql_name().data(), + table->get_sql_name().size())); + if (m_table_post_copy_hook) + m_table_post_copy_hook( + std::move(table->get_db()), + std::move(table->get_table()), + std::move(table->get_version())); + } +exit: + m_tasks_group.finish_task(result); +} + + +void BackupImpl::scan_job(bool no_lock, unsigned thread_num) { + std::unordered_map<std::string, std::unique_ptr<Table>> partitioned_tables; + + std::string aria_log_dir_path_cache(aria_log_dir_path()); + std::string log_control_file_path(aria_log_dir_path_cache); + log_control_file_path.append("/aria_log_control"); + if (!m_ds->copy_file( + log_control_file_path.c_str(), "aria_log_control", + 0, false)) { + msg("Aria log control file copying error."); + m_tasks_group.finish_task(0); + return; + } + + msg(thread_num, "Loading aria_log_control."); + aria_readonly= 1; + maria_data_root= aria_log_dir_path_cache.c_str(); + if (ma_control_file_open(FALSE, FALSE, FALSE, O_RDONLY)) + die("Can't open Aria control file (%d)", errno); + uint32 aria_log_control_last_log_number= last_logno; + msg(thread_num, "aria_log_control: last_log_number: %d", + aria_log_control_last_log_number); + ma_control_file_end(); + + msg(thread_num, "Start scanning aria tables."); + + foreach_file_in_db_dirs(m_datadir_path, [&](const char *file_path)->bool { + + if (check_if_skip_table(file_path)) { + msg(thread_num, "Skipping %s.", file_path); + return true; + } + + if (!ends_with(file_path, ".MAD")) + return true; + + std::unique_ptr<Table> table(new Table()); + if (!table->init(file_path)) { + msg(thread_num, "Can't init aria table %s.\n", file_path); + return true; + } + + if (table->is_log()) + return true; + + if (table->is_partitioned()) { + auto table_it = partitioned_tables.find(table->get_full_name()); + if (table_it == partitioned_tables.end()) { + partitioned_tables[table->get_full_name()] = std::move(table); + } else { + table_it->second->add_partition(*table); + } + return true; + } + + m_tasks_group.push_task( + std::bind(&BackupImpl::process_table_job, this, table.release(), true, + false, no_lock, std::placeholders::_1)); + return true; + }); + + for (auto &table_it : partitioned_tables) { + m_tasks_group.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.release(), + true, false, no_lock, std::placeholders::_1)); + } + + msg(thread_num, "Start scanning aria log files."); + + LogFileCollection logs(aria_log_dir_path_cache.c_str(), + aria_log_control_last_log_number); + logs.report_found(thread_num); + logs.die_if_missing(aria_log_control_last_log_number); + + m_last_log_num= logs.last(); + + DBUG_MARIABACKUP_EVENT("after_scanning_log_files", {}); + + for (uint32 i= logs.first(); i <= logs.last(); ++i) + m_tasks_group.push_task( + std::bind(&BackupImpl::copy_log_file_job, this, + i, std::placeholders::_1)); + + msg(thread_num, "Stop scanning aria tables."); + + m_tasks_group.finish_task(1); +} + +template<typename T> +T align_down(T n, ulint align_no) +{ + DBUG_ASSERT(align_no > 0); + DBUG_ASSERT(ut_is_2pow(align_no)); + return n & ~(static_cast<T>(align_no) - 1); +} + +static ssize_t copy_file_chunk(File src, ds_file_t* dst, size_t size) { + size_t bytes_read; + static const size_t max_buf_size = 10 * 1024 * 1024; + size_t buf_size = size ? std::min(size, max_buf_size) : max_buf_size; + std::unique_ptr<uchar[]> buf(new uchar[buf_size]); + ssize_t copied_size = 0; + bool unlim = !size; + while((unlim || size) && (bytes_read = my_read(src, buf.get(), + unlim ? buf_size : std::min(buf_size, size), MY_WME))) { + if (bytes_read == size_t(-1)) + return -1; + xtrabackup_io_throttling(); + if (ds_write(dst, buf.get(), bytes_read)) + return -1; + copied_size += bytes_read; + if (!unlim) + size -= bytes_read; + } + return copied_size; +} + +bool BackupImpl::copy_log_tail(unsigned thread_num, bool finalize) { + bool result = false; + std::string log_file = log_file_name(aria_log_dir_path().c_str(), m_last_log_num); + std::string prev_log_file; + ssize_t total_bytes_copied = 0; + MY_STAT stat_info; + my_off_t file_offset = 0; + size_t to_copy_size = 0; + +repeat: + memset(&stat_info, 0, sizeof(MY_STAT)); + if (!m_tasks_group.get_result()) { + msg(thread_num, "Skip copying aria lof file tail %s due to error.", + log_file.c_str()); + result = true; + goto exit; + } + + msg(thread_num, "Start copying aria log file tail: %s", log_file.c_str()); + + if (m_last_log_src < 0 && (m_last_log_src = + my_open(log_file.c_str(), O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_WME))) < 0) { + msg("Aria log file %s open failed: %d", log_file.c_str(), my_errno); + goto exit; + } + + if (!m_last_log_dst && + !(m_last_log_dst = ds_open(m_ds, + log_file_name_only(m_last_log_num).c_str(), + &stat_info, false))) { + msg(thread_num, "error: failed to open the target stream for " + "aria log file %s.", + log_file.c_str()); + goto exit; + } + +// If there is no need to finalize log file copying, calculate the size to copy +// without the last page, which can be rewritten by the server +// (see translog_force_current_buffer_to_finish()). + if (!finalize) { + if (my_fstat(m_last_log_src, &stat_info, MYF(0))) { + msg(thread_num, "error: failed to get file size for aria log file: %s.", + log_file.c_str()); + goto exit; + } + if ((file_offset = my_tell(m_last_log_src, MYF(0))) == (my_off_t)(-1)) { + msg(thread_num, "error: failed to get file offset for aria log file: %s.", + log_file.c_str()); + goto exit; + } + DBUG_ASSERT(file_offset <= static_cast<my_off_t>(stat_info.st_size)); + to_copy_size = static_cast<size_t>(stat_info.st_size - file_offset); + to_copy_size = to_copy_size >= TRANSLOG_PAGE_SIZE ? + (align_down(to_copy_size, TRANSLOG_PAGE_SIZE) - TRANSLOG_PAGE_SIZE) : 0; + } + +// Copy from the last position to the end of file, +// excluding the last page is there is no need to finalize the copy. + if ((to_copy_size || finalize) && + (total_bytes_copied = copy_file_chunk(m_last_log_src, + m_last_log_dst, to_copy_size)) < 0) { + msg(thread_num, "Aria log file %s chunk copy error", log_file.c_str()); + goto exit; + } + + msg(thread_num, "Stop copying aria log file tail: %s, copied %zu bytes", + log_file.c_str(), total_bytes_copied); + +// Check if there is new log file, if yes, then copy the last page of the old +// one, and fix it last LSN in the log header, as it is changed on new +// log file creating by the server (see translog_create_new_file() and +// translog_max_lsn_to_header()). Then close the old log file and repeat +// the copying for the new log file. + prev_log_file = std::move(log_file); + log_file = log_file_name(aria_log_dir_path().c_str(), m_last_log_num + 1); + if (file_exists(log_file.c_str())) { + uchar lsn_buff[LSN_STORE_SIZE]; + msg(thread_num, "Found new aria log tail file: %s, start copy %s tail", + log_file.c_str(), prev_log_file.c_str()); + if ((total_bytes_copied = copy_file_chunk(m_last_log_src, + m_last_log_dst, 0)) < 0) { + msg(thread_num, "Aria log file %s tail copy error", + prev_log_file.c_str()); + goto exit; + } + + if (my_pread(m_last_log_src, lsn_buff, LSN_STORE_SIZE, + (LOG_HEADER_DATA_SIZE - LSN_STORE_SIZE), MYF(0)) < LSN_STORE_SIZE) { + msg(thread_num, "Aria lsn store read error for log file %s", + prev_log_file.c_str()); + goto exit; + } + + if (ds_seek_set(m_last_log_dst, (LOG_HEADER_DATA_SIZE - LSN_STORE_SIZE))) { + msg(thread_num, "Set aria log pointer error for log file %s", + prev_log_file.c_str()); + goto exit; + } + + if (ds_write(m_last_log_dst, lsn_buff, LSN_STORE_SIZE)) { + msg(thread_num, "LSN write error for aria log file %s", + prev_log_file.c_str()); + goto exit; + } + + msg(thread_num, "The last %zu bytes were copied for %s.", + total_bytes_copied, prev_log_file.c_str()); + destroy_log_tail(); + ++m_last_log_num; + goto repeat; + } + + result = true; + +exit: + if (!result) + destroy_log_tail(); + return result; +} + +void BackupImpl::copy_log_file_job(size_t log_num, unsigned thread_num) { + DBUG_ASSERT(log_num <= m_last_log_num); + + if (!m_tasks_group.get_result()) { + msg(thread_num, "Skip copying %zu aria log file due to error", log_num); + m_tasks_group.finish_task(0); + return; + } + +// Copy log file if the file is not the last one. + if (log_num < m_last_log_num) { + std::string log_file = log_file_name(aria_log_dir_path().c_str(), log_num); + if (!m_ds->copy_file(log_file.c_str(), + log_file_name_only(log_num).c_str(), + thread_num, false)) { + msg(thread_num, "Error on copying %s aria log file.", log_file.c_str()); + m_tasks_group.finish_task(0); + } + else + m_tasks_group.finish_task(1); + return; + } +// Copy the last log file. + m_tasks_group.finish_task(copy_log_tail(thread_num, false) ? 1 : 0); +} + +void BackupImpl::destroy_log_tail() { + if (m_last_log_src >= 0) { + my_close(m_last_log_src, MYF(MY_WME)); + m_last_log_src = -1; + } + if (m_last_log_dst) { + ds_close(m_last_log_dst); + m_last_log_dst = nullptr; + } +} + +bool BackupImpl::wait_for_finish() { + return m_tasks_group.wait_for_finish(); +} + +bool BackupImpl::copy_offline_tables( + const std::unordered_set<table_key_t> *exclude_tables, bool no_lock, + bool copy_stats) { + DBUG_ASSERT(m_tasks_group.is_finished()); + + std::vector<std::unique_ptr<Table>> ignored_tables; + + while (true) { + std::unique_lock<std::mutex> lock(m_offline_tables_mutex); + if (m_offline_tables.empty()) + break; + auto table = std::move(m_offline_tables.back()); + m_offline_tables.pop_back(); + lock.unlock(); + if ((exclude_tables && + exclude_tables->count(table_key(table->get_db(), table->get_table()))) || + (!copy_stats && table->is_stats())) { + ignored_tables.push_back(std::move(table)); + continue; + } + m_tasks_group.push_task( + std::bind(&BackupImpl::process_table_job, this, table.release(), false, + copy_stats, no_lock, std::placeholders::_1)); + } + + if (!ignored_tables.empty()) { + std::lock_guard<std::mutex> lock(m_offline_tables_mutex); + m_offline_tables = std::move(ignored_tables); + } + + return true; +} + +bool BackupImpl::finalize() { + DBUG_ASSERT(m_tasks_group.is_finished()); + DBUG_ASSERT(!m_con_pool.empty()); + bool result = true; + msg("Start copying statistics aria tables."); + copy_offline_tables(nullptr, true, true); + while (!m_tasks_group.is_finished()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + msg("Stop copying statistics aria tables."); + copy_log_tail(0, true); + destroy_log_tail(); + return result; +} + +Backup::Backup(const char *datadir_path, + const char *aria_log_path, + ds_ctxt_t *datasink, + std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool) : + m_backup_impl( + new BackupImpl(datadir_path, aria_log_path, + datasink, opt_no_lock, con_pool, + thread_pool)) { } + +Backup::~Backup() { + delete m_backup_impl; +} + +bool Backup::init() { + return m_backup_impl->init(); +} + +bool Backup::start(bool no_lock) { + return m_backup_impl->start(no_lock); +} + +bool Backup::wait_for_finish() { + return m_backup_impl->wait_for_finish(); +} + +bool Backup::copy_offline_tables( + const std::unordered_set<table_key_t> *exclude_tables, bool no_lock, + bool copy_stats) { + return m_backup_impl->copy_offline_tables(exclude_tables, no_lock, + copy_stats); +} + +bool Backup::finalize() { + return m_backup_impl->finalize(); +} + +bool Backup::copy_log_tail() { + return m_backup_impl->copy_log_tail(); +} + +void Backup::set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_backup_impl->set_post_copy_table_hook(hook); +} + +bool prepare(const char *target_dir) { + maria_data_root= (char *)target_dir; + + if (maria_init()) + die("Can't init Aria engine (%d)", errno); + + maria_block_size= 0; /* Use block size from file */ + /* we don't want to create a control file, it MUST exist */ + if (ma_control_file_open(FALSE, TRUE, TRUE, control_file_open_flags)) + die("Can't open Aria control file (%d)", errno); + + if (last_logno == FILENO_IMPOSSIBLE) + die("Can't find any Aria log"); + + LogFileCollection logs(target_dir, last_logno); + logs.die_if_missing(last_logno); // Fatal, a broken backup. + /* + "mariadb-backup --backup" can put extra log files, + with log number greater than last_logno. For example, + this combination of files is possible: + - aria_log_control (with last_logno==1) + - aria_log.00000001 (last_logno) + - aria_log.00000002 (last_logno+1, the extra log file) + This can happen if during the ealier run of + "mariadb-backup --backup" a log rotate happened. + The extra log file is copied to the backup directory, + but last_logno in aria_log_control does not get updated. + This mismatch is probably not good and should eventually be fixed. + But during "mariadb-backup --prepare" this mismatch goes away: + aria_log_control gets fixed to say last_logno==2. + See mysql-test/suite/mariabackup/aria_log_rotate_during_backup.test, + it covers the scenario with one extra file created during --backup. + */ + logs.find_logs_after_last(target_dir); + last_logno= logs.last(); // Update last_logno if extra logs were found + + if (init_pagecache(maria_pagecache, 1024L*1024L, 0, 0, + static_cast<uint>(maria_block_size), 0, MY_WME) == 0) + die("Got error in Aria init_pagecache() (errno: %d)", errno); + + if (init_pagecache(maria_log_pagecache, 1024L*1024L, + 0, 0, TRANSLOG_PAGE_SIZE, 0, MY_WME) == 0 || + translog_init(maria_data_root, TRANSLOG_FILE_SIZE, + 0, 0, maria_log_pagecache, TRANSLOG_DEFAULT_FLAGS, FALSE)) + die("Can't init Aria loghandler (%d)", errno); + + if (maria_recovery_from_log()) + die("Aria log apply FAILED"); + + if (maria_recovery_changed_data || recovery_failures) { + if (ma_control_file_write_and_force(last_checkpoint_lsn, last_logno, + max_trid_in_control_file, 0)) + die("Aria control file update error"); +// TODO: find out do we need checkpoint here + } + + maria_end(); + return true; +} + +} // namespace aria diff --git a/extra/mariabackup/aria_backup_client.h b/extra/mariabackup/aria_backup_client.h new file mode 100644 index 00000000..7a581b58 --- /dev/null +++ b/extra/mariabackup/aria_backup_client.h @@ -0,0 +1,38 @@ +#pragma once +#include "my_global.h" +#include "datasink.h" +#include "backup_mysql.h" +#include "thread_pool.h" +#include "xtrabackup.h" + +namespace aria { + +bool prepare(const char *target_dir); + +class BackupImpl; + +class Backup { + public: + Backup(const char *datadir_path, + const char *aria_log_path, + ds_ctxt_t *datasink, + std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool); + ~Backup(); + Backup (Backup &&other) = delete; + Backup & operator= (Backup &&other) = delete; + Backup(const Backup &) = delete; + Backup & operator= (const Backup &) = delete; + bool init(); + bool start(bool no_lock); + bool wait_for_finish(); + bool copy_offline_tables( + const std::unordered_set<table_key_t> *exclude_tables, bool no_lock, + bool copy_stats); + bool finalize(); + bool copy_log_tail(); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook); + private: + BackupImpl *m_backup_impl; +}; + +} // namespace aria diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index f8d315d9..198da01a 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -41,6 +41,9 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA *******************************************************/ #include <my_global.h> +#include <my_config.h> +#include <unireg.h> +#include <datadict.h> #include <os0file.h> #include <my_dir.h> #include <ut0mem.h> @@ -66,19 +69,26 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include <aclapi.h> #endif +#ifdef MYSQL_CLIENT +#define WAS_MYSQL_CLIENT 1 +#undef MYSQL_CLIENT +#endif + +#include "table.h" + +#ifdef WAS_MYSQL_CLIENT +#define MYSQL_CLIENT 1 +#undef WAS_MYSQL_CLIENT +#endif #define ROCKSDB_BACKUP_DIR "#rocksdb" -/* list of files to sync for --rsync mode */ -static std::set<std::string> rsync_list; /* locations of tablespaces read from .isl files */ static std::map<std::string, std::string> tablespace_locations; /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */ bool binlog_locked; -static void rocksdb_create_checkpoint(); -static bool has_rocksdb_plugin(); static void rocksdb_backup_checkpoint(ds_ctxt *ds_data); static void rocksdb_copy_back(ds_ctxt *ds_data); @@ -135,10 +145,6 @@ struct datadir_thread_ctxt_t { bool ret; }; -static bool backup_files_from_datadir(ds_ctxt_t *ds_data, - const char *dir_path, - const char *prefix); - /************************************************************************ Retirn true if character if file separator */ bool @@ -585,7 +591,6 @@ datafile_read(datafile_cur_t *cursor) Check to see if a file exists. Takes name of the file to check. @return true if file exists. */ -static bool file_exists(const char *filename) { @@ -601,7 +606,6 @@ file_exists(const char *filename) /************************************************************************ Trim leading slashes from absolute path so it becomes relative */ -static const char * trim_dotslash(const char *path) { @@ -634,7 +638,7 @@ ends_with(const char *str, const char *suffix) && strcmp(str + str_len - suffix_len, suffix) == 0); } -static bool starts_with(const char *str, const char *prefix) +bool starts_with(const char *str, const char *prefix) { return strncmp(str, prefix, strlen(prefix)) == 0; } @@ -785,7 +789,6 @@ directory_exists_and_empty(const char *dir, const char *comment) /************************************************************************ Check if file name ends with given set of suffixes. @return true if it does. */ -static bool filename_matches(const char *filename, const char **ext_list) { @@ -800,52 +803,127 @@ filename_matches(const char *filename, const char **ext_list) return(false); } - -/************************************************************************ -Copy data file for backup. Also check if it is allowed to copy by -comparing its name to the list of known data file types and checking -if passes the rules for partial backup. -@return true if file backed up or skipped successfully. */ +// TODO: the code can be used to find storage engine of partitions +/* static -bool -datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n) -{ - const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", - "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", - NULL}; +bool is_aria_frm_or_par(const char *path) { + if (!ends_with(path, ".frm") && !ends_with(path, ".par")) + return false; - /* Get the name and the path for the tablespace. node->name always - contains the path (which may be absolute for remote tablespaces in - 5.6+). space->name contains the tablespace name in the form - "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a - multi-node shared tablespace, space->name contains the name of the first - node, but that's irrelevant, since we only need node_name to match them - against filters, and the shared tablespace is always copied regardless - of the filters value. */ + const char *frm_path = path; + if (ends_with(path, ".par")) { + size_t frm_path_len = strlen(path); + DBUG_ASSERT(frm_path_len > strlen("frm")); + frm_path = strdup(path); + strcpy(const_cast<char *>(frm_path) + frm_path_len - strlen("frm"), "frm"); + } - if (check_if_skip_table(filepath)) { - msg(thread_n,"Skipping %s.", filepath); - return(true); + bool result = false; + File file; + uchar header[40]; + legacy_db_type dbt; + + if ((file= mysql_file_open(key_file_frm, frm_path, O_RDONLY | O_SHARE, MYF(0))) + < 0) + goto err; + + if (mysql_file_read(file, (uchar*) header, sizeof(header), MYF(MY_NABP))) + goto err; + + if (!strncmp((char*) header, "TYPE=VIEW\n", 10)) + goto err; + + if (!is_binary_frm_header(header)) + goto err; + + dbt = (legacy_db_type)header[3]; + + if (dbt == DB_TYPE_ARIA) { + result = true; } + else if (dbt == DB_TYPE_PARTITION_DB) { + MY_STAT state; + uchar *frm_image= 0; +// uint n_length; - if (filename_matches(filepath, ext_list)) { - return ds_data->copy_file(filepath, filepath, thread_n); + if (mysql_file_fstat(file, &state, MYF(MY_WME))) + goto err; + + if (mysql_file_seek(file, 0, SEEK_SET, MYF(MY_WME))) + goto err; + + if (read_string(file, &frm_image, (size_t)state.st_size)) + goto err; + + dbt = (legacy_db_type)frm_image[61]; + if (dbt == DB_TYPE_ARIA) { + result = true; + } + my_free(frm_image); } - return(true); +err: + if (file >= 0) + mysql_file_close(file, MYF(MY_WME)); + if (frm_path != path) + free(const_cast<char *>(frm_path)); + return result; } +*/ +void parse_db_table_from_file_path( + const char *filepath, char *dbname, char *tablename) { + dbname[0] = '\0'; + tablename[0] = '\0'; + const char *dbname_start = nullptr; + const char *tablename_start = filepath; + const char *const_ptr; + while ((const_ptr = strchr(tablename_start, FN_LIBCHAR)) != NULL) { + dbname_start = tablename_start; + tablename_start = const_ptr + 1; + } + if (!dbname_start) + return; + size_t dbname_len = tablename_start - dbname_start - 1; + if (dbname_len >= FN_REFLEN) + dbname_len = FN_REFLEN-1; + strmake(dbname, dbname_start, dbname_len); + strmake(tablename, tablename_start, FN_REFLEN-1); + char *ptr; + if ((ptr = strchr(tablename, '.'))) + *ptr = '\0'; + if ((ptr = strstr(tablename, "#P#"))) + *ptr = '\0'; +} + +bool is_system_table(const char *dbname, const char *tablename) +{ + DBUG_ASSERT(dbname); + DBUG_ASSERT(tablename); + + LEX_CSTRING lex_dbname; + LEX_CSTRING lex_tablename; + lex_dbname.str = dbname; + lex_dbname.length = strlen(dbname); + lex_tablename.str = tablename; + lex_tablename.length = strlen(tablename); + + TABLE_CATEGORY tg = get_table_category(&lex_dbname, &lex_tablename); + + return (tg == TABLE_CATEGORY_LOG) || (tg == TABLE_CATEGORY_SYSTEM); +} /************************************************************************ -Same as datafile_copy_backup, but put file name into the list for -rsync command. */ +Copy data file for backup. Also check if it is allowed to copy by +comparing its name to the list of known data file types and checking +if passes the rules for partial backup. +@return true if file backed up or skipped successfully. */ static bool -datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f) +datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n) { - const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", - "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", - NULL}; + const char *ext_list[] = {".frm", ".isl", ".TRG", ".TRN", ".opt", ".par", + NULL}; /* Get the name and the path for the tablespace. node->name always contains the path (which may be absolute for remote tablespaces in @@ -857,15 +935,13 @@ datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f) of the filters value. */ if (check_if_skip_table(filepath)) { + msg(thread_n,"Skipping %s.", filepath); return(true); } if (filename_matches(filepath, ext_list)) { - fprintf(f, "%s\n", filepath); - if (save_to_list) { - rsync_list.insert(filepath); - } - } + return ds_data->copy_file(filepath, filepath, thread_n); + } return(true); } @@ -1004,16 +1080,15 @@ Copy file for backup/restore. bool ds_ctxt_t::copy_file(const char *src_file_path, const char *dst_file_path, - uint thread_n) + uint thread_n, + bool rewrite) { char dst_name[FN_REFLEN]; ds_file_t *dstfile = NULL; datafile_cur_t cursor; xb_fil_cur_result_t res; DBUG_ASSERT(datasink->remove); - const char *dst_path = - (xtrabackup_copy_back || xtrabackup_move_back)? - dst_file_path : trim_dotslash(dst_file_path); + const char *dst_path = convert_dst(dst_file_path); if (!datafile_open(src_file_path, &cursor, thread_n)) { goto error_close; @@ -1021,7 +1096,7 @@ ds_ctxt_t::copy_file(const char *src_file_path, strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); - dstfile = ds_open(this, dst_path, &cursor.statinfo); + dstfile = ds_open(this, dst_path, &cursor.statinfo, rewrite); if (dstfile == NULL) { msg(thread_n,"error: " "cannot open the destination stream for %s", dst_name); @@ -1245,278 +1320,45 @@ cleanup: } - - -static bool -backup_files(ds_ctxt *ds_data, const char *from, bool prep_mode) +backup_files(ds_ctxt *ds_data, const char *from) { - char rsync_tmpfile_name[FN_REFLEN]; - FILE *rsync_tmpfile = NULL; datadir_iter_t *it; datadir_node_t node; bool ret = true; - - if (prep_mode && !opt_rsync) { - return(true); - } - - if (opt_rsync) { - snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), - "%s/%s%d", opt_mysql_tmpdir, - "xtrabackup_rsyncfiles_pass", - prep_mode ? 1 : 2); - rsync_tmpfile = fopen(rsync_tmpfile_name, "w"); - if (rsync_tmpfile == NULL) { - msg("Error: can't create file %s", - rsync_tmpfile_name); - return(false); - } - } - - msg("Starting %s non-InnoDB tables and files", - prep_mode ? "prep copy of" : "to backup"); - + msg("Starting to backup non-InnoDB tables and files"); datadir_node_init(&node); it = datadir_iter_new(from); - while (datadir_iter_next(it, &node)) { - if (!node.is_empty_dir) { - if (opt_rsync) { - ret = datafile_rsync_backup(node.filepath, - !prep_mode, rsync_tmpfile); - } else { - ret = datafile_copy_backup(ds_data, node.filepath, 1); - } + ret = datafile_copy_backup(ds_data, node.filepath, 1); if (!ret) { msg("Failed to copy file %s", node.filepath); goto out; } - } else if (!prep_mode) { + } else { /* backup fake file into empty directory */ char path[FN_REFLEN]; - snprintf(path, sizeof(path), - "%s/db.opt", node.filepath); - if (!(ret = ds_data->backup_file_printf( - trim_dotslash(path), "%s", ""))) { + snprintf(path, sizeof(path), "%s/db.opt", node.filepath); + if (!(ret = ds_data->backup_file_printf(trim_dotslash(path), "%s", ""))) { msg("Failed to create file %s", path); goto out; } } } - - if (opt_rsync) { - std::stringstream cmd; - int err; - - if (buffer_pool_filename && file_exists(buffer_pool_filename)) { - fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename); - rsync_list.insert(buffer_pool_filename); - } - if (file_exists("ib_lru_dump")) { - fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump"); - rsync_list.insert("ib_lru_dump"); - } - - fclose(rsync_tmpfile); - rsync_tmpfile = NULL; - - cmd << "rsync -t . --files-from=" << rsync_tmpfile_name - << " " << xtrabackup_target_dir; - - msg("Starting rsync as: %s", cmd.str().c_str()); - if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) { - msg("Error: rsync failed with error code %d", err); - ret = false; - goto out; - } - msg("rsync finished successfully."); - - if (!prep_mode && !opt_no_lock) { - char path[FN_REFLEN]; - char dst_path[FN_REFLEN]; - char *newline; - - /* Remove files that have been removed between first and - second passes. Cannot use "rsync --delete" because it - does not work with --files-from. */ - snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), - "%s/%s", opt_mysql_tmpdir, - "xtrabackup_rsyncfiles_pass1"); - - rsync_tmpfile = fopen(rsync_tmpfile_name, "r"); - if (rsync_tmpfile == NULL) { - msg("Error: can't open file %s", - rsync_tmpfile_name); - ret = false; - goto out; - } - - while (fgets(path, sizeof(path), rsync_tmpfile)) { - - newline = strchr(path, '\n'); - if (newline) { - *newline = 0; - } - if (rsync_list.count(path) < 1) { - snprintf(dst_path, sizeof(dst_path), - "%s/%s", xtrabackup_target_dir, - path); - msg("Removing %s", dst_path); - unlink(dst_path); - } - } - - fclose(rsync_tmpfile); - rsync_tmpfile = NULL; - } - } - - msg("Finished %s non-InnoDB tables and files", - prep_mode ? "a prep copy of" : "backing up"); - + msg("Finished backing up non-InnoDB tables and files"); out: datadir_iter_free(it); datadir_node_free(&node); - - if (rsync_tmpfile != NULL) { - fclose(rsync_tmpfile); - } - return(ret); } - -lsn_t get_current_lsn(MYSQL *connection) -{ - static const char lsn_prefix[] = "\nLog sequence number "; - lsn_t lsn = 0; - if (MYSQL_RES *res = xb_mysql_query(connection, - "SHOW ENGINE INNODB STATUS", - true, false)) { - if (MYSQL_ROW row = mysql_fetch_row(res)) { - const char *p= strstr(row[2], lsn_prefix); - DBUG_ASSERT(p); - if (p) { - p += sizeof lsn_prefix - 1; - lsn = lsn_t(strtoll(p, NULL, 10)); - } - } - mysql_free_result(res); - } - return lsn; -} - lsn_t server_lsn_after_lock; extern void backup_wait_for_lsn(lsn_t lsn); -/** Start --backup */ -bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, - CorruptedPages &corrupted_pages) -{ - if (!opt_no_lock) { - if (opt_safe_slave_backup) { - if (!wait_for_safe_slave(mysql_connection)) { - return(false); - } - } - - if (!backup_files(ds_data, fil_path_to_mysql_datadir, true)) { - return(false); - } - - history_lock_time = time(NULL); - - if (!lock_tables(mysql_connection)) { - return(false); - } - server_lsn_after_lock = get_current_lsn(mysql_connection); - } - - if (!backup_files(ds_data, fil_path_to_mysql_datadir, false)) { - return(false); - } - - if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir, - "aws-kms-key") || - !backup_files_from_datadir(ds_data, - aria_log_dir_path, - "aria_log")) { - return false; - } - if (has_rocksdb_plugin()) { - rocksdb_create_checkpoint(); - } - - msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock); - backup_wait_for_lsn(server_lsn_after_lock); - DBUG_EXECUTE_FOR_KEY("sleep_after_waiting_for_lsn", {}, - { - ulong milliseconds = strtoul(dbug_val, NULL, 10); - msg("sleep_after_waiting_for_lsn"); - my_sleep(milliseconds*1000UL); - }); - - corrupted_pages.backup_fix_ddl(ds_data, ds_meta); - - // There is no need to stop slave thread before coping non-Innodb data when - // --no-lock option is used because --no-lock option requires that no DDL or - // DML to non-transaction tables can occur. - if (opt_no_lock) { - if (opt_safe_slave_backup) { - if (!wait_for_safe_slave(mysql_connection)) { - return(false); - } - } - } - - if (opt_slave_info) { - lock_binlog_maybe(mysql_connection); - - if (!write_slave_info(ds_data, mysql_connection)) { - return(false); - } - } - - /* The only reason why Galera/binlog info is written before - wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup - binary will start streamig a temporary copy of REDO log to stdout and - thus, any streaming from innobackupex would interfere. The only way to - avoid that is to have a single process, i.e. merge innobackupex and - xtrabackup. */ - if (opt_galera_info) { - if (!write_galera_info(ds_data, mysql_connection)) { - return(false); - } - } - - if (opt_binlog_info == BINLOG_INFO_ON) { - - lock_binlog_maybe(mysql_connection); - write_binlog_info(ds_data, mysql_connection); - } - - if (!opt_no_lock) { - msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS..."); - xb_mysql_query(mysql_connection, - "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false); - } - - return(true); -} - -/** Release resources after backup_start() */ +/** Release resources after backup_files() */ void backup_release() { - /* release all locks */ - if (!opt_no_lock) { - unlock_all(mysql_connection); - history_lock_time = 0; - } else { - history_lock_time = time(NULL) - history_lock_time; - } - if (opt_lock_ddl_per_table) { mdl_unlock_all(); } @@ -1530,11 +1372,11 @@ void backup_release() static const char *default_buffer_pool_file = "ib_buffer_pool"; -/** Finish after backup_start() and backup_release() */ +/** Finish after backup_files() and backup_release() */ bool backup_finish(ds_ctxt *ds_data) { /* Copy buffer pool dump or LRU dump */ - if (!opt_rsync && opt_galera_info) { + if (opt_galera_info) { if (buffer_pool_filename && file_exists(buffer_pool_filename)) { ds_data->copy_file(buffer_pool_filename, default_buffer_pool_file, 0); } @@ -1893,8 +1735,6 @@ copy_back() return(false); } - srv_max_n_threads = 1000; - /* copy undo tablespaces */ Copy_back_dst_dir dst_dir_buf; @@ -1922,7 +1762,8 @@ copy_back() dst_dir = dst_dir_buf.make(srv_log_group_home_dir); - /* --backup generates a single ib_logfile0, which we must copy. */ + /* --backup generates a single LOG_FILE_NAME, which we must copy + if it exists. */ ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); if (!(ret = copy_or_move_file(ds_tmp, LOG_FILE_NAME, LOG_FILE_NAME, @@ -2155,8 +1996,6 @@ decrypt_decompress() bool ret; datadir_iter_t *it = NULL; - srv_max_n_threads = 1000; - /* cd to backup directory */ if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) { @@ -2169,8 +2008,6 @@ decrypt_decompress() it = datadir_iter_new(".", false); - ut_a(xtrabackup_parallel >= 0); - ret = run_data_threads(it, decrypt_decompress_thread_func, xtrabackup_parallel ? xtrabackup_parallel : 1); @@ -2192,9 +2029,9 @@ decrypt_decompress() Do not copy the Innodb files (ibdata1, redo log files), as this is done in a separate step. */ -static bool backup_files_from_datadir(ds_ctxt_t *ds_data, - const char *dir_path, - const char *prefix) +bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix) { os_file_dir_t dir = os_file_opendir(dir_path); if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false; @@ -2218,10 +2055,6 @@ static bool backup_files_from_datadir(ds_ctxt_t *ds_data, pname = info.name; if (!starts_with(pname, prefix)) - /* For ES exchange the above line with the following code: - (!xtrabackup_prepare || !xtrabackup_incremental_dir || - !starts_with(pname, "aria_log"))) - */ continue; if (xtrabackup_prepare && xtrabackup_incremental_dir && @@ -2244,7 +2077,7 @@ static int rocksdb_remove_checkpoint_directory() return 0; } -static bool has_rocksdb_plugin() +bool has_rocksdb_plugin() { static bool first_time = true; static bool has_plugin= false; @@ -2390,7 +2223,7 @@ static void rocksdb_unlock_checkpoint() #define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint" static char rocksdb_checkpoint_dir[FN_REFLEN]; -static void rocksdb_create_checkpoint() +void rocksdb_create_checkpoint() { MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true); MYSQL_ROW row = mysql_fetch_row(result); @@ -2470,3 +2303,39 @@ static void rocksdb_copy_back(ds_ctxt *ds_data) { mkdirp(rocksdb_home_dir, 0777, MYF(0)); ds_data->copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back); } + +void foreach_file_in_db_dirs( + const char *dir_path, std::function<bool(const char *)> func) { + DBUG_ASSERT(dir_path); + + datadir_iter_t *it; + datadir_node_t node; + + datadir_node_init(&node); + it = datadir_iter_new(dir_path); + + while (datadir_iter_next(it, &node)) + if (!node.is_empty_dir && !func(node.filepath)) + break; + + datadir_iter_free(it); + datadir_node_free(&node); +} + +void foreach_file_in_datadir( + const char *dir_path, std::function<bool(const char *)> func) +{ + DBUG_ASSERT(dir_path); + os_file_dir_t dir = os_file_opendir(dir_path); + os_file_stat_t info; + while (os_file_readdir_next_file(dir_path, dir, &info) == 0) { + if (info.type != OS_FILE_TYPE_FILE) + continue; + const char *pname = strrchr(info.name, IF_WIN('\\', '/')); + if (!pname) + pname = info.name; + if (!func(pname)) + break; + } + os_file_closedir(dir); +} diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index b5aaf312..409e7839 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -2,6 +2,7 @@ #ifndef XTRABACKUP_BACKUP_COPY_H #define XTRABACKUP_BACKUP_COPY_H +#include <functional> #include <my_global.h> #include <mysql.h> #include "datasink.h" @@ -21,11 +22,10 @@ bool equal_paths(const char *first, const char *second); /** Start --backup */ -bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, - CorruptedPages &corrupted_pages); -/** Release resources after backup_start() */ +bool backup_files(ds_ctxt *ds_data, const char *from); +/** Release resources after backup_files() */ void backup_release(); -/** Finish after backup_start() and backup_release() */ +/** Finish after backup_files() and backup_release() */ bool backup_finish(ds_ctxt *ds_data); bool apply_log_finish(); @@ -38,7 +38,25 @@ is_path_separator(char); bool directory_exists(const char *dir, bool create); -lsn_t -get_current_lsn(MYSQL *connection); - +bool has_rocksdb_plugin(); +void rocksdb_create_checkpoint(); +void foreach_file_in_db_dirs( + const char *dir_path, std::function<bool(const char *)> func); +void foreach_file_in_datadir( + const char *dir_path, std::function<bool(const char *)> func); +bool ends_with(const char *str, const char *suffix); +bool starts_with(const char *str, const char *prefix); +void parse_db_table_from_file_path( + const char *filepath, char *dbname, char *tablename); +const char *trim_dotslash(const char *path); +bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix); + +bool is_system_table(const char *dbname, const char *tablename); +std::unique_ptr<std::vector<std::string>> + find_files(const char *dir_path, const char *prefix, const char *suffix); +bool file_exists(const char *filename); +bool +filename_matches(const char *filename, const char **ext_list); #endif diff --git a/extra/mariabackup/backup_debug.h b/extra/mariabackup/backup_debug.h index 777b4f4a..9286bc7b 100644 --- a/extra/mariabackup/backup_debug.h +++ b/extra/mariabackup/backup_debug.h @@ -1,5 +1,6 @@ #pragma once #include "my_dbug.h" + #ifndef DBUG_OFF char *dbug_mariabackup_get_val(const char *event, fil_space_t::name_type key); /* @@ -14,11 +15,21 @@ To use this facility, you need to for the variable) 3. start mariabackup with --dbug=+d,debug_mariabackup_events */ -#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) \ - DBUG_EXECUTE_IF("mariabackup_inject_code", \ - { char *dbug_val= dbug_mariabackup_get_val(EVENT, KEY); \ - if (dbug_val) CODE }) +extern void dbug_mariabackup_event( + const char *event, const fil_space_t::name_type key, bool need_lock); +#define DBUG_MARIABACKUP_EVENT(A, B) \ + DBUG_EXECUTE_IF("mariabackup_events", \ + dbug_mariabackup_event(A,B,false);); +#define DBUG_MARIABACKUP_EVENT_LOCK(A, B) \ + DBUG_EXECUTE_IF("mariabackup_events", \ + dbug_mariabackup_event(A,B, true);); +#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) \ + DBUG_EXECUTE_IF("mariabackup_inject_code", {\ + char *dbug_val = dbug_mariabackup_get_val(EVENT, KEY); \ + if (dbug_val && *dbug_val) CODE \ + }) #else +#define DBUG_MARIABACKUP_EVENT(A,B) +#define DBUG_MARIABACKUP_EVENT_LOCK(A,B) #define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) #endif - diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index c2f15da4..2aad6004 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -47,6 +47,12 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include <stdlib.h> #include <string.h> #include <limits> +#ifdef HAVE_PWD_H +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#include <pwd.h> +#endif #include "common.h" #include "xtrabackup.h" #include "srv0srv.h" @@ -54,19 +60,19 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include "backup_copy.h" #include "backup_mysql.h" #include "mysqld.h" -#include "xb_plugin.h" +#include "encryption_plugin.h" #include <sstream> #include <sql_error.h> #include "page0zip.h" +#include "backup_debug.h" char *tool_name; -char tool_args[2048]; +char tool_args[8192]; ulong mysql_server_version; /* server capabilities */ bool have_changed_page_bitmaps = false; -bool have_backup_locks = false; bool have_lock_wait_timeout = false; bool have_galera_enabled = false; bool have_multi_threaded_slave = false; @@ -92,11 +98,54 @@ MYSQL *mysql_connection; extern my_bool opt_ssl_verify_server_cert, opt_use_ssl; + +/* + get_os_user() + Ressemles read_user_name() from libmariadb/libmariadb/mariadb_lib.c. +*/ + +#if !defined(_WIN32) + +#if defined(HAVE_GETPWUID) && defined(NO_GETPWUID_DECL) +struct passwd *getpwuid(uid_t); +char* getlogin(void); +#endif + +static const char *get_os_user() // Posix +{ + if (!geteuid()) + return "root"; +#ifdef HAVE_GETPWUID + struct passwd *pw; + const char *str; + if ((pw= getpwuid(geteuid())) != NULL) + return pw->pw_name; + if ((str= getlogin()) != NULL) + return str; +#endif + if ((str= getenv("USER")) || + (str= getenv("LOGNAME")) || + (str= getenv("LOGIN"))) + return str; + return NULL; +} + +#else + +static const char *get_os_user() // Windows +{ + return getenv("USERNAME"); +} + +#endif // _WIN32 + + MYSQL * xb_mysql_connect() { MYSQL *connection = mysql_init(NULL); char mysql_port_str[std::numeric_limits<int>::digits10 + 3]; + const char *user= opt_user ? opt_user : get_os_user(); sprintf(mysql_port_str, "%d", opt_port); @@ -126,7 +175,7 @@ xb_mysql_connect() msg("Connecting to MariaDB server host: %s, user: %s, password: %s, " "port: %s, socket: %s", opt_host ? opt_host : "localhost", - opt_user ? opt_user : "not set", + user ? user : "not set", opt_password ? "set" : "not set", opt_port != 0 ? mysql_port_str : "not set", opt_socket ? opt_socket : "not set"); @@ -147,7 +196,7 @@ xb_mysql_connect() if (!mysql_real_connect(connection, opt_host ? opt_host : "localhost", - opt_user, + user, opt_password, "" /*database*/, opt_port, opt_socket, 0)) { @@ -203,13 +252,14 @@ struct mysql_variable { static -void +uint read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, bool vertical_result) { MYSQL_RES *mysql_result; MYSQL_ROW row; mysql_variable *var; + uint n_values=0; mysql_result = xb_mysql_query(connection, query, true); @@ -223,6 +273,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, if (strcmp(var->name, name) == 0 && value != NULL) { *(var->value) = strdup(value); + n_values++; } } } @@ -239,6 +290,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, if (strcmp(var->name, name) == 0 && value != NULL) { *(var->value) = strdup(value); + n_values++; } } ++i; @@ -247,6 +299,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, } mysql_free_result(mysql_result); + return n_values; } @@ -311,7 +364,6 @@ bool get_mysql_vars(MYSQL *connection) { char *gtid_mode_var= NULL; char *version_var= NULL; - char *have_backup_locks_var= NULL; char *log_bin_var= NULL; char *lock_wait_timeout_var= NULL; char *wsrep_on_var= NULL; @@ -336,7 +388,6 @@ bool get_mysql_vars(MYSQL *connection) bool ret= true; mysql_variable mysql_vars[]= { - {"have_backup_locks", &have_backup_locks_var}, {"log_bin", &log_bin_var}, {"lock_wait_timeout", &lock_wait_timeout_var}, {"gtid_mode", >id_mode_var}, @@ -361,11 +412,6 @@ bool get_mysql_vars(MYSQL *connection) read_mysql_variables(connection, "SHOW VARIABLES", mysql_vars, true); - if (have_backup_locks_var != NULL && !opt_no_backup_locks) - { - have_backup_locks= true; - } - if (opt_binlog_info == BINLOG_INFO_AUTO) { if (log_bin_var != NULL && !strcmp(log_bin_var, "ON")) @@ -512,24 +558,6 @@ Query the server to find out what backup capabilities it supports. bool detect_mysql_capabilities_for_backup() { - const char *query = "SELECT 'INNODB_CHANGED_PAGES', COUNT(*) FROM " - "INFORMATION_SCHEMA.PLUGINS " - "WHERE PLUGIN_NAME LIKE 'INNODB_CHANGED_PAGES'"; - char *innodb_changed_pages = NULL; - mysql_variable vars[] = { - {"INNODB_CHANGED_PAGES", &innodb_changed_pages}, {NULL, NULL}}; - - if (xtrabackup_incremental) { - - read_mysql_variables(mysql_connection, query, vars, true); - - ut_ad(innodb_changed_pages != NULL); - - have_changed_page_bitmaps = (atoi(innodb_changed_pages) == 1); - - free_mysql_variables(vars); - } - /* do some sanity checks */ if (opt_galera_info && !have_galera_enabled) { msg("--galera-info is specified on the command " @@ -837,11 +865,11 @@ static void stop_query_killer() /*********************************************************************//** -Function acquires either a backup tables lock, if supported -by the server, or a global read lock (FLUSH TABLES WITH READ LOCK) -otherwise. +Function acquires backup locks @returns true if lock acquired */ -bool lock_tables(MYSQL *connection) + +bool +lock_for_backup_stage_start(MYSQL *connection) { if (have_lock_wait_timeout || opt_lock_wait_timeout) { @@ -854,12 +882,6 @@ bool lock_tables(MYSQL *connection) xb_mysql_query(connection, buf, false); } - if (have_backup_locks) - { - msg("Executing LOCK TABLES FOR BACKUP..."); - xb_mysql_query(connection, "LOCK TABLES FOR BACKUP", false); - return (true); - } if (opt_lock_wait_timeout) { @@ -884,8 +906,6 @@ bool lock_tables(MYSQL *connection) xb_mysql_query(connection, "BACKUP STAGE START", true); DBUG_MARIABACKUP_EVENT("after_backup_stage_start", {}); - xb_mysql_query(connection, "BACKUP STAGE BLOCK_COMMIT", true); - DBUG_MARIABACKUP_EVENT("after_backup_stage_block_commit", {}); /* Set the maximum supported session value for lock_wait_timeout to prevent unnecessary timeouts when the global value is changed from the default */ @@ -901,24 +921,68 @@ bool lock_tables(MYSQL *connection) return (true); } -/*********************************************************************//** -If backup locks are used, execute LOCK BINLOG FOR BACKUP provided that we are -not in the --no-lock mode and the lock has not been acquired already. -@returns true if lock acquired */ bool -lock_binlog_maybe(MYSQL *connection) -{ - if (have_backup_locks && !opt_no_lock && !binlog_locked) { - msg("Executing LOCK BINLOG FOR BACKUP..."); - xb_mysql_query(connection, "LOCK BINLOG FOR BACKUP", false); - binlog_locked = true; +lock_for_backup_stage_flush(MYSQL *connection) { + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + xb_mysql_query(connection, "BACKUP STAGE FLUSH", true); + if (opt_kill_long_queries_timeout) { + stop_query_killer(); + } + return true; +} - return(true); +bool +lock_for_backup_stage_block_ddl(MYSQL *connection) { + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + xb_mysql_query(connection, "BACKUP STAGE BLOCK_DDL", true); + DBUG_MARIABACKUP_EVENT("after_backup_stage_block_ddl", {}); + if (opt_kill_long_queries_timeout) { + stop_query_killer(); } + return true; +} - return(false); +bool +lock_for_backup_stage_commit(MYSQL *connection) { + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + xb_mysql_query(connection, "BACKUP STAGE BLOCK_COMMIT", true); + DBUG_MARIABACKUP_EVENT("after_backup_stage_block_commit", {}); + if (opt_kill_long_queries_timeout) { + stop_query_killer(); + } + return true; } +bool backup_lock(MYSQL *con, const char *table_name) { + static const std::string backup_lock_prefix("BACKUP LOCK "); + std::string backup_lock_query = backup_lock_prefix + table_name; + xb_mysql_query(con, backup_lock_query.c_str(), true); + return true; +} + +bool backup_unlock(MYSQL *con) { + xb_mysql_query(con, "BACKUP UNLOCK", true); + return true; +} + +std::unordered_set<std::string> +get_tables_in_use(MYSQL *con) { + std::unordered_set<std::string> result; + MYSQL_RES *q_res = + xb_mysql_query(con, "SHOW OPEN TABLES WHERE In_use = 1", true); + while (MYSQL_ROW row = mysql_fetch_row(q_res)) { + auto tk = table_key(row[0], row[1]); + msg("Table %s is in use", tk.c_str()); + result.insert(std::move(tk)); + } + return result; +} /*********************************************************************//** Releases either global read lock acquired with FTWRL and the binlog @@ -1353,77 +1417,103 @@ write_slave_info(ds_ctxt *datasink, MYSQL *connection) /*********************************************************************//** -Retrieves MySQL Galera and -saves it in a file. It also prints it to stdout. */ +Retrieves MySQL Galera and saves it in a file. It also prints it to stdout. + +We should create xtrabackup_galelera_info file even when backup locks +are used because donor's wsrep_gtid_domain_id is needed later in joiner. +Note that at this stage wsrep_local_state_uuid and wsrep_last_committed +are inconsistent but they are not used in joiner. Joiner will rewrite this file +at mariabackup --prepare phase and thus there is extra file donor_galera_info. +Information is needed to maitain wsrep_gtid_domain_id and gtid_binlog_pos +same across the cluster. If joiner node have different wsrep_gtid_domain_id +we should still receive effective domain id from the donor node, +and use it. +*/ bool write_galera_info(ds_ctxt *datasink, MYSQL *connection) { - char *state_uuid = NULL, *state_uuid55 = NULL; - char *last_committed = NULL, *last_committed55 = NULL; - char *domain_id = NULL, *domain_id55 = NULL; - bool result; - - mysql_variable status[] = { - {"Wsrep_local_state_uuid", &state_uuid}, - {"wsrep_local_state_uuid", &state_uuid55}, - {"Wsrep_last_committed", &last_committed}, - {"wsrep_last_committed", &last_committed55}, - {NULL, NULL} - }; + char *state_uuid = NULL, *state_uuid55 = NULL; + char *last_committed = NULL, *last_committed55 = NULL; + char *domain_id = NULL, *domain_id55 = NULL; + bool result=true; + uint n_values=0; + char *wsrep_on = NULL, *wsrep_on55 = NULL; + + mysql_variable vars[] = { + {"Wsrep_on", &wsrep_on}, + {"wsrep_on", &wsrep_on55}, + {NULL, NULL} + }; + + mysql_variable status[] = { + {"Wsrep_local_state_uuid", &state_uuid}, + {"wsrep_local_state_uuid", &state_uuid55}, + {"Wsrep_last_committed", &last_committed}, + {"wsrep_last_committed", &last_committed55}, + {NULL, NULL} + }; + + mysql_variable value[] = { + {"Wsrep_gtid_domain_id", &domain_id}, + {"wsrep_gtid_domain_id", &domain_id55}, + {NULL, NULL} + }; + + n_values= read_mysql_variables(connection, "SHOW VARIABLES", vars, true); + + if (n_values == 0 || (wsrep_on == NULL && wsrep_on55 == NULL)) + { + msg("Server is not Galera node thus --galera-info does not " + "have any effect."); + result = true; + goto cleanup; + } - mysql_variable value[] = { - {"Wsrep_gtid_domain_id", &domain_id}, - {"wsrep_gtid_domain_id", &domain_id55}, - {NULL, NULL} - }; + read_mysql_variables(connection, "SHOW STATUS", status, true); - /* When backup locks are supported by the server, we should skip - creating xtrabackup_galera_info file on the backup stage, because - wsrep_local_state_uuid and wsrep_last_committed will be inconsistent - without blocking commits. The state file will be created on the prepare - stage using the WSREP recovery procedure. */ - if (have_backup_locks) { - return(true); - } + if ((state_uuid == NULL && state_uuid55 == NULL) + || (last_committed == NULL && last_committed55 == NULL)) + { + msg("Warning: failed to get master wsrep state from SHOW STATUS."); + result = true; + goto cleanup; + } - read_mysql_variables(connection, "SHOW STATUS", status, true); + n_values= read_mysql_variables(connection, "SHOW VARIABLES LIKE 'wsrep%'", value, true); - if ((state_uuid == NULL && state_uuid55 == NULL) - || (last_committed == NULL && last_committed55 == NULL)) { - msg("Warning: failed to get master wsrep state from SHOW STATUS."); - result = true; - goto cleanup; - } + if (n_values == 0 || (domain_id == NULL && domain_id55 == NULL)) + { + msg("Warning: failed to get master wsrep state from SHOW VARIABLES."); + result = true; + goto cleanup; + } - read_mysql_variables(connection, "SHOW VARIABLES LIKE 'wsrep%'", value, true); + result= datasink->backup_file_printf(XTRABACKUP_GALERA_INFO, + "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, + last_committed ? last_committed : last_committed55, + domain_id ? domain_id : domain_id55); - if (domain_id == NULL && domain_id55 == NULL) { - msg("Warning: failed to get master wsrep state from SHOW VARIABLES."); - result = true; - goto cleanup; - } + if (result) + { + result= datasink->backup_file_printf(XTRABACKUP_DONOR_GALERA_INFO, + "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, + last_committed ? last_committed : last_committed55, + domain_id ? domain_id : domain_id55); + } - result = datasink->backup_file_printf(XTRABACKUP_GALERA_INFO, - "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, - last_committed ? last_committed : last_committed55, - domain_id ? domain_id : domain_id55); + if (result) + write_current_binlog_file(datasink, connection); - if (result) - { - result= datasink->backup_file_printf(XTRABACKUP_DONOR_GALERA_INFO, - "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, - last_committed ? last_committed : last_committed55, - domain_id ? domain_id : domain_id55); - } - if (result) - { - write_current_binlog_file(datasink, connection); - } + if (result) + msg("Writing Galera info succeeded with %s:%s %s", + state_uuid ? state_uuid : state_uuid55, + last_committed ? last_committed : last_committed55, + domain_id ? domain_id : domain_id55); cleanup: - free_mysql_variables(status); + free_mysql_variables(status); - return(result); + return(result); } @@ -1466,8 +1556,6 @@ write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection) if (gtid_exists) { size_t log_bin_dir_length; - lock_binlog_maybe(connection); - xb_mysql_query(connection, "FLUSH BINARY LOGS", false); read_mysql_variables(connection, "SHOW MASTER STATUS", @@ -1826,13 +1914,13 @@ bool write_backup_config_file(ds_ctxt *datasink) srv_log_file_size, srv_page_size, srv_undo_dir, - srv_undo_tablespaces, + (uint) srv_undo_tablespaces, page_zip_level, innobase_buffer_pool_filename ? "innodb_buffer_pool_filename=" : "", innobase_buffer_pool_filename ? innobase_buffer_pool_filename : "", - xb_plugin_get_config()); + encryption_plugin_get_config()); return rc; } @@ -1851,9 +1939,11 @@ char *make_argv(char *buf, size_t len, int argc, char **argv) if (strncmp(*argv, "--password", strlen("--password")) == 0) { arg = "--password=..."; } - left-= snprintf(buf + len - left, left, + uint l= snprintf(buf + len - left, left, "%s%c", arg, argc > 1 ? ' ' : 0); ++argv; --argc; + if (l < left) + left-= l; } return buf; @@ -1882,18 +1972,6 @@ select_history() return(true); } -bool -flush_changed_page_bitmaps() -{ - if (xtrabackup_incremental && have_changed_page_bitmaps && - !xtrabackup_incremental_force_scan) { - xb_mysql_query(mysql_connection, - "FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS", false); - } - return(true); -} - - /*********************************************************************//** Deallocate memory, disconnect from server, etc. @return true on success. */ @@ -1969,3 +2047,23 @@ mdl_unlock_all() mysql_close(mdl_con); spaceid_to_tablename.clear(); } + +ulonglong get_current_lsn(MYSQL *connection) +{ + static const char lsn_prefix[] = "\nLog sequence number "; + ulonglong lsn = 0; + if (MYSQL_RES *res = xb_mysql_query(connection, + "SHOW ENGINE INNODB STATUS", + true, false)) { + if (MYSQL_ROW row = mysql_fetch_row(res)) { + const char *p= strstr(row[2], lsn_prefix); + DBUG_ASSERT(p); + if (p) { + p += sizeof lsn_prefix - 1; + lsn = lsn_t(strtoll(p, NULL, 10)); + } + } + mysql_free_result(res); + } + return lsn; +} diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index 4b08da0b..c87efd21 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -2,13 +2,15 @@ #define XTRABACKUP_BACKUP_MYSQL_H #include <mysql.h> +#include <string> +#include <unordered_set> +#include "datasink.h" /* MariaDB version */ extern ulong mysql_server_version; /* server capabilities */ extern bool have_changed_page_bitmaps; -extern bool have_backup_locks; extern bool have_lock_wait_timeout; extern bool have_galera_enabled; extern bool have_multi_threaded_slave; @@ -35,9 +37,6 @@ capture_tool_command(int argc, char **argv); bool select_history(); -bool -flush_changed_page_bitmaps(); - void backup_cleanup(); @@ -75,7 +74,21 @@ bool lock_binlog_maybe(MYSQL *connection); bool -lock_tables(MYSQL *connection); +lock_for_backup_stage_start(MYSQL *connection); + +bool +lock_for_backup_stage_flush(MYSQL *connection); + +bool +lock_for_backup_stage_block_ddl(MYSQL *connection); + +bool +lock_for_backup_stage_commit(MYSQL *connection); + +bool backup_lock(MYSQL *con, const char *table_name); +bool backup_unlock(MYSQL *con); + +std::unordered_set<std::string> get_tables_in_use(MYSQL *con); bool wait_for_safe_slave(MYSQL *connection); @@ -86,5 +99,6 @@ write_galera_info(ds_ctxt *datasink, MYSQL *connection); bool write_slave_info(ds_ctxt *datasink, MYSQL *connection); +ulonglong get_current_lsn(MYSQL *connection); #endif diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc deleted file mode 100644 index 39a07a25..00000000 --- a/extra/mariabackup/changed_page_bitmap.cc +++ /dev/null @@ -1,1040 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2012 Percona Inc. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -/* Changed page bitmap implementation */ - -#include "changed_page_bitmap.h" - -#include "common.h" -#include "xtrabackup.h" -#include "srv0srv.h" - -/* TODO: copy-pasted shared definitions from the XtraDB bitmap write code. -Remove these on the first opportunity, i.e. single-binary XtraBackup. */ - -/* log0online.h */ - -/** Single bitmap file information */ -struct log_online_bitmap_file_t { - char name[FN_REFLEN]; /*!< Name with full path */ - pfs_os_file_t file; /*!< Handle to opened file */ - ib_uint64_t size; /*!< Size of the file */ - ib_uint64_t offset; /*!< Offset of the next read, - or count of already-read bytes - */ -}; - -/** A set of bitmap files containing some LSN range */ -struct log_online_bitmap_file_range_t { - size_t count; /*!< Number of files */ - /*!< Dynamically-allocated array of info about individual files */ - struct files_t { - char name[FN_REFLEN];/*!< Name of a file */ - lsn_t start_lsn; /*!< Starting LSN of data in this - file */ - ulong seq_num; /*!< Sequence number of this file */ - } *files; -}; - -/* log0online.c */ - -/** File name stem for bitmap files. */ -static const char* bmp_file_name_stem = "ib_modified_log_"; - -/** The bitmap file block size in bytes. All writes will be multiples of this. - */ -enum { - MODIFIED_PAGE_BLOCK_SIZE = 4096 -}; - -/** Offsets in a file bitmap block */ -enum { - MODIFIED_PAGE_IS_LAST_BLOCK = 0,/* 1 if last block in the current - write, 0 otherwise. */ - MODIFIED_PAGE_START_LSN = 4, /* The starting tracked LSN of this and - other blocks in the same write */ - MODIFIED_PAGE_END_LSN = 12, /* The ending tracked LSN of this and - other blocks in the same write */ - MODIFIED_PAGE_SPACE_ID = 20, /* The space ID of tracked pages in - this block */ - MODIFIED_PAGE_1ST_PAGE_ID = 24, /* The page ID of the first tracked - page in this block */ - MODIFIED_PAGE_BLOCK_UNUSED_1 = 28,/* Unused in order to align the start - of bitmap at 8 byte boundary */ - MODIFIED_PAGE_BLOCK_BITMAP = 32,/* Start of the bitmap itself */ - MODIFIED_PAGE_BLOCK_UNUSED_2 = MODIFIED_PAGE_BLOCK_SIZE - 8, - /* Unused in order to align the end of - bitmap at 8 byte boundary */ - MODIFIED_PAGE_BLOCK_CHECKSUM = MODIFIED_PAGE_BLOCK_SIZE - 4 - /* The checksum of the current block */ -}; - -/** Length of the bitmap data in a block */ -enum { MODIFIED_PAGE_BLOCK_BITMAP_LEN - = MODIFIED_PAGE_BLOCK_UNUSED_2 - MODIFIED_PAGE_BLOCK_BITMAP }; - -/** Length of the bitmap data in a block in page ids */ -enum { MODIFIED_PAGE_BLOCK_ID_COUNT = MODIFIED_PAGE_BLOCK_BITMAP_LEN * 8 }; - -typedef ib_uint64_t bitmap_word_t; - -/****************************************************************//** -Calculate a bitmap block checksum. Algorithm borrowed from -log_block_calc_checksum. -@return checksum */ -UNIV_INLINE -ulint -log_online_calc_checksum( -/*=====================*/ - const byte* block); /*!<in: bitmap block */ - -/****************************************************************//** -Provide a comparisson function for the RB-tree tree (space, -block_start_page) pairs. Actual implementation does not matter as -long as the ordering is full. -@return -1 if p1 < p2, 0 if p1 == p2, 1 if p1 > p2 -*/ -static -int -log_online_compare_bmp_keys( -/*========================*/ - const void* p1, /*!<in: 1st key to compare */ - const void* p2) /*!<in: 2nd key to compare */ -{ - const byte *k1 = (const byte *)p1; - const byte *k2 = (const byte *)p2; - - ulint k1_space = mach_read_from_4(k1 + MODIFIED_PAGE_SPACE_ID); - ulint k2_space = mach_read_from_4(k2 + MODIFIED_PAGE_SPACE_ID); - if (k1_space == k2_space) { - - ulint k1_start_page - = mach_read_from_4(k1 + MODIFIED_PAGE_1ST_PAGE_ID); - ulint k2_start_page - = mach_read_from_4(k2 + MODIFIED_PAGE_1ST_PAGE_ID); - return k1_start_page < k2_start_page - ? -1 : k1_start_page > k2_start_page ? 1 : 0; - } - return k1_space < k2_space ? -1 : 1; -} - -/****************************************************************//** -Calculate a bitmap block checksum. Algorithm borrowed from -log_block_calc_checksum. -@return checksum */ -UNIV_INLINE -ulint -log_online_calc_checksum( -/*=====================*/ - const byte* block) /*!<in: bitmap block */ -{ - ulint sum; - ulint sh; - ulint i; - - sum = 1; - sh = 0; - - for (i = 0; i < MODIFIED_PAGE_BLOCK_CHECKSUM; i++) { - - ulint b = block[i]; - sum &= 0x7FFFFFFFUL; - sum += b; - sum += b << sh; - sh++; - if (sh > 24) { - - sh = 0; - } - } - - return sum; -} - -/****************************************************************//** -Read one bitmap data page and check it for corruption. - -@return TRUE if page read OK, FALSE if I/O error */ -static -ibool -log_online_read_bitmap_page( -/*========================*/ - log_online_bitmap_file_t *bitmap_file, /*!<in/out: bitmap - file */ - byte *page, /*!<out: read page. Must be at - least MODIFIED_PAGE_BLOCK_SIZE - bytes long */ - ibool *checksum_ok) /*!<out: TRUE if page - checksum OK */ -{ - ulint checksum; - ulint actual_checksum; - - ut_a(bitmap_file->size >= MODIFIED_PAGE_BLOCK_SIZE); - ut_a(bitmap_file->offset - <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE); - ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); - if (DB_SUCCESS != - os_file_read(IORequestRead, bitmap_file->file, page, - bitmap_file->offset, MODIFIED_PAGE_BLOCK_SIZE, - nullptr)) { - /* The following call prints an error message */ - os_file_get_last_error(TRUE); - msg("InnoDB: Warning: failed reading changed page bitmap " - "file \'%s\'", bitmap_file->name); - return FALSE; - } - - bitmap_file->offset += MODIFIED_PAGE_BLOCK_SIZE; - ut_ad(bitmap_file->offset <= bitmap_file->size); - - checksum = mach_read_from_4(page + MODIFIED_PAGE_BLOCK_CHECKSUM); - actual_checksum = log_online_calc_checksum(page); - *checksum_ok = (checksum == actual_checksum); - - return TRUE; -} - -/*********************************************************************//** -Check the name of a given file if it's a changed page bitmap file and -return file sequence and start LSN name components if it is. If is not, -the values of output parameters are undefined. - -@return TRUE if a given file is a changed page bitmap file. */ -static -ibool -log_online_is_bitmap_file( -/*======================*/ - const os_file_stat_t* file_info, /*!<in: file to - check */ - ulong* bitmap_file_seq_num, /*!<out: bitmap file - sequence number */ - lsn_t* bitmap_file_start_lsn) /*!<out: bitmap file - start LSN */ -{ - char stem[FN_REFLEN]; - - ut_ad (strlen(file_info->name) < OS_FILE_MAX_PATH); - - return ((file_info->type == OS_FILE_TYPE_FILE - || file_info->type == OS_FILE_TYPE_LINK) - && (sscanf(file_info->name, "%[a-z_]%lu_" LSN_PF ".xdb", stem, - bitmap_file_seq_num, bitmap_file_start_lsn) == 3) - && (!strcmp(stem, bmp_file_name_stem))); -} - -/*********************************************************************//** -List the bitmap files in srv_data_home and setup their range that contains the -specified LSN interval. This range, if non-empty, will start with a file that -has the greatest LSN equal to or less than the start LSN and will include all -the files up to the one with the greatest LSN less than the end LSN. Caller -must free bitmap_files->files when done if bitmap_files set to non-NULL and -this function returned TRUE. Field bitmap_files->count might be set to a -larger value than the actual count of the files, and space for the unused array -slots will be allocated but cleared to zeroes. - -@return TRUE if succeeded -*/ -static -ibool -log_online_setup_bitmap_file_range( -/*===============================*/ - log_online_bitmap_file_range_t *bitmap_files, /*!<in/out: bitmap file - range */ - lsn_t range_start, /*!<in: start LSN */ - lsn_t range_end) /*!<in: end LSN */ -{ - os_file_dir_t bitmap_dir; - os_file_stat_t bitmap_dir_file_info; - ulong first_file_seq_num = ULONG_MAX; - ulong last_file_seq_num = 0; - lsn_t first_file_start_lsn = LSN_MAX; - - xb_ad(range_end >= range_start); - - bitmap_files->count = 0; - bitmap_files->files = NULL; - - /* 1st pass: size the info array */ - - bitmap_dir = os_file_opendir(srv_data_home); - if (UNIV_UNLIKELY(bitmap_dir == IF_WIN(INVALID_HANDLE_VALUE, NULL))) { - msg("InnoDB: Error: failed to open bitmap directory \'%s\'", - srv_data_home); - return FALSE; - } - - while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, - &bitmap_dir_file_info)) { - - ulong file_seq_num; - lsn_t file_start_lsn; - - if (!log_online_is_bitmap_file(&bitmap_dir_file_info, - &file_seq_num, - &file_start_lsn) - || file_start_lsn >= range_end) { - - continue; - } - - if (file_seq_num > last_file_seq_num) { - - last_file_seq_num = file_seq_num; - } - - if (file_start_lsn >= range_start - || file_start_lsn == first_file_start_lsn - || first_file_start_lsn > range_start) { - - /* A file that falls into the range */ - - if (file_start_lsn < first_file_start_lsn) { - - first_file_start_lsn = file_start_lsn; - } - if (file_seq_num < first_file_seq_num) { - - first_file_seq_num = file_seq_num; - } - } else if (file_start_lsn > first_file_start_lsn) { - - /* A file that has LSN closer to the range start - but smaller than it, replacing another such file */ - first_file_start_lsn = file_start_lsn; - first_file_seq_num = file_seq_num; - } - } - - if (UNIV_UNLIKELY(os_file_closedir_failed(bitmap_dir))) { - os_file_get_last_error(TRUE); - msg("InnoDB: Error: cannot close \'%s\'",srv_data_home); - return FALSE; - } - - if (first_file_seq_num == ULONG_MAX && last_file_seq_num == 0) { - - bitmap_files->count = 0; - return TRUE; - } - - bitmap_files->count = last_file_seq_num - first_file_seq_num + 1; - - /* 2nd pass: get the file names in the file_seq_num order */ - - bitmap_dir = os_file_opendir(srv_data_home); - if (UNIV_UNLIKELY(bitmap_dir == IF_WIN(INVALID_HANDLE_VALUE, NULL))) { - msg("InnoDB: Error: failed to open bitmap directory \'%s\'", - srv_data_home); - return FALSE; - } - - bitmap_files->files = - static_cast<log_online_bitmap_file_range_t::files_t *> - (malloc(bitmap_files->count * sizeof(bitmap_files->files[0]))); - memset(bitmap_files->files, 0, - bitmap_files->count * sizeof(bitmap_files->files[0])); - - while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, - &bitmap_dir_file_info)) { - - ulong file_seq_num; - lsn_t file_start_lsn; - size_t array_pos; - - if (!log_online_is_bitmap_file(&bitmap_dir_file_info, - &file_seq_num, - &file_start_lsn) - || file_start_lsn >= range_end - || file_start_lsn < first_file_start_lsn) { - - continue; - } - - array_pos = file_seq_num - first_file_seq_num; - if (UNIV_UNLIKELY(array_pos >= bitmap_files->count)) { - - msg("InnoDB: Error: inconsistent bitmap file " - "directory"); - os_file_closedir(bitmap_dir); - free(bitmap_files->files); - return FALSE; - } - - if (file_seq_num > bitmap_files->files[array_pos].seq_num) { - - bitmap_files->files[array_pos].seq_num = file_seq_num; - strncpy(bitmap_files->files[array_pos].name, - bitmap_dir_file_info.name, FN_REFLEN - 1); - bitmap_files->files[array_pos].name[FN_REFLEN - 1] - = '\0'; - bitmap_files->files[array_pos].start_lsn - = file_start_lsn; - } - } - - if (UNIV_UNLIKELY(os_file_closedir_failed(bitmap_dir))) { - os_file_get_last_error(TRUE); - msg("InnoDB: Error: cannot close \'%s\'", srv_data_home); - free(bitmap_files->files); - return FALSE; - } - -#ifdef UNIV_DEBUG - ut_ad(bitmap_files->files[0].seq_num == first_file_seq_num); - - for (size_t i = 1; i < bitmap_files->count; i++) { - if (!bitmap_files->files[i].seq_num) { - - break; - } - ut_ad(bitmap_files->files[i].seq_num - > bitmap_files->files[i - 1].seq_num); - ut_ad(bitmap_files->files[i].start_lsn - >= bitmap_files->files[i - 1].start_lsn); - } -#endif - - return TRUE; -} - -/****************************************************************//** -Open a bitmap file for reading. - -@return whether opened successfully */ -static -bool -log_online_open_bitmap_file_read_only( -/*==================================*/ - const char* name, /*!<in: bitmap file - name without directory, - which is assumed to be - srv_data_home */ - log_online_bitmap_file_t* bitmap_file) /*!<out: opened bitmap - file */ -{ - bool success = false; - - xb_ad(name[0] != '\0'); - - snprintf(bitmap_file->name, FN_REFLEN, "%s%s", srv_data_home, name); - bitmap_file->file = os_file_create_simple_no_error_handling( - 0, bitmap_file->name, - OS_FILE_OPEN, OS_FILE_READ_ONLY, true, &success); - if (UNIV_UNLIKELY(!success)) { - - /* Here and below assume that bitmap file names do not - contain apostrophes, thus no need for ut_print_filename(). */ - msg("InnoDB: Warning: error opening the changed page " - "bitmap \'%s\'", bitmap_file->name); - return success; - } - - bitmap_file->size = os_file_get_size(bitmap_file->file); - bitmap_file->offset = 0; - -#ifdef __linux__ - posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_SEQUENTIAL); - posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE); -#endif - - return success; -} - -/****************************************************************//** -Diagnose one or both of the following situations if we read close to -the end of bitmap file: -1) Warn if the remainder of the file is less than one page. -2) Error if we cannot read any more full pages but the last read page -did not have the last-in-run flag set. - -@return FALSE for the error */ -static -ibool -log_online_diagnose_bitmap_eof( -/*===========================*/ - const log_online_bitmap_file_t* bitmap_file, /*!< in: bitmap file */ - ibool last_page_in_run)/*!< in: "last page in - run" flag value in the - last read page */ -{ - /* Check if we are too close to EOF to read a full page */ - if ((bitmap_file->size < MODIFIED_PAGE_BLOCK_SIZE) - || (bitmap_file->offset - > bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { - - if (UNIV_UNLIKELY(bitmap_file->offset != bitmap_file->size)) { - - /* If we are not at EOF and we have less than one page - to read, it's junk. This error is not fatal in - itself. */ - - msg("InnoDB: Warning: junk at the end of changed " - "page bitmap file \'%s\'.", bitmap_file->name); - } - - if (UNIV_UNLIKELY(!last_page_in_run)) { - - /* We are at EOF but the last read page did not finish - a run */ - /* It's a "Warning" here because it's not a fatal error - for the whole server */ - msg("InnoDB: Warning: changed page bitmap " - "file \'%s\' does not contain a complete run " - "at the end.", bitmap_file->name); - return FALSE; - } - } - return TRUE; -} - -/* End of copy-pasted definitions */ - -/** Iterator structure over changed page bitmap */ -struct xb_page_bitmap_range_struct { - const xb_page_bitmap *bitmap; /* Bitmap with data */ - ulint space_id; /* Space id for this - iterator */ - ulint bit_i; /* Bit index of the iterator - position in the current page */ - const ib_rbt_node_t *bitmap_node; /* Current bitmap tree node */ - const byte *bitmap_page; /* Current bitmap page */ - ulint current_page_id;/* Current page id */ -}; - -/****************************************************************//** -Print a diagnostic message on missing bitmap data for an LSN range. */ -static -void -xb_msg_missing_lsn_data( -/*====================*/ - lsn_t missing_interval_start, /*!<in: interval start */ - lsn_t missing_interval_end) /*!<in: interval end */ -{ - msg("mariabackup: warning: changed page data missing for LSNs between " - LSN_PF " and " LSN_PF, missing_interval_start, - missing_interval_end); -} - -/****************************************************************//** -Scan a bitmap file until data for a desired LSN or EOF is found and check that -the page before the starting one is not corrupted to ensure that the found page -indeed contains the very start of the desired LSN data. The caller must check -the page LSN values to determine if the bitmap file was scanned until the data -was found or until EOF. Page must be at least MODIFIED_PAGE_BLOCK_SIZE big. - -@return TRUE if the scan successful without corruption detected -*/ -static -ibool -xb_find_lsn_in_bitmap_file( -/*=======================*/ - log_online_bitmap_file_t *bitmap_file, /*!<in/out: bitmap - file */ - byte *page, /*!<in/out: last read - bitmap page */ - lsn_t *page_end_lsn, /*!<out: end LSN of the - last read page */ - lsn_t lsn) /*!<in: LSN to find */ -{ - ibool last_page_ok = TRUE; - ibool next_to_last_page_ok = TRUE; - - xb_ad (bitmap_file->size >= MODIFIED_PAGE_BLOCK_SIZE); - - *page_end_lsn = 0; - - while ((*page_end_lsn <= lsn) - && (bitmap_file->offset - <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { - - next_to_last_page_ok = last_page_ok; - if (!log_online_read_bitmap_page(bitmap_file, page, - &last_page_ok)) { - - return FALSE; - } - - *page_end_lsn = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); - } - - /* We check two pages here because the last read page already contains - the required LSN data. If the next to the last one page is corrupted, - then we have no way of telling if that page contained the required LSN - range data too */ - return last_page_ok && next_to_last_page_ok; -} - -/****************************************************************//** -Read the disk bitmap and build the changed page bitmap tree for the -LSN interval incremental_lsn to log_sys.next_checkpoint_lsn. - -@return the built bitmap tree or NULL if unable to read the full interval for -any reason. */ -xb_page_bitmap* -xb_page_bitmap_init(void) -/*=====================*/ -{ - log_online_bitmap_file_t bitmap_file; - lsn_t bmp_start_lsn = incremental_lsn; - const lsn_t bmp_end_lsn{log_sys.next_checkpoint_lsn}; - byte page[MODIFIED_PAGE_BLOCK_SIZE]; - lsn_t current_page_end_lsn; - xb_page_bitmap *result; - ibool last_page_in_run= FALSE; - log_online_bitmap_file_range_t bitmap_files; - size_t bmp_i; - ibool last_page_ok = TRUE; - - if (UNIV_UNLIKELY(bmp_start_lsn > bmp_end_lsn)) { - - msg("mariabackup: incremental backup LSN " LSN_PF - " is larger than than the last checkpoint LSN " LSN_PF - , bmp_start_lsn, bmp_end_lsn); - return NULL; - } - - if (!log_online_setup_bitmap_file_range(&bitmap_files, bmp_start_lsn, - bmp_end_lsn)) { - - return NULL; - } - - /* Only accept no bitmap files returned if start LSN == end LSN */ - if (bitmap_files.count == 0 && bmp_end_lsn != bmp_start_lsn) { - - return NULL; - } - - result = rbt_create(MODIFIED_PAGE_BLOCK_SIZE, - log_online_compare_bmp_keys); - - if (bmp_start_lsn == bmp_end_lsn) { - - /* Empty range - empty bitmap */ - return result; - } - - bmp_i = 0; - - if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].start_lsn - > bmp_start_lsn)) { - - /* The 1st file does not have the starting LSN data */ - xb_msg_missing_lsn_data(bmp_start_lsn, - bitmap_files.files[bmp_i].start_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* Skip any zero-sized files at the start */ - while ((bmp_i < bitmap_files.count - 1) - && (bitmap_files.files[bmp_i].start_lsn - == bitmap_files.files[bmp_i + 1].start_lsn)) { - - bmp_i++; - } - - /* Is the 1st bitmap file missing? */ - if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] == '\0')) { - - /* TODO: this is not the exact missing range */ - xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* Open the 1st bitmap file */ - if (UNIV_UNLIKELY(!log_online_open_bitmap_file_read_only( - bitmap_files.files[bmp_i].name, - &bitmap_file))) { - - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* If the 1st file is truncated, no data. Not merged with the case - below because zero-length file indicates not a corruption but missing - subsequent files instead. */ - if (UNIV_UNLIKELY(bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE)) { - - xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - /* Find the start of the required LSN range in the file */ - if (UNIV_UNLIKELY(!xb_find_lsn_in_bitmap_file(&bitmap_file, page, - ¤t_page_end_lsn, - bmp_start_lsn))) { - - msg("mariabackup: Warning: changed page bitmap file " - "\'%s\' corrupted", bitmap_file.name); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - last_page_in_run - = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); - - if (UNIV_UNLIKELY(!log_online_diagnose_bitmap_eof(&bitmap_file, - last_page_in_run))) { - - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - if (UNIV_UNLIKELY(current_page_end_lsn < bmp_start_lsn)) { - - xb_msg_missing_lsn_data(current_page_end_lsn, bmp_start_lsn); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - /* 1st bitmap page found, add it to the tree. */ - rbt_insert(result, page, page); - - /* Read next pages/files until all required data is read */ - while (last_page_ok - && (current_page_end_lsn < bmp_end_lsn - || (current_page_end_lsn == bmp_end_lsn - && !last_page_in_run))) { - - ib_rbt_bound_t tree_search_pos; - - /* If EOF, advance the file skipping over any empty files */ - while (bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE - || (bitmap_file.offset - > bitmap_file.size - MODIFIED_PAGE_BLOCK_SIZE)) { - - os_file_close(bitmap_file.file); - - if (UNIV_UNLIKELY( - !log_online_diagnose_bitmap_eof( - &bitmap_file, last_page_in_run))) { - - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - bmp_i++; - - if (UNIV_UNLIKELY(bmp_i == bitmap_files.count - || (bitmap_files.files[bmp_i].seq_num - == 0))) { - - xb_msg_missing_lsn_data(current_page_end_lsn, - bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - /* Is the next file missing? */ - if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] - == '\0')) { - - /* TODO: this is not the exact missing range */ - xb_msg_missing_lsn_data(bitmap_files.files - [bmp_i - 1].start_lsn, - bmp_end_lsn); - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - - if (UNIV_UNLIKELY( - !log_online_open_bitmap_file_read_only( - bitmap_files.files[bmp_i].name, - &bitmap_file))) { - - rbt_free(result); - free(bitmap_files.files); - return NULL; - } - } - - if (UNIV_UNLIKELY( - !log_online_read_bitmap_page(&bitmap_file, page, - &last_page_ok))) { - - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - if (UNIV_UNLIKELY(!last_page_ok)) { - - msg("mariabackup: warning: changed page bitmap file " - "\'%s\' corrupted.", bitmap_file.name); - rbt_free(result); - free(bitmap_files.files); - os_file_close(bitmap_file.file); - return NULL; - } - - /* Merge the current page with an existing page or insert a new - page into the tree */ - - if (!rbt_search(result, &tree_search_pos, page)) { - - /* Merge the bitmap pages */ - byte *existing_page - = rbt_value(byte, tree_search_pos.last); - bitmap_word_t *bmp_word_1 = (bitmap_word_t *) - (existing_page + MODIFIED_PAGE_BLOCK_BITMAP); - bitmap_word_t *bmp_end = (bitmap_word_t *) - (existing_page + MODIFIED_PAGE_BLOCK_UNUSED_2); - bitmap_word_t *bmp_word_2 = (bitmap_word_t *) - (page + MODIFIED_PAGE_BLOCK_BITMAP); - while (bmp_word_1 < bmp_end) { - - *bmp_word_1++ |= *bmp_word_2++; - } - xb_a (bmp_word_1 == bmp_end); - } else { - - /* Add a new page */ - rbt_add_node(result, &tree_search_pos, page); - } - - current_page_end_lsn - = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); - last_page_in_run - = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); - } - - xb_a (current_page_end_lsn >= bmp_end_lsn); - - free(bitmap_files.files); - os_file_close(bitmap_file.file); - - return result; -} - -/****************************************************************//** -Free the bitmap tree. */ -void -xb_page_bitmap_deinit( -/*==================*/ - xb_page_bitmap* bitmap) /*!<in/out: bitmap tree */ -{ - if (bitmap) { - - rbt_free(bitmap); - } -} - -/****************************************************************//** -Advance to the next bitmap page or setup the first bitmap page for the -given bitmap range. Assumes that bitmap_range->bitmap_page has been -already found/bumped by rbt_search()/rbt_next(). - -@return FALSE if no more bitmap data for the range space ID */ -static -ibool -xb_page_bitmap_setup_next_page( -/*===========================*/ - xb_page_bitmap_range* bitmap_range) /*!<in/out: the bitmap range */ -{ - ulint new_space_id; - ulint new_1st_page_id; - - if (bitmap_range->bitmap_node == NULL) { - - bitmap_range->current_page_id = ULINT_UNDEFINED; - return FALSE; - } - - bitmap_range->bitmap_page = rbt_value(byte, bitmap_range->bitmap_node); - - new_space_id = mach_read_from_4(bitmap_range->bitmap_page - + MODIFIED_PAGE_SPACE_ID); - if (new_space_id != bitmap_range->space_id) { - - /* No more data for the current page id. */ - xb_a(new_space_id > bitmap_range->space_id); - bitmap_range->current_page_id = ULINT_UNDEFINED; - return FALSE; - } - - new_1st_page_id = mach_read_from_4(bitmap_range->bitmap_page + - MODIFIED_PAGE_1ST_PAGE_ID); - xb_a (new_1st_page_id >= bitmap_range->current_page_id - || bitmap_range->current_page_id == ULINT_UNDEFINED); - - bitmap_range->current_page_id = new_1st_page_id; - bitmap_range->bit_i = 0; - - return TRUE; -} - -/** Find the node with the smallest key that greater than equal to search key. -@param[in] tree red-black tree -@param[in] key search key -@return node with the smallest greater-than-or-equal key -@retval NULL if none was found */ -static -const ib_rbt_node_t* -rbt_lower_bound(const ib_rbt_t* tree, const void* key) -{ - ut_ad(!tree->cmp_arg); - const ib_rbt_node_t* ge = NULL; - - for (const ib_rbt_node_t *node = tree->root->left; - node != tree->nil; ) { - int result = tree->compare(node->value, key); - - if (result < 0) { - node = node->right; - } else { - ge = node; - if (result == 0) { - break; - } - - node = node->left; - } - } - - return(ge); -} - -/****************************************************************//** -Set up a new bitmap range iterator over a given space id changed -pages in a given bitmap. - -@return bitmap range iterator */ -xb_page_bitmap_range* -xb_page_bitmap_range_init( -/*======================*/ - xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */ - ulint space_id) /*!< in: space id */ -{ - byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; - xb_page_bitmap_range *result - = static_cast<xb_page_bitmap_range *>(malloc(sizeof(*result))); - - memset(result, 0, sizeof(*result)); - result->bitmap = bitmap; - result->space_id = space_id; - result->current_page_id = ULINT_UNDEFINED; - - /* Search for the 1st page for the given space id */ - /* This also sets MODIFIED_PAGE_1ST_PAGE_ID to 0, which is what we - want. */ - memset(search_page, 0, MODIFIED_PAGE_BLOCK_SIZE); - mach_write_to_4(search_page + MODIFIED_PAGE_SPACE_ID, space_id); - - result->bitmap_node = rbt_lower_bound(result->bitmap, search_page); - - xb_page_bitmap_setup_next_page(result); - - return result; -} - -/****************************************************************//** -Get the value of the bitmap->range->bit_i bitmap bit - -@return the current bit value */ -static inline -ibool -is_bit_set( -/*=======*/ - const xb_page_bitmap_range* bitmap_range) /*!< in: bitmap - range */ -{ - return ((*(((bitmap_word_t *)(bitmap_range->bitmap_page - + MODIFIED_PAGE_BLOCK_BITMAP)) - + (bitmap_range->bit_i >> 6))) - & (1ULL << (bitmap_range->bit_i & 0x3F))) ? TRUE : FALSE; -} - -/****************************************************************//** -Get the next page id that has its bit set or cleared, i.e. equal to -bit_value. - -@return page id */ -ulint -xb_page_bitmap_range_get_next_bit( -/*==============================*/ - xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */ - ibool bit_value) /*!< in: bit value */ -{ - if (UNIV_UNLIKELY(bitmap_range->current_page_id - == ULINT_UNDEFINED)) { - - return ULINT_UNDEFINED; - } - - do { - while (bitmap_range->bit_i < MODIFIED_PAGE_BLOCK_ID_COUNT) { - - while (is_bit_set(bitmap_range) != bit_value - && (bitmap_range->bit_i - < MODIFIED_PAGE_BLOCK_ID_COUNT)) { - - bitmap_range->current_page_id++; - bitmap_range->bit_i++; - } - - if (bitmap_range->bit_i - < MODIFIED_PAGE_BLOCK_ID_COUNT) { - - ulint result = bitmap_range->current_page_id; - bitmap_range->current_page_id++; - bitmap_range->bit_i++; - return result; - } - } - - bitmap_range->bitmap_node - = rbt_next(bitmap_range->bitmap, - bitmap_range->bitmap_node); - - } while (xb_page_bitmap_setup_next_page(bitmap_range)); - - return ULINT_UNDEFINED; -} - -/****************************************************************//** -Free the bitmap range iterator. */ -void -xb_page_bitmap_range_deinit( -/*========================*/ - xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */ -{ - free(bitmap_range); -} diff --git a/extra/mariabackup/changed_page_bitmap.h b/extra/mariabackup/changed_page_bitmap.h deleted file mode 100644 index 8d504359..00000000 --- a/extra/mariabackup/changed_page_bitmap.h +++ /dev/null @@ -1,85 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2012 Percona Inc. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -/* Changed page bitmap interface */ - -#ifndef XB_CHANGED_PAGE_BITMAP_H -#define XB_CHANGED_PAGE_BITMAP_H - -#include <ut0rbt.h> -#include <fil0fil.h> - -/* The changed page bitmap structure */ -typedef ib_rbt_t xb_page_bitmap; - -struct xb_page_bitmap_range_struct; - -/* The bitmap range iterator over one space id */ -typedef struct xb_page_bitmap_range_struct xb_page_bitmap_range; - -/****************************************************************//** -Read the disk bitmap and build the changed page bitmap tree for the -LSN interval incremental_lsn to log_sys.next_checkpoint_lsn. - -@return the built bitmap tree */ -xb_page_bitmap* -xb_page_bitmap_init(void); -/*=====================*/ - -/****************************************************************//** -Free the bitmap tree. */ -void -xb_page_bitmap_deinit( -/*==================*/ - xb_page_bitmap* bitmap); /*!<in/out: bitmap tree */ - - -/****************************************************************//** -Set up a new bitmap range iterator over a given space id changed -pages in a given bitmap. - -@return bitmap range iterator */ -xb_page_bitmap_range* -xb_page_bitmap_range_init( -/*======================*/ - xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */ - ulint space_id); /*!< in: space id */ - -/****************************************************************//** -Get the next page id that has its bit set or cleared, i.e. equal to -bit_value. - -@return page id */ -ulint -xb_page_bitmap_range_get_next_bit( -/*==============================*/ - xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */ - ibool bit_value); /*!< in: bit value */ - -/****************************************************************//** -Free the bitmap range iterator. */ -void -xb_page_bitmap_range_deinit( -/*========================*/ - xb_page_bitmap_range* bitmap_range); /*! in/out: bitmap range */ - -#endif diff --git a/extra/mariabackup/common.h b/extra/mariabackup/common.h index 89b189e3..6fde514e 100644 --- a/extra/mariabackup/common.h +++ b/extra/mariabackup/common.h @@ -23,7 +23,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include <my_global.h> #include <mysql_version.h> -#include <fcntl.h> #include <stdarg.h> #include <my_sys.h> @@ -143,7 +142,7 @@ static inline ATTRIBUTE_FORMAT(printf, 1,2) ATTRIBUTE_NORETURN void die(const ch # define POSIX_FADV_NORMAL # define POSIX_FADV_SEQUENTIAL # define POSIX_FADV_DONTNEED -# define posix_fadvise(a,b,c,d) do {} while(0) +# define posix_fadvise(fd, offset, len, advice) do { (void)offset; } while(0) #endif /*********************************************************************** diff --git a/extra/mariabackup/common_engine.cc b/extra/mariabackup/common_engine.cc new file mode 100644 index 00000000..a4a87062 --- /dev/null +++ b/extra/mariabackup/common_engine.cc @@ -0,0 +1,512 @@ +#include "common_engine.h" +#include "backup_copy.h" +#include "xtrabackup.h" +#include "common.h" +#include "backup_debug.h" + +#include <unordered_map> +#include <atomic> +#include <memory> +#include <chrono> + +namespace common_engine { + +class Table { +public: + Table(std::string &db, std::string &table, std::string &fs_name) : + m_db(std::move(db)), m_table(std::move(table)), + m_fs_name(std::move(fs_name)) {} + virtual ~Table() {} + void add_file_name(const char *file_name) { m_fnames.push_back(file_name); } + virtual bool copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, + bool finalize, unsigned thread_num); + std::string &get_db() { return m_db; } + std::string &get_table() { return m_table; } + std::string &get_version() { return m_version; } + +protected: + std::string m_db; + std::string m_table; + std::string m_fs_name; + std::string m_version; + std::vector<std::string> m_fnames; +}; + +bool +Table::copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool, unsigned thread_num) { + static const size_t buf_size = 10 * 1024 * 1024; + std::unique_ptr<uchar[]> buf; + bool result = false; + File frm_file = -1; + std::vector<File> files; + bool locked = false; + std::string full_tname("`"); + full_tname.append(m_db).append("`.`").append(m_table).append("`"); + + if (!no_lock && !backup_lock(con, full_tname.c_str())) { + msg(thread_num, "Error on executing BACKUP LOCK for table %s", + full_tname.c_str()); + goto exit; + } + else + locked = !no_lock; + + if ((frm_file = mysql_file_open(key_file_frm, (m_fs_name + ".frm").c_str(), + O_RDONLY | O_SHARE, MYF(0))) < 0 && !m_fnames.empty() && + !ends_with(m_fnames[0].c_str(), ".ARZ") && + !ends_with(m_fnames[0].c_str(), ".ARM")) { + // Don't treat it as error, as the table can be dropped after it + // was added to queue for copying + result = true; + goto exit; + } + + for (const auto &fname : m_fnames) { + File file = mysql_file_open(0, fname.c_str(),O_RDONLY | O_SHARE, MYF(0)); + if (file < 0) { + msg(thread_num, "Error on file %s open during %s table copy", + fname.c_str(), full_tname.c_str()); + goto exit; + } + files.push_back(file); + } + + if (locked && !backup_unlock(con)) { + msg(thread_num, "Error on BACKUP UNLOCK for table %s", full_tname.c_str()); + locked = false; + goto exit; + } + + locked = false; + + buf.reset(new uchar[buf_size]); + + for (size_t i = 0; i < m_fnames.size(); ++i) { + ds_file_t *dst_file = nullptr; + size_t bytes_read; + size_t copied_size = 0; + MY_STAT stat_info; + + if (my_fstat(files[i], &stat_info, MYF(0))) { + msg(thread_num, "error: failed to get stat info for file %s of " + "table %s", m_fnames[i].c_str(), full_tname.c_str()); + goto exit; + } + + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back) ? + m_fnames[i].c_str() : trim_dotslash(m_fnames[i].c_str()); + + dst_file = ds_open(ds, dst_path, &stat_info, false); + if (!dst_file) { + msg(thread_num, "error: cannot open destination stream for %s, table %s", + dst_path, full_tname.c_str()); + goto exit; + } + + while ((bytes_read = my_read(files[i], buf.get(), buf_size, MY_WME))) { + if (bytes_read == size_t(-1)) { + msg(thread_num, "error: file %s read for table %s", + m_fnames[i].c_str(), full_tname.c_str()); + ds_close(dst_file); + goto exit; + } + xtrabackup_io_throttling(); + if (ds_write(dst_file, buf.get(), bytes_read)) { + msg(thread_num, "error: file %s write for table %s", + dst_path, full_tname.c_str()); + ds_close(dst_file); + goto exit; + } + copied_size += bytes_read; + } + mysql_file_close(files[i], MYF(MY_WME)); + files[i] = -1; + ds_close(dst_file); + msg(thread_num, "Copied file %s for table %s, %zu bytes", + m_fnames[i].c_str(), full_tname.c_str(), copied_size); + } + + result = true; + +#ifndef DBUG_OFF + { + std::string sql_name(m_db); + sql_name.append("/").append(m_table); + DBUG_MARIABACKUP_EVENT_LOCK("after_ce_table_copy", fil_space_t::name_type(sql_name.data(), sql_name.size())); + } +#endif // DBUG_OFF +exit: + if (frm_file >= 0) { + m_version = ::read_table_version_id(frm_file); + mysql_file_close(frm_file, MYF(MY_WME)); + } + if (locked && !backup_unlock(con)) { + msg(thread_num, "Error on BACKUP UNLOCK for table %s", full_tname.c_str()); + result = false; + } + for (auto file : files) + if (file >= 0) + mysql_file_close(file, MYF(MY_WME)); + return result; +} + +// Append-only tables +class LogTable : public Table { + public: + LogTable(std::string &db, std::string &table, std::string &fs_name) : + Table(db, table, fs_name) {} + + virtual ~LogTable() { (void)close(); } + bool + copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool finalize, + unsigned thread_num) override; + bool close(); + private: + bool open(ds_ctxt_t *ds, unsigned thread_num); + std::vector<File> m_src; + std::vector<ds_file_t *> m_dst; +}; + +bool +LogTable::open(ds_ctxt_t *ds, unsigned thread_num) { + DBUG_ASSERT(m_src.empty()); + DBUG_ASSERT(m_dst.empty()); + + std::string full_tname("`"); + full_tname.append(m_db).append("`.`").append(m_table).append("`"); + + for (const auto &fname : m_fnames) { + File file = mysql_file_open(0, fname.c_str(),O_RDONLY | O_SHARE, MYF(0)); + if (file < 0) { + msg(thread_num, "Error on file %s open during %s log table copy", + fname.c_str(), full_tname.c_str()); + return false; + } + m_src.push_back(file); + + MY_STAT stat_info; + if (my_fstat(file, &stat_info, MYF(0))) { + msg(thread_num, "error: failed to get stat info for file %s of " + "log table %s", fname.c_str(), full_tname.c_str()); + return false; + } + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back) ? + fname.c_str() : trim_dotslash(fname.c_str()); + ds_file_t *dst_file = ds_open(ds, dst_path, &stat_info, false); + if (!dst_file) { + msg(thread_num, "error: cannot open destination stream for %s, " + "log table %s", dst_path, full_tname.c_str()); + return false; + } + m_dst.push_back(dst_file); + } + + File frm_file; + if ((frm_file = mysql_file_open(key_file_frm, (m_fs_name + ".frm").c_str(), + O_RDONLY | O_SHARE, MYF(0))) < 0 && !m_fnames.empty() && + !ends_with(m_fnames[0].c_str(), ".ARZ") && + !ends_with(m_fnames[0].c_str(), ".ARM")) { + msg(thread_num, "Error on .frm file open for log table %s", + full_tname.c_str()); + return false; + } + + m_version = ::read_table_version_id(frm_file); + mysql_file_close(frm_file, MYF(MY_WME)); + + return true; +} + +bool LogTable::close() { + while (!m_src.empty()) { + auto f = m_src.back(); + m_src.pop_back(); + mysql_file_close(f, MYF(MY_WME)); + } + while (!m_dst.empty()) { + auto f = m_dst.back(); + m_dst.pop_back(); + ds_close(f); + } + return true; +} + +bool +LogTable::copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool finalize, + unsigned thread_num) { + static const size_t buf_size = 10 * 1024 * 1024; + DBUG_ASSERT(ds); + DBUG_ASSERT(con); + if (m_src.empty() && !open(ds, thread_num)) { + close(); + return false; + } + DBUG_ASSERT(m_src.size() == m_dst.size()); + + std::unique_ptr<uchar[]> buf(new uchar[buf_size]); + for (size_t i = 0; i < m_src.size(); ++i) { + // .CSM can be rewritten (see write_meta_file() usage in ha_tina.cc) + if (!finalize && ends_with(m_fnames[i].c_str(), ".CSM")) + continue; + size_t bytes_read; + size_t copied_size = 0; + while ((bytes_read = my_read(m_src[i], buf.get(), buf_size, MY_WME))) { + if (bytes_read == size_t(-1)) { + msg(thread_num, "error: file %s read for log table %s", + m_fnames[i].c_str(), + std::string("`").append(m_db).append("`.`"). + append(m_table).append("`").c_str()); + close(); + return false; + } + xtrabackup_io_throttling(); + if (ds_write(m_dst[i], buf.get(), bytes_read)) { + msg(thread_num, "error: file %s write for log table %s", + m_fnames[i].c_str(), std::string("`").append(m_db).append("`.`"). + append(m_table).append("`").c_str()); + close(); + return false; + } + copied_size += bytes_read; + } + msg(thread_num, "Copied file %s for log table %s, %zu bytes", + m_fnames[i].c_str(), std::string("`").append(m_db).append("`.`"). + append(m_table).append("`").c_str(), copied_size); + } + + return true; +} + +class BackupImpl { + public: + BackupImpl( + const char *datadir_path, ds_ctxt_t *datasink, + std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool) : + m_datadir_path(datadir_path), m_ds(datasink), m_con_pool(con_pool), + m_process_table_jobs(thread_pool) {} + ~BackupImpl() { } + bool scan( + const std::unordered_set<std::string> &exclude_tables, + std::unordered_set<std::string> *out_processed_tables, + bool no_lock, bool collect_log_and_stats); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_table_post_copy_hook = hook; + } + bool copy_log_tables(bool finalize); + bool copy_stats_tables(); + bool wait_for_finish(); + bool close_log_tables(); + private: + + void process_table_job(Table *table, bool no_lock, bool delete_table, + bool finalize, unsigned thread_num); + + const char *m_datadir_path; + ds_ctxt_t *m_ds; + std::vector<MYSQL *> &m_con_pool; + TasksGroup m_process_table_jobs; + + post_copy_table_hook_t m_table_post_copy_hook; + std::unordered_map<table_key_t, std::unique_ptr<LogTable>> m_log_tables; + std::unordered_map<table_key_t, std::unique_ptr<Table>> m_stats_tables; +}; + +void BackupImpl::process_table_job(Table *table, bool no_lock, + bool delete_table, bool finalize, unsigned thread_num) { + int result = 0; + + if (!m_process_table_jobs.get_result()) + goto exit; + + if (!table->copy(m_ds, m_con_pool[thread_num], no_lock, finalize, thread_num)) + goto exit; + + if (m_table_post_copy_hook) + m_table_post_copy_hook(table->get_db(), table->get_table(), + table->get_version()); + + result = 1; + +exit: + if (delete_table) + delete table; + m_process_table_jobs.finish_task(result); +} + +bool BackupImpl::scan(const std::unordered_set<table_key_t> &exclude_tables, + std::unordered_set<table_key_t> *out_processed_tables, bool no_lock, + bool collect_log_and_stats) { + + msg("Start scanning common engine tables, need backup locks: %d, " + "collect log and stat tables: %d", no_lock, collect_log_and_stats); + + std::unordered_map<table_key_t, std::unique_ptr<Table>> found_tables; + + foreach_file_in_db_dirs(m_datadir_path, + [&](const char *file_path)->bool { + + static const char *ext_list[] = + {".MYD", ".MYI", ".MRG", ".ARM", ".ARZ", ".CSM", ".CSV", NULL}; + + bool is_aria = ends_with(file_path, ".MAD") || ends_with(file_path, ".MAI"); + + if (!collect_log_and_stats && is_aria) + return true; + + if (!is_aria && !filename_matches(file_path, ext_list)) + return true; + + if (check_if_skip_table(file_path)) { + msg("Skipping %s.", file_path); + return true; + } + + auto db_table_fs = convert_filepath_to_tablename(file_path); + auto tk = + table_key(std::get<0>(db_table_fs), std::get<1>(db_table_fs)); + + // log and stats tables are only collected in this function, + // so there is no need to filter out them with exclude_tables. + if (collect_log_and_stats) { + if (is_log_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str())) { + auto table_it = m_log_tables.find(tk); + if (table_it == m_log_tables.end()) { + msg("Log table found: %s", tk.c_str()); + table_it = m_log_tables.emplace(tk, + std::unique_ptr<LogTable>(new LogTable(std::get<0>(db_table_fs), + std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first; + } + msg("Collect log table file: %s", file_path); + table_it->second->add_file_name(file_path); + return true; + } + // Aria can handle statistics tables + else if (is_stats_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str()) && !is_aria) { + auto table_it = m_stats_tables.find(tk); + if (table_it == m_stats_tables.end()) { + msg("Stats table found: %s", tk.c_str()); + table_it = m_stats_tables.emplace(tk, + std::unique_ptr<Table>(new Table(std::get<0>(db_table_fs), + std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first; + } + msg("Collect stats table file: %s", file_path); + table_it->second->add_file_name(file_path); + return true; + } + } else if (is_log_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str()) || + is_stats_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str())) + return true; + + if (is_aria) + return true; + + if (exclude_tables.count(tk)) { + msg("Skip table %s at it is in exclude list", tk.c_str()); + return true; + } + + auto table_it = found_tables.find(tk); + if (table_it == found_tables.end()) { + table_it = found_tables.emplace(tk, + std::unique_ptr<Table>(new Table(std::get<0>(db_table_fs), + std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first; + } + + table_it->second->add_file_name(file_path); + + return true; + }); + + for (auto &table_it : found_tables) { + m_process_table_jobs.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.release(), + no_lock, true, false, std::placeholders::_1)); + if (out_processed_tables) + out_processed_tables->insert(table_it.first); + } + + msg("Stop scanning common engine tables"); + return true; +} + +bool BackupImpl::copy_log_tables(bool finalize) { + for (auto &table_it : m_log_tables) { + // Do not execute BACKUP LOCK for log tables as it's supposed + // that they must be copied on BLOCK_DDL and BLOCK_COMMIT locks. + m_process_table_jobs.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.get(), + true, false, finalize, std::placeholders::_1)); + } + return true; +} + +bool BackupImpl::copy_stats_tables() { + for (auto &table_it : m_stats_tables) { + // Do not execute BACKUP LOCK for stats tables as it's supposed + // that they must be copied on BLOCK_DDL and BLOCK_COMMIT locks. + // Delete stats table object after copy (see process_table_job()) + m_process_table_jobs.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.release(), + true, true, false, std::placeholders::_1)); + } + m_stats_tables.clear(); + return true; +} + +bool BackupImpl::wait_for_finish() { + /* Wait for threads to exit */ + return m_process_table_jobs.wait_for_finish(); +} + +bool BackupImpl::close_log_tables() { + bool result = wait_for_finish(); + for (auto &table_it : m_log_tables) + table_it.second->close(); + return result; +} + +Backup::Backup(const char *datadir_path, ds_ctxt_t *datasink, + std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool) : + m_backup_impl( + new BackupImpl(datadir_path, datasink, con_pool, + thread_pool)) { } + +Backup::~Backup() { + delete m_backup_impl; +} + +bool Backup::scan( + const std::unordered_set<table_key_t> &exclude_tables, + std::unordered_set<table_key_t> *out_processed_tables, + bool no_lock, bool collect_log_and_stats) { + return m_backup_impl->scan(exclude_tables, out_processed_tables, no_lock, + collect_log_and_stats); +} + +bool Backup::copy_log_tables(bool finalize) { + return m_backup_impl->copy_log_tables(finalize); +} + +bool Backup::copy_stats_tables() { + return m_backup_impl->copy_stats_tables(); +} + +bool Backup::wait_for_finish() { + return m_backup_impl->wait_for_finish(); +} + +bool Backup::close_log_tables() { + return m_backup_impl->close_log_tables(); +} + +void Backup::set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_backup_impl->set_post_copy_table_hook(hook); +} + +} // namespace common_engine diff --git a/extra/mariabackup/common_engine.h b/extra/mariabackup/common_engine.h new file mode 100644 index 00000000..6f5d8062 --- /dev/null +++ b/extra/mariabackup/common_engine.h @@ -0,0 +1,39 @@ +#pragma once +#include "my_global.h" +#include "backup_mysql.h" +#include "datasink.h" +#include "thread_pool.h" +#include "xtrabackup.h" + +#include <unordered_set> +#include <string> +#include <vector> + +namespace common_engine { + +class BackupImpl; + +class Backup { + public: + Backup(const char *datadir_path, ds_ctxt_t *datasink, + std::vector<MYSQL *> &con_pool, ThreadPool &thread_pool); + ~Backup(); + Backup (Backup &&other) = delete; + Backup & operator= (Backup &&other) = delete; + Backup(const Backup &) = delete; + Backup & operator= (const Backup &) = delete; + bool scan( + const std::unordered_set<table_key_t> &exclude_tables, + std::unordered_set<table_key_t> *out_processed_tables, + bool no_lock, bool collect_log_and_stats); + bool copy_log_tables(bool finalize); + bool copy_stats_tables(); + bool wait_for_finish(); + bool close_log_tables(); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook); + private: + BackupImpl *m_backup_impl; +}; + +} // namespace common_engine + diff --git a/extra/mariabackup/datasink.cc b/extra/mariabackup/datasink.cc index a576526d..132ff3fc 100644 --- a/extra/mariabackup/datasink.cc +++ b/extra/mariabackup/datasink.cc @@ -80,11 +80,11 @@ ds_create(const char *root, ds_type_t type) /************************************************************************ Open a datasink file */ ds_file_t * -ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat) +ds_open(ds_ctxt_t *ctxt, const char *path, const MY_STAT *stat, bool rewrite) { ds_file_t *file; - file = ctxt->datasink->open(ctxt, path, stat); + file = ctxt->datasink->open(ctxt, path, stat, rewrite); if (file != NULL) { file->datasink = ctxt->datasink; } @@ -104,6 +104,30 @@ ds_write(ds_file_t *file, const void *buf, size_t len) return file->datasink->write(file, (const uchar *)buf, len); } +int ds_seek_set(ds_file_t *file, my_off_t offset) { + DBUG_ASSERT(file); + DBUG_ASSERT(file->datasink); + if (file->datasink->seek_set) + return file->datasink->seek_set(file, offset); + return 0; +} + +int ds_rename(ds_ctxt_t *ctxt, const char *old_path, const char *new_path) { + DBUG_ASSERT(ctxt); + DBUG_ASSERT(ctxt->datasink); + if (ctxt->datasink->rename) + return ctxt->datasink->rename(ctxt, old_path, new_path); + return 0; +} + +int ds_remove(ds_ctxt_t *ctxt, const char *path) { + DBUG_ASSERT(ctxt); + DBUG_ASSERT(ctxt->datasink); + if (ctxt->datasink->remove) + return ctxt->datasink->mremove(ctxt, path); + return 0; +} + /************************************************************************ Close a datasink file. @return 0 on success, 1, on error. */ diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h index 57468e0c..98cbe525 100644 --- a/extra/mariabackup/datasink.h +++ b/extra/mariabackup/datasink.h @@ -43,7 +43,8 @@ typedef struct ds_ctxt { */ bool copy_file(const char *src_file_path, const char *dst_file_path, - uint thread_n); + uint thread_n, + bool rewrite = false); bool move_file(const char *src_file_path, const char *dst_file_path, @@ -76,10 +77,15 @@ typedef struct { struct datasink_struct { ds_ctxt_t *(*init)(const char *root); - ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); + ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *stat, bool rewrite); int (*write)(ds_file_t *file, const unsigned char *buf, size_t len); + int (*seek_set)(ds_file_t *file, my_off_t offset); int (*close)(ds_file_t *file); int (*remove)(const char *path); + // TODO: consider to return bool from "rename" and "remove" + int (*rename)(ds_ctxt_t *ctxt, const char *old_path, const char *new_path); + int (*mremove)(ds_ctxt_t *ctxt, const char *path); void (*deinit)(ds_ctxt_t *ctxt); }; @@ -106,12 +112,17 @@ ds_ctxt_t *ds_create(const char *root, ds_type_t type); /************************************************************************ Open a datasink file */ -ds_file_t *ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); +ds_file_t *ds_open( + ds_ctxt_t *ctxt, const char *path, const MY_STAT *stat, bool rewrite = false); /************************************************************************ Write to a datasink file. @return 0 on success, 1 on error. */ int ds_write(ds_file_t *file, const void *buf, size_t len); +int ds_seek_set(ds_file_t *file, my_off_t offset); + +int ds_rename(ds_ctxt_t *ctxt, const char *old_path, const char *new_path); +int ds_remove(ds_ctxt_t *ctxt, const char *path); /************************************************************************ Close a datasink file. diff --git a/extra/mariabackup/ddl_log.cc b/extra/mariabackup/ddl_log.cc new file mode 100644 index 00000000..6af34172 --- /dev/null +++ b/extra/mariabackup/ddl_log.cc @@ -0,0 +1,553 @@ +#include "ddl_log.h" +#include "common.h" +#include "my_sys.h" +#include "sql_table.h" +#include "backup_copy.h" +#include "xtrabackup.h" +#include <unordered_set> +#include <functional> +#include <memory> +#include <cstddef> + +namespace ddl_log { + +struct Entry { + enum Type { + CREATE, + ALTER, + RENAME, + REPAIR, + OPTIMIZE, + DROP, + TRUNCATE, + CHANGE_INDEX, + BULK_INSERT + }; + Type type; + std::string date; + std::string engine; + bool partitioned; + std::string db; + std::string table; + std::string id; + std::string new_engine; + bool new_partitioned; + std::string new_db; + std::string new_table; + std::string new_id; +}; + +typedef std::vector<std::unique_ptr<Entry>> entries_t; +typedef std::function<bool(std::unique_ptr<Entry>)> store_entry_func_t; + +const char *aria_engine_name = "Aria"; +static const char *frm_ext = ".frm"; +static const char *database_keyword = "DATABASE"; + +const std::unordered_map<std::string, std::vector<const char *>> engine_exts = +{ + {"Aria", {".MAD", ".MAI"}}, + {"MyISAM", {".MYD", ".MYI"}}, + {"MRG_MyISAM", {".MRG"}}, + {"ARCHIVE", {".ARM", ".ARZ"}}, + {"CSV", {".CSM", ".CSV"}} +}; + +static inline bool known_engine(const std::string &engine) { + return engine_exts.count(engine); +} + +// TODO: add error messages +size_t parse(const uchar *buf, size_t buf_size, bool &error_flag, + store_entry_func_t &store_entry_func) { + DBUG_ASSERT(buf); + static constexpr char token_delimiter = '\t'; + static constexpr char line_delimiter = '\n'; + enum { + TOKEN_FIRST = 0, + TOKEN_DATE = TOKEN_FIRST, + TOKEN_TYPE, + TOKEN_ENGINE, + TOKEN_PARTITIONED, + TOKEN_DB, + TOKEN_TABLE, + TOKEN_ID, + TOKEN_MANDATORY = TOKEN_ID, + TOKEN_NEW_ENGINE, + TOKEN_NEW_PARTITIONED, + TOKEN_NEW_DB, + TOKEN_NEW_TABLE, + TOKEN_NEW_ID, + TOKEN_LAST = TOKEN_NEW_ID + }; + const size_t string_offsets[TOKEN_LAST + 1] = { + offsetof(Entry, date), + offsetof(Entry, type), // not a string, be careful + offsetof(Entry, engine), + offsetof(Entry, partitioned), // not a string, be careful + offsetof(Entry, db), + offsetof(Entry, table), + offsetof(Entry, id), + offsetof(Entry, new_engine), + offsetof(Entry, new_partitioned), // not a string, be careful + offsetof(Entry, new_db), + offsetof(Entry, new_table), + offsetof(Entry, new_id) + }; + const std::unordered_map<std::string, Entry::Type> str_to_type = { + {"CREATE", Entry::CREATE}, + {"ALTER", Entry::ALTER}, + {"RENAME", Entry::RENAME}, + // TODO: fix to use uppercase-only + {"repair", Entry::REPAIR}, + {"optimize", Entry::OPTIMIZE}, + {"DROP", Entry::DROP}, + {"TRUNCATE", Entry::TRUNCATE}, + {"CHANGE_INDEX", Entry::CHANGE_INDEX}, + {"BULK_INSERT", Entry::BULK_INSERT} + }; + + const uchar *new_line = buf; + const uchar *token_start = buf; + unsigned token_num = TOKEN_FIRST; + + error_flag = false; + + std::unique_ptr<Entry> entry(new Entry()); + + for (const uchar *ptr = buf; ptr < buf + buf_size; ++ptr) { + + if (*ptr != token_delimiter && *ptr != line_delimiter) + continue; + + if (token_start != ptr) { + std::string token(token_start, ptr); + + if (token_num == TOKEN_TYPE) { + const auto type_it = str_to_type.find(token); + if (type_it == str_to_type.end()) { + error_flag = true; + goto exit; + } + entry->type = type_it->second; + } + else if (token_num == TOKEN_PARTITIONED) { + entry->partitioned = token[0] - '0'; + } + else if (token_num == TOKEN_NEW_PARTITIONED) { + entry->new_partitioned = token[0] - '0'; + } + else if (token_num <= TOKEN_LAST) { + DBUG_ASSERT(token_num != TOKEN_TYPE); + DBUG_ASSERT(token_num != TOKEN_PARTITIONED); + DBUG_ASSERT(token_num != TOKEN_NEW_PARTITIONED); + reinterpret_cast<std::string *> + (reinterpret_cast<uchar *>(entry.get()) + string_offsets[token_num])-> + assign(std::move(token)); + } + else { + error_flag = true; + goto exit; + } + } + token_start = ptr + 1; + + if (*ptr == line_delimiter) { + if (token_num < TOKEN_MANDATORY) { + error_flag = true; + goto exit; + } + if (!store_entry_func(std::move(entry))) { + error_flag = true; + goto exit; + } + entry.reset(new Entry()); + token_num = TOKEN_FIRST; + new_line = ptr + 1; + } else + ++token_num; + } + +exit: + return new_line - buf; +} + +bool parse(const char *file_path, store_entry_func_t store_entry_func) { + DBUG_ASSERT(file_path); + DBUG_ASSERT(store_entry_func); + File file= -1; + bool result = true; + uchar buf[1024]; + size_t bytes_read = 0; + size_t buf_read_offset = 0; + + if ((file= my_open(file_path, O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_WME))) < 0) { + msg("DDL log file %s open failed: %d", file_path, my_errno); + result = false; + goto exit; + } + + while((bytes_read = my_read( + file, &buf[buf_read_offset], sizeof(buf) - buf_read_offset, MY_WME)) > 0) { + if (bytes_read == size_t(-1)) { + msg("DDL log file %s read error: %d", file_path, my_errno); + result = false; + break; + } + bytes_read += buf_read_offset; + bool parse_error_flag = false; + size_t bytes_parsed = parse( + buf, bytes_read, parse_error_flag, store_entry_func); + if (parse_error_flag) { + result = false; + break; + } + size_t rest_size = bytes_read - bytes_parsed; + if (rest_size) + memcpy(buf, buf + bytes_parsed, rest_size); + buf_read_offset = rest_size; + } + +exit: + if (file >= 0) + my_close(file, MYF(MY_WME)); + return result; +}; + + +static +bool process_database( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry, + std::unordered_set<std::string> &dropped_databases) { + + if (entry.type == Entry::Type::CREATE || + entry.type == Entry::Type::ALTER) { + std::string opt_file(datadir_path); + opt_file.append("/").append(entry.db).append("/db.opt"); + if (!ds->copy_file(opt_file.c_str(), opt_file.c_str(), 0, true)) { + msg("Failed to re-copy %s.", opt_file.c_str()); + return false; + } + if (entry.type == Entry::Type::CREATE) + dropped_databases.erase(entry.db); + return true; + } + + DBUG_ASSERT(entry.type == Entry::Type::DROP); + + std::string db_path(datadir_path); + db_path.append("/").append(entry.db); + const char *dst_path = convert_dst(db_path.c_str()); + if (!ds_remove(ds, dst_path)) { + dropped_databases.insert(entry.db); + return true; + } + return false; +} + +static +std::unique_ptr<std::vector<std::string>> + find_table_files( + const char *dir_path, + const std::string &db, + const std::string &table) { + + std::unique_ptr<std::vector<std::string>> + result(new std::vector<std::string>()); + + std::string prefix = convert_tablename_to_filepath(dir_path, db, table); + foreach_file_in_db_dirs(dir_path, [&](const char *file_name)->bool { + if (!strncmp(file_name, prefix.c_str(), prefix.size())) { + DBUG_ASSERT(strlen(file_name) >= prefix.size()); + if (file_name[prefix.size()] == '.' || + !strncmp(file_name + prefix.size(), "#P#", strlen("#P#"))) + result->push_back(std::string(file_name)); + } + return true; + }); + + return result; +} + +static +bool process_remove( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry, + bool remove_frm) { + + if (check_if_skip_table( + std::string(entry.db).append("/").append(entry.table).c_str())) + return true; + + auto ext_it = engine_exts.find(entry.engine); + if (ext_it == engine_exts.end()) + return true; + + std::string file_preffix = convert_tablename_to_filepath(datadir_path, + entry.db, entry.table); + const char *dst_preffix = convert_dst(file_preffix.c_str()); + + for (const char *ext : ext_it->second) { + std::string old_name(dst_preffix); + if (!entry.partitioned) + old_name.append(ext); + else + old_name.append("#P#*"); + if (ds_remove(ds, old_name.c_str())) { + msg("Failed to remove %s.", old_name.c_str()); + return false; + } + } + + if (remove_frm) { + std::string old_frm_name(dst_preffix); + old_frm_name.append(frm_ext); + if (ds_remove(ds, old_frm_name.c_str())) { + msg("Failed to remove %s.", old_frm_name.c_str()); + return false; + } + } + return true; + +} + +static +bool process_recopy( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry, + const tables_t &tables) { + + if (check_if_skip_table( + std::string(entry.db).append("/").append(entry.table).c_str())) + return true; + + const std::string &new_table_id = + entry.new_id.empty() ? entry.id : entry.new_id; + DBUG_ASSERT(!new_table_id.empty()); + const std::string &new_table = + entry.new_table.empty() ? entry.table : entry.new_table; + DBUG_ASSERT(!new_table.empty()); + const std::string &new_db = + entry.new_db.empty() ? entry.db : entry.new_db; + DBUG_ASSERT(!new_db.empty()); + const std::string &new_engine = + entry.new_engine.empty() ? entry.engine : entry.new_engine; + DBUG_ASSERT(!new_engine.empty()); + + if (entry.type != Entry::Type::BULK_INSERT) { + auto table_it = tables.find(table_key(new_db, new_table)); + if (table_it != tables.end() && + table_it->second == new_table_id) + return true; + } + + if (!entry.new_engine.empty() && + entry.engine != entry.new_engine && + !known_engine(entry.new_engine)) { + return process_remove(datadir_path, ds, entry, false); + } + + if ((entry.partitioned || entry.new_partitioned) && + !process_remove(datadir_path, ds, entry, false)) + return false; + + if (entry.partitioned || entry.new_partitioned) { + auto files = find_table_files(datadir_path, new_db, new_table); + if (!files.get()) + return true; + for (const auto &file : *files) { + const char *dst_path = convert_dst(file.c_str()); + if (!ds->copy_file(file.c_str(), dst_path, 0, true)) { + msg("Failed to re-copy %s.", file.c_str()); + return false; + } + } + return true; + } + + auto ext_it = engine_exts.find(new_engine); + if (ext_it == engine_exts.end()) + return false; + + for (const char *ext : ext_it->second) { + std::string file_name = + convert_tablename_to_filepath(datadir_path, new_db, new_table). + append(ext); + const char *dst_path = convert_dst(file_name.c_str()); + if (file_exists(file_name.c_str()) && + !ds->copy_file(file_name.c_str(), dst_path, 0, true)) { + msg("Failed to re-copy %s.", file_name.c_str()); + return false; + } + } + + std::string frm_file = + convert_tablename_to_filepath(datadir_path, new_db, new_table). + append(frm_ext); + const char *frm_dst_path = convert_dst(frm_file.c_str()); + if (file_exists(frm_file.c_str()) && + !ds->copy_file(frm_file.c_str(), frm_dst_path, 0, true)) { + msg("Failed to re-copy %s.", frm_file.c_str()); + return false; + } + + return true; +} + +static +bool process_rename( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry) { + + if (check_if_skip_table( + std::string(entry.db).append("/").append(entry.table).c_str())) + return true; + + DBUG_ASSERT(entry.db != "partition"); + + auto ext_it = engine_exts.find(entry.engine); + if (ext_it == engine_exts.end()) + return false; + + std::string new_preffix = convert_tablename_to_filepath(datadir_path, + entry.new_db, entry.new_table); + const char *dst_path = convert_dst(new_preffix.c_str()); + + std::string old_preffix = convert_tablename_to_filepath(datadir_path, + entry.db, entry.table); + const char *src_path = convert_dst(old_preffix.c_str()); + + for (const char *ext : ext_it->second) { + std::string old_name(src_path); + old_name.append(ext); + std::string new_name(dst_path); + new_name.append(ext); + if (ds_rename(ds, old_name.c_str(), new_name.c_str())) { + msg("Failed to rename %s to %s.", + old_name.c_str(), new_name.c_str()); + return false; + } + } + + std::string new_frm_file = new_preffix + frm_ext; + const char *new_frm_dst = convert_dst(new_frm_file.c_str()); + if (file_exists(new_frm_file.c_str()) && + !ds->copy_file(new_frm_file.c_str(), new_frm_dst, 0, true)) { + msg("Failed to re-copy %s.", new_frm_file.c_str()); + return false; + } + +// TODO: return this code if .frm is copied not under BLOCK_DDL +/* + std::string old_frm_name(src_path); + old_frm_name.append(frm_ext); + std::string new_frm_name(dst_path); + new_frm_name.append(frm_ext); + if (ds_rename(ds, old_frm_name.c_str(), new_frm_name.c_str())) { + msg("Failed to rename %s to %s.", + old_frm_name.c_str(), new_frm_name.c_str()); + return false; + } +*/ + return true; +} + +bool backup( + const char *datadir_path, + ds_ctxt_t *ds, + const tables_t &tables) { + DBUG_ASSERT(datadir_path); + DBUG_ASSERT(ds); + char ddl_log_path[FN_REFLEN]; + fn_format(ddl_log_path, "ddl", datadir_path, ".log", 0); + std::vector<std::unique_ptr<Entry>> entries; + + std::unordered_set<std::string> processed_tables; + std::unordered_set<std::string> dropped_databases; + + bool parsing_result = + parse(ddl_log_path, [&](std::unique_ptr<Entry> entry)->bool { + + if (entry->engine == database_keyword) + return process_database(datadir_path, ds, *entry, dropped_databases); + + if (!known_engine(entry->engine) && !known_engine(entry->new_engine)) + return true; + + if (entry->type == Entry::Type::CREATE || + (entry->type == Entry::Type::ALTER && + !entry->new_engine.empty() && + entry->engine != entry->new_engine)) { + if (!process_recopy(datadir_path, ds, *entry, tables)) + return false; + processed_tables.insert(table_key(entry->db, entry->table)); + if (entry->type == Entry::Type::ALTER) + processed_tables.insert(table_key(entry->new_db, entry->new_table)); + return true; + } + + if (entry->type == Entry::Type::DROP) { + if (!process_remove(datadir_path, ds, *entry, true)) + return false; + processed_tables.insert(table_key(entry->db, entry->table)); + return true; + } + if (entry->type == Entry::Type::RENAME) { + if (entry->partitioned) { + if (!process_remove(datadir_path, ds, *entry, true)) + return false; + Entry recopy_entry { + entry->type, + {}, + entry->new_engine.empty() ? entry->engine : entry->new_engine, + true, + entry->new_db, + entry->new_table, + entry->new_id, + {}, true, {}, {}, {} + }; + if (!process_recopy(datadir_path, ds, recopy_entry, tables)) + return false; + } + else if (!process_rename(datadir_path, ds, *entry)) + return false; + processed_tables.insert(table_key(entry->db, entry->table)); + processed_tables.insert(table_key(entry->new_db, entry->new_table)); + return true; + } + + entries.push_back(std::move(entry)); + return true; + + }); + + if (!parsing_result) + return false; + + + while (!entries.empty()) { + auto entry = std::move(entries.back()); + entries.pop_back(); + auto tk = table_key( + entry->new_db.empty() ? entry->db : entry->new_db, + entry->new_table.empty() ? entry->table : entry->new_table); + if (dropped_databases.count(entry->db) || + dropped_databases.count(entry->new_db)) + continue; + if (processed_tables.count(tk)) + continue; + processed_tables.insert(std::move(tk)); + if (!process_recopy(datadir_path, ds, *entry, tables)) + return false; + } + + return true; +} + +} // namespace ddl_log diff --git a/extra/mariabackup/ddl_log.h b/extra/mariabackup/ddl_log.h new file mode 100644 index 00000000..5cac3e5d --- /dev/null +++ b/extra/mariabackup/ddl_log.h @@ -0,0 +1,15 @@ +#pragma once +#include "my_global.h" +#include "datasink.h" +#include "aria_backup_client.h" +#include <string> +#include <memory> +#include <vector> +#include <unordered_map> + +namespace ddl_log { + +typedef std::unordered_map<std::string, std::string> tables_t; +bool backup(const char *datadir_path, ds_ctxt_t *ds, const tables_t &tables); + +} // namespace ddl_log diff --git a/extra/mariabackup/ds_buffer.cc b/extra/mariabackup/ds_buffer.cc index d6a42095..bc1d4663 100644 --- a/extra/mariabackup/ds_buffer.cc +++ b/extra/mariabackup/ds_buffer.cc @@ -44,7 +44,7 @@ typedef struct { static ds_ctxt_t *buffer_init(const char *root); static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int buffer_write(ds_file_t *file, const uchar *buf, size_t len); static int buffer_close(ds_file_t *file); static void buffer_deinit(ds_ctxt_t *ctxt); @@ -53,8 +53,11 @@ datasink_t datasink_buffer = { &buffer_init, &buffer_open, &buffer_write, + nullptr, &buffer_close, &dummy_remove, + nullptr, + nullptr, &buffer_deinit }; @@ -84,8 +87,10 @@ buffer_init(const char *root) } static ds_file_t * -buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +buffer_open(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *mystat, bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_buffer_ctxt_t *buffer_ctxt; ds_ctxt_t *pipe_ctxt; ds_file_t *dst_file; diff --git a/extra/mariabackup/ds_compress.cc b/extra/mariabackup/ds_compress.cc index f7a9b7a1..0cb52e97 100644 --- a/extra/mariabackup/ds_compress.cc +++ b/extra/mariabackup/ds_compress.cc @@ -65,7 +65,7 @@ extern ulonglong xtrabackup_compress_chunk_size; static ds_ctxt_t *compress_init(const char *root); static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int compress_write(ds_file_t *file, const uchar *buf, size_t len); static int compress_close(ds_file_t *file); static void compress_deinit(ds_ctxt_t *ctxt); @@ -74,8 +74,11 @@ datasink_t datasink_compress = { &compress_init, &compress_open, &compress_write, + nullptr, &compress_close, &dummy_remove, + nullptr, + nullptr, &compress_deinit }; @@ -116,8 +119,10 @@ compress_init(const char *root) static ds_file_t * -compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +compress_open(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *mystat, bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_compress_ctxt_t *comp_ctxt; ds_ctxt_t *dest_ctxt; ds_file_t *dest_file; diff --git a/extra/mariabackup/ds_local.cc b/extra/mariabackup/ds_local.cc index f86612b9..ff2021fc 100644 --- a/extra/mariabackup/ds_local.cc +++ b/extra/mariabackup/ds_local.cc @@ -42,8 +42,9 @@ typedef struct { static ds_ctxt_t *local_init(const char *root); static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int local_write(ds_file_t *file, const uchar *buf, size_t len); +static int local_seek_set(ds_file_t *file, my_off_t offset); static int local_close(ds_file_t *file); static void local_deinit(ds_ctxt_t *ctxt); @@ -52,13 +53,20 @@ static int local_remove(const char *path) return unlink(path); } +static int local_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path); +static int local_mremove(ds_ctxt_t *ctxt, const char *path); + extern "C" { datasink_t datasink_local = { &local_init, &local_open, &local_write, + &local_seek_set, &local_close, &local_remove, + &local_rename, + &local_mremove, &local_deinit }; } @@ -89,7 +97,7 @@ local_init(const char *root) static ds_file_t * local_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat __attribute__((unused))) + const MY_STAT *mystat __attribute__((unused)), bool rewrite) { char fullpath[FN_REFLEN]; char dirpath[FN_REFLEN]; @@ -111,8 +119,10 @@ local_open(ds_ctxt_t *ctxt, const char *path, return NULL; } - fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW, - MYF(MY_WME)); + // TODO: check in Windows and set the corresponding flags on fail + fd = my_create(fullpath, 0, + O_WRONLY | O_BINARY | (rewrite ? O_TRUNC : O_EXCL) | O_NOFOLLOW, + MYF(MY_WME)); if (fd < 0) { return NULL; } @@ -194,8 +204,8 @@ static void init_ibd_data(ds_local_file_t *local_file, const uchar *buf, size_t return; } - auto flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]); - auto ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + uint32_t flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]); + uint32_t ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); local_file->pagesize= ssize == 0 ? UNIV_PAGE_SIZE_ORIG : ((UNIV_ZIP_SIZE_MIN >> 1) << ssize); local_file->compressed = fil_space_t::full_crc32(flags) ? fil_space_t::is_compressed(flags) @@ -239,6 +249,15 @@ local_write(ds_file_t *file, const uchar *buf, size_t len) return 1; } +static +int +local_seek_set(ds_file_t *file, my_off_t offset) { + ds_local_file_t *local_file= (ds_local_file_t *)file->ptr; + if (my_seek(local_file->fd, offset, SEEK_SET, MYF(0)) == MY_FILEPOS_ERROR) + return 1; + return 0; +} + /* Set EOF at file's current position.*/ static int set_eof(File fd) { @@ -276,3 +295,77 @@ local_deinit(ds_ctxt_t *ctxt) my_free(ctxt->root); my_free(ctxt); } + + +static int local_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path) { + char full_old_path[FN_REFLEN]; + char full_new_path[FN_REFLEN]; + fn_format(full_old_path, old_path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + fn_format(full_new_path, new_path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + // Ignore errors as .frm files can me copied separately. + // TODO: return error processing here after the corresponding changes in + // xtrabackup.cc + (void)my_rename(full_old_path, full_new_path, MYF(0)); +// if (my_rename(full_old_path, full_new_path, MYF(0))) { +// msg("Failed to rename file %s to %s", old_path, new_path); +// return 1; +// } + return 0; +} + +// It's ok if destination does not contain the file or folder +static int local_mremove(ds_ctxt_t *ctxt, const char *path) { + char full_path[FN_REFLEN]; + fn_format(full_path, path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + size_t full_path_len = strlen(full_path); + if (full_path[full_path_len - 1] == '*') { + full_path[full_path_len - 1] = '\0'; + char *preffix = strrchr(full_path, '/'); + const char *full_path_dir = full_path; + size_t preffix_len; + if (preffix) { + preffix_len = (full_path_len - 1) - (preffix - full_path); + *(preffix++) = '\0'; + } + else { + preffix = full_path; + preffix_len = full_path_len - 1; + full_path_dir= IF_WIN(".\\", "./"); + } + if (!preffix_len) + return 0; + MY_DIR *dir= my_dir(full_path_dir, 0); + if (!dir) + return 0; + for (size_t i = 0; i < dir->number_of_files; ++i) { + char full_fpath[FN_REFLEN]; + if (strncmp(dir->dir_entry[i].name, preffix, preffix_len)) + continue; + fn_format(full_fpath, dir->dir_entry[i].name, + full_path_dir, "", MYF(MY_RELATIVE_PATH)); + (void)my_delete(full_fpath, MYF(0)); + } + my_dirend(dir); + } + else { + MY_STAT stat; + if (!my_stat(full_path, &stat, MYF(0))) + return 0; + MY_DIR *dir= my_dir(full_path, 0); + if (!dir) { + // TODO: check for error here if necessary + (void)my_delete(full_path, MYF(0)); + return 0; + } + for (size_t i = 0; i < dir->number_of_files; ++i) { + char full_fpath[FN_REFLEN]; + fn_format(full_fpath, dir->dir_entry[i].name, + full_path, "", MYF(MY_RELATIVE_PATH)); + (void)my_delete(full_fpath, MYF(0)); + } + my_dirend(dir); + (void)my_rmtree(full_path, MYF(0)); + } + return 0; +} diff --git a/extra/mariabackup/ds_stdout.cc b/extra/mariabackup/ds_stdout.cc index a9639ff7..3fc0873b 100644 --- a/extra/mariabackup/ds_stdout.cc +++ b/extra/mariabackup/ds_stdout.cc @@ -30,7 +30,7 @@ typedef struct { static ds_ctxt_t *stdout_init(const char *root); static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int stdout_write(ds_file_t *file, const uchar *buf, size_t len); static int stdout_close(ds_file_t *file); static void stdout_deinit(ds_ctxt_t *ctxt); @@ -39,8 +39,11 @@ datasink_t datasink_stdout = { &stdout_init, &stdout_open, &stdout_write, + nullptr, &stdout_close, &dummy_remove, + nullptr, + nullptr, &stdout_deinit }; @@ -61,8 +64,9 @@ static ds_file_t * stdout_open(ds_ctxt_t *ctxt __attribute__((unused)), const char *path __attribute__((unused)), - MY_STAT *mystat __attribute__((unused))) + const MY_STAT *mystat __attribute__((unused)), bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_stdout_file_t *stdout_file; ds_file_t *file; size_t pathlen; diff --git a/extra/mariabackup/ds_tmpfile.cc b/extra/mariabackup/ds_tmpfile.cc index 80b9d3bb..6bafee25 100644 --- a/extra/mariabackup/ds_tmpfile.cc +++ b/extra/mariabackup/ds_tmpfile.cc @@ -41,7 +41,7 @@ typedef struct { static ds_ctxt_t *tmpfile_init(const char *root); static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int tmpfile_write(ds_file_t *file, const uchar *buf, size_t len); static int tmpfile_close(ds_file_t *file); static void tmpfile_deinit(ds_ctxt_t *ctxt); @@ -50,8 +50,11 @@ datasink_t datasink_tmpfile = { &tmpfile_init, &tmpfile_open, &tmpfile_write, + nullptr, &tmpfile_close, &dummy_remove, + nullptr, + nullptr, &tmpfile_deinit }; @@ -80,8 +83,9 @@ tmpfile_init(const char *root) static ds_file_t * tmpfile_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat) + const MY_STAT *mystat, bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_tmpfile_ctxt_t *tmpfile_ctxt; char tmp_path[FN_REFLEN]; ds_tmp_file_t *tmp_file; diff --git a/extra/mariabackup/ds_xbstream.cc b/extra/mariabackup/ds_xbstream.cc index 3bf8bd08..96e0cf7a 100644 --- a/extra/mariabackup/ds_xbstream.cc +++ b/extra/mariabackup/ds_xbstream.cc @@ -40,24 +40,31 @@ General streaming interface */ static ds_ctxt_t *xbstream_init(const char *root); static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int xbstream_write(ds_file_t *file, const uchar *buf, size_t len); +static int xbstream_seek_set(ds_file_t *file, my_off_t offset); static int xbstream_close(ds_file_t *file); static void xbstream_deinit(ds_ctxt_t *ctxt); +static int xbstream_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path); +static int xbstream_mremove(ds_ctxt_t *ctxt, const char *path); + datasink_t datasink_xbstream = { &xbstream_init, &xbstream_open, &xbstream_write, + &xbstream_seek_set, &xbstream_close, &dummy_remove, + &xbstream_rename, + &xbstream_mremove, &xbstream_deinit }; static ssize_t -my_xbstream_write_callback(xb_wstream_file_t *f __attribute__((unused)), - void *userdata, const void *buf, size_t len) +my_xbstream_write_callback(void *userdata, const void *buf, size_t len) { ds_stream_ctxt_t *stream_ctxt; @@ -89,7 +96,7 @@ xbstream_init(const char *root __attribute__((unused))) goto err; } - xbstream = xb_stream_write_new(); + xbstream = xb_stream_write_new(my_xbstream_write_callback, stream_ctxt); if (xbstream == NULL) { msg("xb_stream_write_new() failed."); goto err; @@ -108,7 +115,8 @@ err: static ds_file_t * -xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +xbstream_open(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *mystat, bool rewrite) { ds_file_t *file; ds_stream_file_t *stream_file; @@ -144,9 +152,7 @@ xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) xbstream = stream_ctxt->xbstream; - xbstream_file = xb_stream_write_open(xbstream, path, mystat, - stream_ctxt, - my_xbstream_write_callback); + xbstream_file = xb_stream_write_open(xbstream, path, mystat, rewrite); if (xbstream_file == NULL) { msg("xb_stream_write_open() failed."); @@ -192,6 +198,45 @@ xbstream_write(ds_file_t *file, const uchar *buf, size_t len) static int +xbstream_seek_set(ds_file_t *file, my_off_t offset) +{ + ds_stream_file_t *stream_file; + xb_wstream_file_t *xbstream_file; + + + stream_file = (ds_stream_file_t *) file->ptr; + + xbstream_file = stream_file->xbstream_file; + + if (xb_stream_write_seek_set(xbstream_file, offset)) { + msg("xb_stream_write_seek_set() failed."); + return 1; + } + + return 0; +} + +static +int +xbstream_mremove(ds_ctxt_t *ctxt, const char *path) { + ds_stream_ctxt_t *stream_ctxt = + reinterpret_cast<ds_stream_ctxt_t *>(ctxt->ptr); + xb_wstream_t *xbstream = stream_ctxt->xbstream; + return xb_stream_write_remove(xbstream, path); +} + +static +int +xbstream_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path) { + ds_stream_ctxt_t *stream_ctxt = + reinterpret_cast<ds_stream_ctxt_t *>(ctxt->ptr); + xb_wstream_t *xbstream = stream_ctxt->xbstream; + return xb_stream_write_rename(xbstream, old_path, new_path); +} + +static +int xbstream_close(ds_file_t *file) { ds_stream_file_t *stream_file; diff --git a/extra/mariabackup/xb_plugin.cc b/extra/mariabackup/encryption_plugin.cc index 7470d376..ab0c5140 100644 --- a/extra/mariabackup/xb_plugin.cc +++ b/extra/mariabackup/encryption_plugin.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2017, 2022, MariaDB Corporation. +/* Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -17,18 +17,18 @@ #include <mysqld.h> #include <mysql.h> #include <xtrabackup.h> -#include <xb_plugin.h> +#include <encryption_plugin.h> #include <sql_plugin.h> #include <sstream> #include <vector> #include <common.h> #include <backup_mysql.h> -#include <srv0srv.h> +#include <log0crypt.h> extern struct st_maria_plugin *mysql_optional_plugins[]; extern struct st_maria_plugin *mysql_mandatory_plugins[]; -static void xb_plugin_init(int argc, char **argv); +static void encryption_plugin_init(int argc, char **argv); extern char *xb_plugin_load; extern char *xb_plugin_dir; @@ -42,7 +42,7 @@ const char *QUERY_PLUGIN = " OR (plugin_type = 'DAEMON' AND plugin_name LIKE 'provider\\_%')" " AND plugin_status='ACTIVE'"; -std::string xb_plugin_config; +std::string encryption_plugin_config; static void add_to_plugin_load_list(const char *plugin_def) { @@ -52,16 +52,16 @@ static void add_to_plugin_load_list(const char *plugin_def) static char XTRABACKUP_EXE[] = "xtrabackup"; /* - Read "plugin-load" value from backup-my.cnf during prepare phase. + Read "plugin-load" value (encryption plugin) from backup-my.cnf during + prepare phase. The value is stored during backup phase. */ -static std::string get_plugin_from_cnf(const char *dir) +static std::string get_encryption_plugin_from_cnf() { - std::string path = dir + std::string("/backup-my.cnf"); - FILE *f = fopen(path.c_str(), "r"); + FILE *f = fopen("backup-my.cnf", "r"); if (!f) { - die("Can't open %s for reading", path.c_str()); + die("Can't open backup-my.cnf for reading"); } char line[512]; std::string plugin_load; @@ -72,7 +72,16 @@ static std::string get_plugin_from_cnf(const char *dir) plugin_load = line + 12; // remote \n at the end of string plugin_load.resize(plugin_load.size() - 1); - break; + } + + if (strncmp(line, "innodb_encrypt_tables=", 22) == 0) + { + if (!strncmp(line + 22, "ON", 2) || + !strncmp(line + 22, "1", 1)) + srv_encrypt_tables= 1; + else if (!strncmp(line + 22, "FORCE", 5) || + !strncmp(line + 22, "2", 1)) + srv_encrypt_tables= 2; } } fclose(f); @@ -80,7 +89,7 @@ static std::string get_plugin_from_cnf(const char *dir) } -void xb_plugin_backup_init(MYSQL *mysql) +void encryption_plugin_backup_init(MYSQL *mysql) { MYSQL_RES *result; MYSQL_ROW row; @@ -163,7 +172,18 @@ void xb_plugin_backup_init(MYSQL *mysql) mysql_free_result(result); } - xb_plugin_config = oss.str(); + result = xb_mysql_query(mysql, "select @@innodb_encrypt_tables", true, true); + row = mysql_fetch_row(result); + if (!row); + else if (const char *r= row[0]) + { + if (!strcmp(r, "ON")) srv_encrypt_tables= 1; + else if (!strcmp(r, "FORCE")) srv_encrypt_tables= 2; + oss << "innodb_encrypt_tables=" << r << std::endl; + } + + mysql_free_result(result); + encryption_plugin_config = oss.str(); argc = 0; argv[argc++] = XTRABACKUP_EXE; @@ -175,23 +195,23 @@ void xb_plugin_backup_init(MYSQL *mysql) } argv[argc] = 0; - xb_plugin_init(argc, argv); + encryption_plugin_init(argc, argv); } -const char *xb_plugin_get_config() +const char *encryption_plugin_get_config() { - return xb_plugin_config.c_str(); + return encryption_plugin_config.c_str(); } extern int finalize_encryption_plugin(st_plugin_int *plugin); -void xb_plugin_prepare_init(int argc, char **argv, const char *dir) +void encryption_plugin_prepare_init(int argc, char **argv) { - std::string plugin_load= get_plugin_from_cnf(dir ? dir : "."); + std::string plugin_load= get_encryption_plugin_from_cnf(); if (plugin_load.size()) { - msg("Loading plugins from %s", plugin_load.c_str()); + msg("Loading encryption plugin from %s", plugin_load.c_str()); } else { @@ -211,19 +231,19 @@ void xb_plugin_prepare_init(int argc, char **argv, const char *dir) new_argv[0] = XTRABACKUP_EXE; memcpy(&new_argv[1], argv, argc*sizeof(char *)); - xb_plugin_init(argc+1, new_argv); + encryption_plugin_init(argc+1, new_argv); delete[] new_argv; } -static void xb_plugin_init(int argc, char **argv) +static void encryption_plugin_init(int argc, char **argv) { /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */ mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0; plugin_maturity = MariaDB_PLUGIN_MATURITY_UNKNOWN; /* mariabackup accepts all plugins */ - msg("Loading plugins"); + msg("Loading encryption plugin"); for (int i= 1; i < argc; i++) - msg("\t Plugin parameter : '%s'", argv[i]); + msg("\t Encryption plugin parameter : '%s'", argv[i]); plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE); } diff --git a/extra/mariabackup/encryption_plugin.h b/extra/mariabackup/encryption_plugin.h new file mode 100644 index 00000000..16d74790 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.h @@ -0,0 +1,7 @@ +#include <mysql.h> +#include <string> +extern void encryption_plugin_backup_init(MYSQL *mysql); +extern const char* encryption_plugin_get_config(); +extern void encryption_plugin_prepare_init(int argc, char **argv); + +//extern void encryption_plugin_init(int argc, char **argv); diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index 4f5d67a5..be871e4a 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -199,14 +199,6 @@ xb_fil_cur_open( return(XB_FIL_CUR_SKIP); } -#ifdef HAVE_FCNTL_DIRECT - if (srv_file_flush_method == SRV_O_DIRECT - || srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) { - - os_file_set_nocache(cursor->file, node->name, "OPEN"); - } -#endif - posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); cursor->page_size = node->space->physical_size(); @@ -239,12 +231,14 @@ xb_fil_cur_open( / cursor->page_size); cursor->read_filter = read_filter; - cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, - node->space->id); + cursor->read_filter->init(&cursor->read_filter_ctxt, cursor); return(XB_FIL_CUR_SUCCESS); } +/* Stack usage 131224 with clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + static bool page_is_corrupted(const byte *page, ulint page_no, const xb_fil_cur_t *cursor, const fil_space_t *space) @@ -348,6 +342,7 @@ static bool page_is_corrupted(const byte *page, ulint page_no, return buf_page_is_corrupted(true, page, space->flags); } +PRAGMA_REENABLE_CHECK_STACK_FRAME /** Reads and verifies the next block of pages from the source file. Positions the cursor after the last read non-corrupted page. @@ -510,10 +505,6 @@ xb_fil_cur_close( /*=============*/ xb_fil_cur_t *cursor) /*!< in/out: source file cursor */ { - if (cursor->read_filter) { - cursor->read_filter->deinit(&cursor->read_filter_ctxt); - } - aligned_free(cursor->buf); cursor->buf = NULL; diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h index b7812f65..46c8cb03 100644 --- a/extra/mariabackup/fil_cur.h +++ b/extra/mariabackup/fil_cur.h @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include <my_dir.h> #include "read_filt.h" +#include "mtr0types.h" #include "srv0start.h" #include "srv0srv.h" #include "xtrabackup.h" diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index b925b415..2de57a14 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -78,10 +78,8 @@ my_bool opt_ibx_galera_info = FALSE; my_bool opt_ibx_slave_info = FALSE; my_bool opt_ibx_no_lock = FALSE; my_bool opt_ibx_safe_slave_backup = FALSE; -my_bool opt_ibx_rsync = FALSE; my_bool opt_ibx_force_non_empty_dirs = FALSE; my_bool opt_ibx_noversioncheck = FALSE; -my_bool opt_ibx_no_backup_locks = FALSE; my_bool opt_ibx_decompress = FALSE; char *opt_ibx_incremental_history_name = NULL; @@ -268,8 +266,10 @@ static struct my_option ibx_long_options[] = (uchar *) &opt_ibx_incremental, (uchar *) &opt_ibx_incremental, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock " - "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + {"no-lock", OPT_NO_LOCK, "This option should not be used as " + "mariadb-backup now is using BACKUP LOCKS, which minimizes the " + "lock time. ALTER TABLE can run in parallel with BACKUP LOCKS." + "Use the --no-lock option it only if ALL your " "tables are InnoDB and you DO NOT CARE about the binary log " "position of the backup. This option shouldn't be used if there " "are any DDL statements being executed or if any updates are " @@ -297,15 +297,6 @@ static struct my_option ibx_long_options[] = (uchar *) &opt_ibx_safe_slave_backup, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file " - "transfers. When this option is specified, innobackupex uses rsync " - "to copy all non-InnoDB files instead of spawning a separate cp for " - "each file, which can be much faster for servers with a large number " - "of databases or tables. This option cannot be used together with " - "--stream.", - (uchar *) &opt_ibx_rsync, (uchar *) &opt_ibx_rsync, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " "option, when specified, makes --copy-back or --move-back transfer " "files to non-empty directories. Note that no existing files will be " @@ -330,13 +321,9 @@ static struct my_option ibx_long_options[] = (uchar *) &opt_ibx_noversioncheck, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if " - "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " - "on the backup stage. The option has no effect when backup locks are " - "not supported by the server. This option is enabled by default, " - "disable with --no-backup-locks.", - (uchar *) &opt_ibx_no_backup_locks, - (uchar *) &opt_ibx_no_backup_locks, + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, + "Old disabled option which has no effect anymore.", + (uchar *) 0, (uchar*) 0, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " @@ -402,11 +389,10 @@ static struct my_option ibx_long_options[] = REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, - "This option specifies which types of queries are allowed to complete " - "before innobackupex will issue the global lock. Default is all.", - (uchar*) &opt_ibx_lock_wait_query_type, - (uchar*) &opt_ibx_lock_wait_query_type, &query_type_typelib, - GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, &query_type_typelib, GET_ENUM, + REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, "This option specifies which types of queries should be killed to " @@ -447,32 +433,32 @@ static struct my_option ibx_long_options[] = REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, - "This option specifies the number of seconds innobackupex waits " - "between starting FLUSH TABLES WITH READ LOCK and killing those " - "queries that block it. Default is 0 seconds, which means " - "innobackupex will not attempt to kill any queries.", - (uchar*) &opt_ibx_kill_long_queries_timeout, - (uchar*) &opt_ibx_kill_long_queries_timeout, 0, GET_UINT, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, - "This option specifies time in seconds that innobackupex should wait " - "for queries that would block FTWRL before running it. If there are " - "still such queries when the timeout expires, innobackupex terminates " - "with an error. Default is 0, in which case innobackupex does not " - "wait for queries to complete and starts FTWRL immediately.", - (uchar*) &opt_ibx_lock_wait_timeout, - (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT, + "Alias for startup-wait-timeout", + (uchar*) &opt_ibx_lock_wait_timeout, + (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"startup-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that mariadb-backup should wait for " + "BACKUP STAGE START to complete. BACKUP STAGE START has to wait until all " + "currently running queries using explicite LOCK TABLES has ended. " + "If there are still such queries when the timeout expires, mariadb-backup " + "terminates with an error. Default is 0, in which case mariadb-backup waits " + "indefinitely for BACKUP STAGE START to finish", + (uchar*) &opt_ibx_lock_wait_timeout, + (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, - "This option specifies the query run time threshold which is used by " - "innobackupex to detect long-running queries with a non-zero value " - "of --ftwrl-wait-timeout. FTWRL is not started until such " - "long-running queries exist. This option has no effect if " - "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", - (uchar*) &opt_ibx_lock_wait_threshold, - (uchar*) &opt_ibx_lock_wait_threshold, 0, GET_UINT, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, 0, GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, @@ -864,10 +850,8 @@ ibx_init() opt_slave_info = opt_ibx_slave_info; opt_no_lock = opt_ibx_no_lock; opt_safe_slave_backup = opt_ibx_safe_slave_backup; - opt_rsync = opt_ibx_rsync; opt_force_non_empty_dirs = opt_ibx_force_non_empty_dirs; opt_noversioncheck = opt_ibx_noversioncheck; - opt_no_backup_locks = opt_ibx_no_backup_locks; opt_decompress = opt_ibx_decompress; opt_incremental_history_name = opt_ibx_incremental_history_name; diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc index 58920055..c7c0aa55 100644 --- a/extra/mariabackup/read_filt.cc +++ b/extra/mariabackup/read_filt.cc @@ -32,29 +32,13 @@ Perform read filter context initialization that is common to all read filters. */ static void -common_init( -/*========*/ +rf_pass_through_init( xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter context */ const xb_fil_cur_t* cursor) /*!<in: file cursor */ { ctxt->offset = 0; ctxt->data_file_size = cursor->statinfo.st_size; ctxt->buffer_capacity = cursor->buf_size; - ctxt->page_size = cursor->page_size; -} - -/****************************************************************//** -Initialize the pass-through read filter. */ -static -void -rf_pass_through_init( -/*=================*/ - xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter context */ - const xb_fil_cur_t* cursor, /*!<in: file cursor */ - ulint space_id __attribute__((unused))) - /*!<in: space id we are reading */ -{ - common_init(ctxt, cursor); } /****************************************************************//** @@ -65,143 +49,25 @@ rf_pass_through_get_next_batch( /*===========================*/ xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter context */ - ib_int64_t* read_batch_start, /*!<out: starting read + int64_t* read_batch_start, /*!<out: starting read offset in bytes for the next batch of pages */ - ib_int64_t* read_batch_len) /*!<out: length in + int64_t* read_batch_len) /*!<out: length in bytes of the next batch of pages */ { *read_batch_start = ctxt->offset; *read_batch_len = ctxt->data_file_size - ctxt->offset; - if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { - *read_batch_len = ctxt->buffer_capacity; - } - - ctxt->offset += *read_batch_len; -} - -/****************************************************************//** -Deinitialize the pass-through read filter. */ -static -void -rf_pass_through_deinit( -/*===================*/ - xb_read_filt_ctxt_t* ctxt __attribute__((unused))) - /*!<in: read filter context */ -{ -} - -/****************************************************************//** -Initialize the changed page bitmap-based read filter. Assumes that -the bitmap is already set up in changed_page_bitmap. */ -static -void -rf_bitmap_init( -/*===========*/ - xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter - context */ - const xb_fil_cur_t* cursor, /*!<in: read cursor */ - ulint space_id) /*!<in: space id */ -{ - common_init(ctxt, cursor); - ctxt->bitmap_range = xb_page_bitmap_range_init(changed_page_bitmap, - space_id); - ctxt->filter_batch_end = 0; -} - -/****************************************************************//** -Get the next batch of pages for the bitmap read filter. */ -static -void -rf_bitmap_get_next_batch( -/*=====================*/ - xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter - context */ - ib_int64_t* read_batch_start, /*!<out: starting read - offset in bytes for the - next batch of pages */ - ib_int64_t* read_batch_len) /*!<out: length in - bytes of the next batch - of pages */ -{ - ulint start_page_id; - const ulint page_size = ctxt->page_size; - - start_page_id = (ulint)(ctxt->offset / page_size); - - xb_a (ctxt->offset % page_size == 0); - - if (start_page_id == ctxt->filter_batch_end) { - - /* Used up all the previous bitmap range, get some more */ - ulint next_page_id; - - /* Find the next changed page using the bitmap */ - next_page_id = xb_page_bitmap_range_get_next_bit - (ctxt->bitmap_range, TRUE); - - if (next_page_id == ULINT_UNDEFINED) { - *read_batch_len = 0; - return; - } - - ctxt->offset = next_page_id * page_size; - - /* Find the end of the current changed page block by searching - for the next cleared bitmap bit */ - ctxt->filter_batch_end - = xb_page_bitmap_range_get_next_bit(ctxt->bitmap_range, - FALSE); - xb_a(next_page_id < ctxt->filter_batch_end); - } - - *read_batch_start = ctxt->offset; - if (ctxt->filter_batch_end == ULINT_UNDEFINED) { - /* No more cleared bits in the bitmap, need to copy all the - remaining pages. */ - *read_batch_len = ctxt->data_file_size - ctxt->offset; - } else { - *read_batch_len = ctxt->filter_batch_end * page_size - - ctxt->offset; - } - - /* If the page block is larger than the buffer capacity, limit it to - buffer capacity. The subsequent invocations will continue returning - the current block in buffer-sized pieces until ctxt->filter_batch_end - is reached, trigerring the next bitmap query. */ - if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { + if (*read_batch_len > (int64_t)ctxt->buffer_capacity) { *read_batch_len = ctxt->buffer_capacity; } ctxt->offset += *read_batch_len; - xb_a (ctxt->offset % page_size == 0); - xb_a (*read_batch_start % page_size == 0); - xb_a (*read_batch_len % page_size == 0); -} - -/****************************************************************//** -Deinitialize the changed page bitmap-based read filter. */ -static -void -rf_bitmap_deinit( -/*=============*/ - xb_read_filt_ctxt_t* ctxt) /*!<in/out: read filter context */ -{ - xb_page_bitmap_range_deinit(ctxt->bitmap_range); } /* The pass-through read filter */ xb_read_filt_t rf_pass_through = { &rf_pass_through_init, &rf_pass_through_get_next_batch, - &rf_pass_through_deinit -}; - -/* The changed page bitmap-based read filter */ -xb_read_filt_t rf_bitmap = { - &rf_bitmap_init, - &rf_bitmap_get_next_batch, - &rf_bitmap_deinit }; diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h index 51150705..caf8ac56 100644 --- a/extra/mariabackup/read_filt.h +++ b/extra/mariabackup/read_filt.h @@ -25,42 +25,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #ifndef XB_READ_FILT_H #define XB_READ_FILT_H -#include "changed_page_bitmap.h" - -typedef uint32_t space_id_t; +#include <cstdint> +#include <cstddef> struct xb_fil_cur_t; /* The read filter context */ struct xb_read_filt_ctxt_t { - ib_int64_t offset; /*!< current file offset */ - ib_int64_t data_file_size; /*!< data file size */ + int64_t offset; /*!< current file offset */ + int64_t data_file_size; /*!< data file size */ size_t buffer_capacity;/*!< read buffer capacity */ - space_id_t space_id; /*!< space id */ - /* The following fields used only in bitmap filter */ - /* Move these to union if any other filters are added in future */ - xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range - iterator for space_id */ - ulint page_size; /*!< page size */ - ulint filter_batch_end;/*!< the ending page id of the - current changed page block in - the bitmap */ - /** TODO: remove this default constructor */ - xb_read_filt_ctxt_t() : page_size(0) {} }; /* The read filter */ struct xb_read_filt_t { void (*init)(xb_read_filt_ctxt_t* ctxt, - const xb_fil_cur_t* cursor, - ulint space_id); + const xb_fil_cur_t* cursor); void (*get_next_batch)(xb_read_filt_ctxt_t* ctxt, - ib_int64_t* read_batch_start, - ib_int64_t* read_batch_len); - void (*deinit)(xb_read_filt_ctxt_t* ctxt); + int64_t* read_batch_start, + int64_t* read_batch_len); }; extern xb_read_filt_t rf_pass_through; -extern xb_read_filt_t rf_bitmap; #endif diff --git a/extra/mariabackup/thread_pool.cc b/extra/mariabackup/thread_pool.cc new file mode 100644 index 00000000..e18581f4 --- /dev/null +++ b/extra/mariabackup/thread_pool.cc @@ -0,0 +1,50 @@ +#include "thread_pool.h" +#include "common.h" + +bool ThreadPool::start(size_t threads_count) { + if (!m_stopped) + return false; + m_stopped = false; + for (unsigned i = 0; i < threads_count; ++i) + m_threads.emplace_back(&ThreadPool::thread_func, this, i); + return true; +} + +void ThreadPool::stop() { + if (m_stopped) + return; + m_stop = true; + m_cv.notify_all(); + for (auto &t : m_threads) + t.join(); + m_stopped = true; +}; + +void ThreadPool::push(ThreadPool::job_t &&j) { + std::unique_lock<std::mutex> lock(m_mutex); + m_jobs.push(j); + lock.unlock(); + m_cv.notify_one(); +} + +void ThreadPool::thread_func(unsigned thread_num) { + if (my_thread_init()) + die("Can't init mysql thread"); + std::unique_lock<std::mutex> lock(m_mutex); + while(true) { + if (m_stop) + goto exit; + while (!m_jobs.empty()) { + if (m_stop) + goto exit; + job_t j = std::move(m_jobs.front()); + m_jobs.pop(); + lock.unlock(); + j(thread_num); + lock.lock(); + } + m_cv.wait(lock, [&] { return m_stop || !m_jobs.empty(); }); + } +exit: + my_thread_end(); +} diff --git a/extra/mariabackup/thread_pool.h b/extra/mariabackup/thread_pool.h new file mode 100644 index 00000000..10ad74c6 --- /dev/null +++ b/extra/mariabackup/thread_pool.h @@ -0,0 +1,62 @@ +#pragma once +#include <queue> +#include <vector> +#include <functional> +#include <thread> +#include <mutex> +#include <condition_variable> +#include <atomic> +#include "trx0sys.h" + +class ThreadPool { +public: + typedef std::function<void(unsigned)> job_t; + + ThreadPool() { m_stop = false; m_stopped = true; } + ThreadPool (ThreadPool &&other) = delete; + ThreadPool & operator= (ThreadPool &&other) = delete; + ThreadPool(const ThreadPool &) = delete; + ThreadPool & operator= (const ThreadPool &) = delete; + + bool start(size_t threads_count); + void stop(); + void push(job_t &&j); + size_t threads_count() const { return m_threads.size(); } +private: + void thread_func(unsigned thread_num); + std::mutex m_mutex; + std::condition_variable m_cv; + std::queue<job_t> m_jobs; + std::atomic<bool> m_stop; + std::atomic<bool> m_stopped; + std::vector<std::thread> m_threads; +}; + +class TasksGroup { +public: + TasksGroup(ThreadPool &thread_pool) : m_thread_pool(thread_pool) { + m_tasks_count = 0; + m_tasks_result = 1; + } + void push_task(ThreadPool::job_t &&j) { + ++m_tasks_count; + m_thread_pool.push(std::forward<ThreadPool::job_t>(j)); + } + void finish_task(int res) { + --m_tasks_count; + m_tasks_result.fetch_and(res); + } + int get_result() const { return m_tasks_result; } + bool is_finished() const { + return !m_tasks_count; + } + bool wait_for_finish() { + while (!is_finished()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return get_result(); + } +private: + ThreadPool &m_thread_pool; + std::atomic<size_t> m_tasks_count; + std::atomic<int> m_tasks_result; +}; diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc index 052cea26..13f19ca6 100644 --- a/extra/mariabackup/write_filt.cc +++ b/extra/mariabackup/write_filt.cc @@ -144,6 +144,18 @@ wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) return false; } + /* Check whether TRX_SYS page has been changed */ + if (mach_read_from_4(page + FIL_PAGE_SPACE_ID) + == TRX_SYS_SPACE + && mach_read_from_4(page + FIL_PAGE_OFFSET) + == TRX_SYS_PAGE_NO) { + msg(cursor->thread_n, + "--incremental backup is impossible if " + "the server had been restarted with " + "different innodb_undo_tablespaces."); + return false; + } + /* updated page */ if (cp->npages == page_size / 4) { /* flush buffer */ diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc index acaf5c50..15463a85 100644 --- a/extra/mariabackup/wsrep.cc +++ b/extra/mariabackup/wsrep.cc @@ -55,6 +55,9 @@ permission notice: #define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info" #define XB_GALERA_DONOR_INFO_FILENAME "donor_galera_info" +/* backup copy of galera info file as sent by donor */ +#define XB_GALERA_INFO_FILENAME_SST "xtrabackup_galera_info_SST" + /*********************************************************************** Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that information is present in the trx system header. Otherwise, do nothing. */ @@ -68,21 +71,47 @@ xb_write_galera_info(bool incremental_prepare) long long seqno; MY_STAT statinfo; - /* Do not overwrite an existing file to be compatible with - servers with older server versions */ - if (!incremental_prepare && - my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) { - - return; - } - xid.null(); + /* try to read last wsrep XID from innodb rsegs, we will use it + instead of galera info file received from donor + */ if (!trx_rseg_read_wsrep_checkpoint(xid)) { - + /* no worries yet, SST may have brought in galera info file + from some old MariaDB version, which does not support + wsrep XID storing in innodb rsegs + */ return; } + /* if SST brought in galera info file, copy it as *_SST file + this will not be used, saved just for future reference + */ + if (my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) { + FILE* fp_in = fopen(XB_GALERA_INFO_FILENAME, "r"); + FILE* fp_out = fopen(XB_GALERA_INFO_FILENAME_SST, "w"); + + char buf[BUFSIZ] = {'\0'}; + size_t size; + while ((size = fread(buf, 1, BUFSIZ, fp_in))) { + if (fwrite(buf, 1, size, fp_out) != strlen(buf)) { + die( + "could not write to " + XB_GALERA_INFO_FILENAME_SST + ", errno = %d\n", + errno); + } + } + if (!feof(fp_in)) { + die( + XB_GALERA_INFO_FILENAME_SST + " not fully copied\n" + ); + } + fclose(fp_out); + fclose(fp_in); + } + wsrep_uuid_t uuid; memcpy(uuid.data, wsrep_xid_uuid(&xid), sizeof(uuid.data)); if (wsrep_uuid_print(&uuid, uuid_str, @@ -97,7 +126,6 @@ xb_write_galera_info(bool incremental_prepare) "could not create " XB_GALERA_INFO_FILENAME ", errno = %d\n", errno); - exit(EXIT_FAILURE); } seqno = wsrep_xid_seqno(&xid); diff --git a/extra/mariabackup/xb_plugin.h b/extra/mariabackup/xb_plugin.h deleted file mode 100644 index fea24b6b..00000000 --- a/extra/mariabackup/xb_plugin.h +++ /dev/null @@ -1,5 +0,0 @@ -#include <mysql.h> -#include <string> -extern void xb_plugin_backup_init(MYSQL *mysql); -extern const char* xb_plugin_get_config(); -extern void xb_plugin_prepare_init(int argc, char **argv, const char *dir); diff --git a/extra/mariabackup/xbstream.cc b/extra/mariabackup/xbstream.cc index 6306806b..d69a0029 100644 --- a/extra/mariabackup/xbstream.cc +++ b/extra/mariabackup/xbstream.cc @@ -262,7 +262,7 @@ mode_create(int argc, char **argv) return 1; } - stream = xb_stream_write_new(); + stream = xb_stream_write_new(nullptr, nullptr); if (stream == NULL) { msg("%s: xb_stream_write_new() failed.", my_progname); return 1; @@ -287,7 +287,7 @@ mode_create(int argc, char **argv) goto err; } - file = xb_stream_write_open(stream, filepath, &mystat, NULL, NULL); + file = xb_stream_write_open(stream, filepath, &mystat, false); if (file == NULL) { goto err; } @@ -314,7 +314,8 @@ err: static file_entry_t * -file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) +file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen, + uchar chunk_flags) { file_entry_t *entry; ds_file_t *file; @@ -331,7 +332,8 @@ file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) } entry->pathlen = pathlen; - file = ds_open(ctxt->ds_ctxt, path, NULL); + file = ds_open(ctxt->ds_ctxt, path, NULL, + chunk_flags == XB_STREAM_FLAG_REWRITE); if (file == NULL) { msg("%s: failed to create file.", my_progname); @@ -412,10 +414,50 @@ extract_worker_thread_func(void *arg) (uchar *) chunk.path, chunk.pathlen); + if (entry && (chunk.type == XB_CHUNK_TYPE_REMOVE || + chunk.type == XB_CHUNK_TYPE_RENAME)) { + msg("%s: rename and remove chunks can not be applied to opened file: %s", + my_progname, chunk.path); + pthread_mutex_unlock(ctxt->mutex); + break; + } + + if (chunk.type == XB_CHUNK_TYPE_REMOVE) { + if (ds_remove(ctxt->ds_ctxt, chunk.path)) { + msg("%s: error on file removing: %s", my_progname, chunk.path); + pthread_mutex_unlock(ctxt->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + pthread_mutex_unlock(ctxt->mutex); + continue; + } + + if (chunk.type == XB_CHUNK_TYPE_RENAME) { + if (my_hash_search(ctxt->filehash, + reinterpret_cast<const uchar *>(chunk.data), chunk.length)) { + msg("%s: rename chunks can not be applied to opened file: %s", + my_progname, reinterpret_cast<const uchar *>(chunk.data)); + pthread_mutex_unlock(ctxt->mutex); + break; + } + if (ds_rename(ctxt->ds_ctxt, chunk.path, + reinterpret_cast<const char *>(chunk.data))) { + msg("%s: error on file renaming: %s to %s", my_progname, + reinterpret_cast<const char *>(chunk.data), chunk.path); + pthread_mutex_unlock(ctxt->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + pthread_mutex_unlock(ctxt->mutex); + continue; + } + if (entry == NULL) { entry = file_entry_new(ctxt, chunk.path, - chunk.pathlen); + chunk.pathlen, + chunk.flags); if (entry == NULL) { pthread_mutex_unlock(ctxt->mutex); break; @@ -432,6 +474,18 @@ extract_worker_thread_func(void *arg) pthread_mutex_unlock(ctxt->mutex); + if (chunk.type == XB_CHUNK_TYPE_SEEK) { + if (ds_seek_set(entry->file, chunk.offset)) { + msg("%s: my_seek() failed.", my_progname); + pthread_mutex_unlock(&entry->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + entry->offset = chunk.offset; + pthread_mutex_unlock(&entry->mutex); + continue; + } + res = xb_stream_validate_checksum(&chunk); if (res != XB_STREAM_READ_CHUNK) { diff --git a/extra/mariabackup/xbstream.h b/extra/mariabackup/xbstream.h index 1b36ec24..c8b2997d 100644 --- a/extra/mariabackup/xbstream.h +++ b/extra/mariabackup/xbstream.h @@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA /* Chunk flags */ /* Chunk can be ignored if unknown version/format */ #define XB_STREAM_FLAG_IGNORABLE 0x01 +#define XB_STREAM_FLAG_REWRITE 0x02 /* Magic + flags + type + path len */ #define CHUNK_HEADER_CONSTANT_LEN ((sizeof(XB_STREAM_CHUNK_MAGIC) - 1) + \ @@ -48,18 +49,21 @@ typedef enum { /************************************************************************ Write interface. */ -typedef ssize_t xb_stream_write_callback(xb_wstream_file_t *file, +typedef ssize_t xb_stream_write_callback( void *userdata, const void *buf, size_t len); -xb_wstream_t *xb_stream_write_new(void); - +xb_wstream_t *xb_stream_write_new( + xb_stream_write_callback *write_callback, void *user_data); xb_wstream_file_t *xb_stream_write_open(xb_wstream_t *stream, const char *path, - MY_STAT *mystat, void *userdata, - xb_stream_write_callback *onwrite); + const MY_STAT *mystat, bool rewrite); int xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len); - +int xb_stream_write_seek_set(xb_wstream_file_t *file, my_off_t offset); +int xb_stream_write_remove(xb_wstream_t *stream, const char *path); +int +xb_stream_write_rename( + xb_wstream_t *stream, const char *old_path, const char *new_path); int xb_stream_write_close(xb_wstream_file_t *file); int xb_stream_write_done(xb_wstream_t *stream); @@ -76,6 +80,9 @@ typedef enum { typedef enum { XB_CHUNK_TYPE_UNKNOWN = '\0', XB_CHUNK_TYPE_PAYLOAD = 'P', + XB_CHUNK_TYPE_RENAME = 'R', + XB_CHUNK_TYPE_REMOVE = 'D', + XB_CHUNK_TYPE_SEEK = 'S', XB_CHUNK_TYPE_EOF = 'E' } xb_chunk_type_t; diff --git a/extra/mariabackup/xbstream_read.cc b/extra/mariabackup/xbstream_read.cc index b54a9815..d82176ad 100644 --- a/extra/mariabackup/xbstream_read.cc +++ b/extra/mariabackup/xbstream_read.cc @@ -59,6 +59,9 @@ validate_chunk_type(uchar code) { switch ((xb_chunk_type_t) code) { case XB_CHUNK_TYPE_PAYLOAD: + case XB_CHUNK_TYPE_RENAME: + case XB_CHUNK_TYPE_REMOVE: + case XB_CHUNK_TYPE_SEEK: case XB_CHUNK_TYPE_EOF: return (xb_chunk_type_t) code; default: @@ -159,57 +162,91 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) } chunk->path[pathlen] = '\0'; - if (chunk->type == XB_CHUNK_TYPE_EOF) { + if (chunk->type == XB_CHUNK_TYPE_EOF || + chunk->type == XB_CHUNK_TYPE_REMOVE) { return XB_STREAM_READ_CHUNK; } - /* Payload length */ - F_READ(tmpbuf, 16); - ullval = uint8korr(tmpbuf); - if (ullval > (ulonglong) SIZE_T_MAX) { - msg("xb_stream_read_chunk(): chunk length is too large at " - "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, - ullval); - goto err; + if (chunk->type == XB_CHUNK_TYPE_RENAME) { + F_READ(tmpbuf, 4); + size_t new_pathlen = uint4korr(tmpbuf); + if (new_pathlen >= FN_REFLEN) { + msg("xb_stream_read_chunk(): path length (%lu) for new name of 'rename'" + " chunk is too large", (ulong) new_pathlen); + goto err; + } + chunk->length = new_pathlen; + stream->offset +=4; } - chunk->length = (size_t) ullval; - stream->offset += 8; - - /* Payload offset */ - ullval = uint8korr(tmpbuf + 8); - if (ullval > (ulonglong) MY_OFF_T_MAX) { - msg("xb_stream_read_chunk(): chunk offset is too large at " - "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, - ullval); - goto err; + else if (chunk->type == XB_CHUNK_TYPE_SEEK) { + F_READ(tmpbuf, 8); + chunk->offset = uint8korr(tmpbuf); + stream->offset += 8; + return XB_STREAM_READ_CHUNK; + } + else { + /* Payload length */ + F_READ(tmpbuf, 16); + ullval = uint8korr(tmpbuf); + if (ullval > (ulonglong) SIZE_T_MAX) { + msg("xb_stream_read_chunk(): chunk length is too large at " + "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->length = (size_t) ullval; + stream->offset += 8; + + /* Payload offset */ + ullval = uint8korr(tmpbuf + 8); + if (ullval > (ulonglong) MY_OFF_T_MAX) { + msg("xb_stream_read_chunk(): chunk offset is too large at " + "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->offset = (my_off_t) ullval; + stream->offset += 8; } - chunk->offset = (my_off_t) ullval; - stream->offset += 8; - /* Reallocate the buffer if needed */ - if (chunk->length > chunk->buflen) { - chunk->data = my_realloc(PSI_NOT_INSTRUMENTED, chunk->data, chunk->length, - MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + /* Reallocate the buffer if needed, take into account trailing '\0' for + new file name in the case of XB_CHUNK_TYPE_RENAME */ + if (chunk->length + 1 > chunk->buflen) { + chunk->data = my_realloc(PSI_NOT_INSTRUMENTED, chunk->data, + chunk->length + 1, MYF(MY_WME | MY_ALLOW_ZERO_PTR)); if (chunk->data == NULL) { msg("xb_stream_read_chunk(): failed to increase buffer " - "to %lu bytes.", (ulong) chunk->length); + "to %lu bytes.", (ulong) chunk->length + 1); goto err; } - chunk->buflen = chunk->length; + chunk->buflen = chunk->length + 1; } - /* Checksum */ - F_READ(tmpbuf, 4); - chunk->checksum = uint4korr(tmpbuf); - chunk->checksum_offset = stream->offset; - - /* Payload */ - if (chunk->length > 0) { + if (chunk->type == XB_CHUNK_TYPE_RENAME) { + if (chunk->length == 0) { + msg("xb_stream_read_chunk(): failed to read new name for file to rename " + ": %s", chunk->path); + goto err; + } F_READ(chunk->data, chunk->length); stream->offset += chunk->length; + reinterpret_cast<char *>(chunk->data)[chunk->length] = '\0'; + ++chunk->length; } + else { + /* Checksum */ + F_READ(tmpbuf, 4); + chunk->checksum = uint4korr(tmpbuf); + chunk->checksum_offset = stream->offset; + + /* Payload */ + if (chunk->length > 0) { + F_READ(chunk->data, chunk->length); + stream->offset += chunk->length; + } - stream->offset += 4; + stream->offset += 4; + } return XB_STREAM_READ_CHUNK; diff --git a/extra/mariabackup/xbstream_write.cc b/extra/mariabackup/xbstream_write.cc index 5801e867..926e091b 100644 --- a/extra/mariabackup/xbstream_write.cc +++ b/extra/mariabackup/xbstream_write.cc @@ -21,6 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include <my_global.h> #include <my_base.h> #include <zlib.h> +#include <stdint.h> #include "common.h" #include "xbstream.h" @@ -29,6 +30,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA struct xb_wstream_struct { pthread_mutex_t mutex; + xb_stream_write_callback *write; + void *user_data; }; struct xb_wstream_file_struct { @@ -39,8 +42,7 @@ struct xb_wstream_file_struct { char *chunk_ptr; size_t chunk_free; my_off_t offset; - void *userdata; - xb_stream_write_callback *write; + bool rewrite; }; static int xb_stream_flush(xb_wstream_file_t *file); @@ -50,7 +52,7 @@ static int xb_stream_write_eof(xb_wstream_file_t *file); static ssize_t -xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)), +xb_stream_default_write_callback( void *userdata __attribute__((unused)), const void *buf, size_t len) { @@ -60,21 +62,31 @@ xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)) } xb_wstream_t * -xb_stream_write_new(void) +xb_stream_write_new( + xb_stream_write_callback *write_callback, void *user_data) { xb_wstream_t *stream; stream = (xb_wstream_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(xb_wstream_t), MYF(MY_FAE)); pthread_mutex_init(&stream->mutex, NULL); + if (write_callback) { +#ifdef _WIN32 + setmode(fileno(stdout), _O_BINARY); +#endif + stream->write = write_callback; + stream->user_data = user_data; + } + else { + stream->write = xb_stream_default_write_callback; + stream->user_data = user_data; + } return stream;; } xb_wstream_file_t * xb_stream_write_open(xb_wstream_t *stream, const char *path, - MY_STAT *mystat __attribute__((unused)), - void *userdata, - xb_stream_write_callback *onwrite) + const MY_STAT *mystat __attribute__((unused)), bool rewrite) { xb_wstream_file_t *file; size_t path_len; @@ -109,16 +121,7 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path, file->offset = 0; file->chunk_ptr = file->chunk; file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE; - if (onwrite) { -#ifdef _WIN32 - setmode(fileno(stdout), _O_BINARY); -#endif - file->userdata = userdata; - file->write = onwrite; - } else { - file->userdata = NULL; - file->write = xb_stream_default_write_callback; - } + file->rewrite = rewrite; return file; } @@ -202,7 +205,8 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; - *ptr++ = 0; /* Chunk flags */ + *ptr++ = + file->rewrite ? XB_STREAM_FLAG_REWRITE : 0; /* Chunk flags */ *ptr++ = (uchar) XB_CHUNK_TYPE_PAYLOAD; /* Chunk type */ @@ -227,11 +231,11 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); - if (file->write(file, file->userdata, tmpbuf, ptr-tmpbuf) == -1) + if (stream->write(stream->user_data, tmpbuf, ptr-tmpbuf) == -1) goto err; - if (file->write(file, file->userdata, buf, len) == -1) /* Payload */ + if (stream->write(stream->user_data, buf, len) == -1) /* Payload */ goto err; file->offset+= len; @@ -247,6 +251,38 @@ err: return 1; } +int xb_stream_write_seek_set(xb_wstream_file_t *file, my_off_t offset) +{ + /* Chunk magic + flags + chunk type + path_len + path + offset */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + + FN_REFLEN + 8]; + int error = 0; + xb_wstream_t *stream = file->stream; + uchar *ptr = tmpbuf; + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + *ptr++ = 0; /* Chunk flags */ + *ptr++ = (uchar) XB_CHUNK_TYPE_SEEK; /* Chunk type */ + int4store(ptr, file->path_len); /* Path length */ + ptr += 4; + memcpy(ptr, file->path, file->path_len); /* Path */ + ptr += file->path_len; + int8store(ptr, static_cast<int64_t>(offset)); /* Offset */ + ptr += 8; + if (xb_stream_flush(file)) + return 1; + pthread_mutex_lock(&stream->mutex); + if (stream->write(stream->user_data, tmpbuf, ptr-tmpbuf) == -1) + error = 1; + if (!error) + file->offset = offset; + pthread_mutex_unlock(&stream->mutex); + if (xb_stream_flush(file)) + return 1; + return error; +} + static int xb_stream_write_eof(xb_wstream_file_t *file) @@ -278,7 +314,7 @@ xb_stream_write_eof(xb_wstream_file_t *file) xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); - if (file->write(file, file->userdata, tmpbuf, + if (stream->write(stream->user_data, tmpbuf, (ulonglong) (ptr - tmpbuf)) == -1) goto err; @@ -291,3 +327,77 @@ err: return 1; } + + +int +xb_stream_write_remove(xb_wstream_t *stream, const char *path) { + /* Chunk magic + flags + chunk type + path_len + path */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + FN_REFLEN]; + uchar *ptr = tmpbuf; + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_REMOVE; /* Chunk type */ + size_t path_len = strlen(path); + int4store(ptr, path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, path, path_len); /* Path */ + ptr += path_len; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + pthread_mutex_lock(&stream->mutex); + + ssize_t result = stream->write(stream->user_data, tmpbuf, + (ulonglong) (ptr - tmpbuf)); + + pthread_mutex_unlock(&stream->mutex); + + return result < 0; + +} + +int +xb_stream_write_rename( + xb_wstream_t *stream, const char *old_path, const char *new_path) { + /* Chunk magic + flags + chunk type + path_len + path + path_len + path*/ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + + 4 + FN_REFLEN + 4 + FN_REFLEN]; + uchar *ptr = tmpbuf; + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_RENAME; /* Chunk type */ + size_t path_len = strlen(old_path); + int4store(ptr, path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, old_path, path_len); /* Path */ + ptr += path_len; + + path_len = strlen(new_path); + int4store(ptr, path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, new_path, path_len); /* Path */ + ptr += path_len; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + pthread_mutex_lock(&stream->mutex); + + ssize_t result = stream->write(stream->user_data, tmpbuf, + (ulonglong) (ptr - tmpbuf)); + + pthread_mutex_unlock(&stream->mutex); + + return result < 0; +} + diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 485cb143..5979bbd3 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4,7 +4,7 @@ MariaBackup: hot backup tool for InnoDB Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. -(c) 2017, 2022, MariaDB Corporation. +(c) 2017, 2024, MariaDB Corporation. Portions written by Marko Mäkelä. This program is free software; you can redistribute it and/or modify @@ -54,7 +54,6 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include <scope.h> #include <sql_class.h> -#include <fcntl.h> #include <string.h> #ifdef __linux__ @@ -70,6 +69,7 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA # include <sys/sysctl.h> #endif +#include "aria_backup_client.h" #include <btr0sea.h> #include <lock0lock.h> @@ -82,6 +82,7 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include <buf0dblwr.h> #include <buf0flu.h> #include "ha_innodb.h" +#include "fts0types.h" #include <list> #include <sstream> @@ -97,23 +98,26 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include "xb_regex.h" #include "fil_cur.h" #include "write_filt.h" -#include "xtrabackup.h" #include "ds_buffer.h" #include "ds_tmpfile.h" #include "xbstream.h" -#include "changed_page_bitmap.h" #include "read_filt.h" #include "backup_wsrep.h" #include "innobackupex.h" #include "backup_mysql.h" #include "backup_copy.h" #include "backup_mysql.h" -#include "xb_plugin.h" +#include "encryption_plugin.h" #include <sql_plugin.h> #include <srv0srv.h> #include <log.h> #include <derror.h> #include <thr_timer.h> +#include <tuple> +#include "ddl_log.h" +#include "common_engine.h" +#include "lex_string.h" +#include "sql_table.h" #include "backup_debug.h" #define MB_CORRUPTED_PAGES_FILE "innodb_corrupted_pages" @@ -126,6 +130,9 @@ int sd_notifyf() { return 0; } int sys_var_init(); +extern const char* fts_common_tables[]; +extern const fts_index_selector_t fts_index_selector[]; + /* === xtrabackup specific options === */ #define DEFAULT_TARGET_DIR "./xtrabackup_backupfiles/" char xtrabackup_real_target_dir[FN_REFLEN] = DEFAULT_TARGET_DIR; @@ -140,8 +147,8 @@ my_bool xtrabackup_decrypt_decompress; my_bool xtrabackup_print_param; my_bool xtrabackup_mysqld_args; my_bool xtrabackup_help; - my_bool xtrabackup_export; +my_bool ignored_option; longlong xtrabackup_use_memory; @@ -155,7 +162,6 @@ char *xtrabackup_incremental; lsn_t incremental_lsn; lsn_t incremental_to_lsn; lsn_t incremental_last_lsn; -xb_page_bitmap *changed_page_bitmap; char *xtrabackup_incremental_basedir; /* for --backup */ char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */ @@ -195,10 +201,12 @@ struct xb_filter_entry_t{ xb_filter_entry_t *name_hash; }; +lsn_t checkpoint_lsn_start; +lsn_t checkpoint_no_start; /** whether log_copying_thread() is active; protected by recv_sys.mutex */ static bool log_copying_running; -int xtrabackup_parallel; +uint xtrabackup_parallel; char *xtrabackup_stream_str = NULL; xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE; @@ -255,7 +263,7 @@ recv_sys.mutex. */ static std::set<uint32_t> fail_undo_ids; longlong innobase_page_size = (1LL << 14); /* 16KB */ -char* innobase_buffer_pool_filename = NULL; +char *innobase_buffer_pool_filename = NULL; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ @@ -361,10 +369,8 @@ my_bool opt_galera_info = FALSE; my_bool opt_slave_info = FALSE; my_bool opt_no_lock = FALSE; my_bool opt_safe_slave_backup = FALSE; -my_bool opt_rsync = FALSE; my_bool opt_force_non_empty_dirs = FALSE; my_bool opt_noversioncheck = FALSE; -my_bool opt_no_backup_locks = FALSE; my_bool opt_decompress = FALSE; my_bool opt_remove_original; my_bool opt_log_innodb_page_corruption; @@ -422,6 +428,8 @@ pthread_cond_t scanned_lsn_cond; /** Store the deferred tablespace name during --backup */ static std::set<std::string> defer_space_names; +typedef decltype(fil_space_t::id) space_id_t; + typedef std::map<space_id_t,std::string> space_id_to_name_t; struct ddl_tracker_t { @@ -696,8 +704,190 @@ typedef void (*process_single_tablespace_func_t)(const char *dirname, uint32_t defer_space_id); static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback); +const char *convert_dst(const char *dst) { + return + (xtrabackup_copy_back || xtrabackup_move_back) ? + dst : trim_dotslash(dst); +} + +std::string convert_tablename_to_filepath( + const char *data_dir_path, const std::string &db, const std::string &table) { + char dbbuff[FN_REFLEN]; + char tbbuff[FN_REFLEN]; + (void)tablename_to_filename(db.c_str(), dbbuff, sizeof(dbbuff)); + (void)tablename_to_filename(table.c_str(), tbbuff, sizeof(tbbuff)); + std::string result(data_dir_path); + result.append(1, FN_LIBCHAR).append(dbbuff). + append(1, FN_LIBCHAR).append(tbbuff); + return result; +} + +std::tuple<std::string, std::string, std::string> +convert_filepath_to_tablename(const char *filepath) { + char db_name_orig[FN_REFLEN]; + char table_name_orig[FN_REFLEN]; + parse_db_table_from_file_path(filepath, db_name_orig, table_name_orig); + if (!db_name_orig[0] || !table_name_orig[0]) + return std::make_tuple("", "", ""); + char db_name_conv[FN_REFLEN]; + char table_name_conv[FN_REFLEN]; + filename_to_tablename(db_name_orig, db_name_conv, sizeof(db_name_conv)); + filename_to_tablename( + table_name_orig, table_name_conv, sizeof(table_name_conv)); + if (!db_name_conv[0] || !table_name_conv[0]) + return std::make_tuple("", "", ""); + return std::make_tuple(db_name_conv, table_name_conv, + std::string(db_name_orig).append("/").append(table_name_orig)); +} + +std::string get_table_version_from_image(const std::vector<uchar> &frm_image) { + DBUG_ASSERT(frm_image.size() >= 64); + + if (!strncmp((char*) frm_image.data(), "TYPE=VIEW\n", 10)) + return {}; + + if (!is_binary_frm_header(frm_image.data())) + return {}; + + /* Length of the MariaDB extra2 segment in the form file. */ + uint len = uint2korr(frm_image.data() + 4); + const uchar *extra2= frm_image.data() + 64; + + if (*extra2 == '/') // old frm had '/' there + return {}; + + const uchar *e2end= extra2 + len; + while (extra2 + 3 <= e2end) + { + uchar type= *extra2++; + size_t length= *extra2++; + if (!length) + { + if (extra2 + 2 >= e2end) + return {}; + length= uint2korr(extra2); + extra2+= 2; + if (length < 256) + return {}; + } + if (extra2 + length > e2end) + return {}; + if (type == EXTRA2_TABLEDEF_VERSION) { + char buff[MY_UUID_STRING_LENGTH]; + my_uuid2str(extra2, buff, 1); + return std::string(buff, buff + MY_UUID_STRING_LENGTH); + } + extra2+= length; + } + + return {}; +} + +std::pair<bool, legacy_db_type> + get_table_engine_from_image(const std::vector<uchar> &frm_image) { + + DBUG_ASSERT(frm_image.size() >= 64); + + if (!strncmp((char*) frm_image.data(), "TYPE=VIEW\n", 10)) + return std::make_pair(false, DB_TYPE_UNKNOWN); + + if (!is_binary_frm_header(frm_image.data())) + return std::make_pair(false, DB_TYPE_UNKNOWN); + + legacy_db_type dbt = (legacy_db_type)frm_image[3]; + + if (dbt >= DB_TYPE_FIRST_DYNAMIC) + return std::make_pair(false, DB_TYPE_UNKNOWN); + + if (dbt != DB_TYPE_PARTITION_DB) + return std::make_pair(false, dbt); + + dbt = (legacy_db_type)frm_image[61]; + return std::make_pair(true, + dbt < DB_TYPE_FIRST_DYNAMIC ? dbt : DB_TYPE_UNKNOWN); +} + +std::vector<uchar> read_frm_image(File file) { + std::vector<uchar> frm_image; + MY_STAT state; + + if (mysql_file_fstat(file, &state, MYF(MY_WME))) + return frm_image; + + frm_image.resize((size_t)state.st_size, 0); + + if (mysql_file_read( + file, frm_image.data(), (size_t)state.st_size, MYF(MY_NABP))) + frm_image.clear(); + + return frm_image; +} + +std::string read_table_version_id(File file) { + auto frm_image = read_frm_image(file); + if (frm_image.empty()) + return {}; + return get_table_version_from_image(frm_image); +} + +bool is_log_table(const char *dbname, const char *tablename) { + DBUG_ASSERT(dbname); + DBUG_ASSERT(tablename); + + LEX_CSTRING lex_db; + LEX_CSTRING lex_table; + lex_db.str = dbname; + lex_db.length = strlen(dbname); + lex_table.str = tablename; + lex_table.length = strlen(tablename); + + if (!lex_string_eq(&MYSQL_SCHEMA_NAME, &lex_db)) + return false; + + if (lex_string_eq(&GENERAL_LOG_NAME, &lex_table)) + return true; + + if (lex_string_eq(&SLOW_LOG_NAME, &lex_table)) + return true; + + return false; +} + +bool is_stats_table(const char *dbname, const char *tablename) { + DBUG_ASSERT(dbname); + DBUG_ASSERT(tablename); + + LEX_CSTRING lex_db; + LEX_CSTRING lex_table; + lex_db.str = dbname; + lex_db.length = strlen(dbname); + lex_table.str = tablename; + lex_table.length = strlen(tablename); + + if (!lex_string_eq(&MYSQL_SCHEMA_NAME, &lex_db)) + return false; + + CHARSET_INFO *ci= system_charset_info; + + return (lex_table.length > 4 && + /* one of mysql.*_stat tables, but not mysql.innodb* tables*/ + ((my_tolower(ci, lex_table.str[lex_table.length-5]) == 's' && + my_tolower(ci, lex_table.str[lex_table.length-4]) == 't' && + my_tolower(ci, lex_table.str[lex_table.length-3]) == 'a' && + my_tolower(ci, lex_table.str[lex_table.length-2]) == 't' && + my_tolower(ci, lex_table.str[lex_table.length-1]) == 's') && + !(my_tolower(ci, lex_table.str[0]) == 'i' && + my_tolower(ci, lex_table.str[1]) == 'n' && + my_tolower(ci, lex_table.str[2]) == 'n' && + my_tolower(ci, lex_table.str[3]) == 'o'))); +} + /* ======== Datafiles iterator ======== */ struct datafiles_iter_t { + datafiles_iter_t() : space(fil_system.space_list.end()), node(nullptr), started(FALSE) { + } + ~datafiles_iter_t() { + } space_list_t::iterator space = fil_system.space_list.end(); fil_node_t *node = nullptr; bool started = false; @@ -777,8 +967,6 @@ static void *dbug_execute_in_new_connection(void *arg) return nullptr; } -static pthread_t dbug_alter_thread; - /* Execute query from a new connection, in own thread. @@ -789,8 +977,9 @@ Execute query from a new connection, in own thread. otherwise query should return error. @param expected_errno - if not 0, and query finished with error, expected mysql_errno() +@return created thread id */ -static void dbug_start_query_thread( +static pthread_t dbug_start_query_thread( const char *query, const char *wait_state, int expected_err, @@ -802,12 +991,14 @@ static void dbug_start_query_thread( par->expect_err = expected_err; par->expect_errno = expected_errno; par->con = xb_mysql_connect(); - - mysql_thread_create(0, &dbug_alter_thread, nullptr, + if (mysql_set_server_option(par->con, MYSQL_OPTION_MULTI_STATEMENTS_ON)) + die("Can't set multistatement option for query: %s", query); + pthread_t result_thread; + mysql_thread_create(0, &result_thread, nullptr, dbug_execute_in_new_connection, par); if (!wait_state) - return; + return result_thread; char q[256]; snprintf(q, sizeof(q), @@ -829,7 +1020,11 @@ static void dbug_start_query_thread( end: msg("query '%s' on connection %lu reached state '%s'", query, mysql_thread_id(par->con), wait_state); + return result_thread; } + +static pthread_t dbug_alter_thread; +static pthread_t dbug_emulate_ddl_on_intermediate_table_thread; #endif void mdl_lock_all() @@ -952,6 +1147,31 @@ static void backup_file_op(uint32_t space_id, int type, } } +static bool check_if_fts_table(const char *file_name) { + const char *table_name_start = strrchr(file_name, '/'); + if (table_name_start) + ++table_name_start; + else + table_name_start = file_name; + + if (!starts_with(table_name_start,"FTS_")) + return false; + + const char *table_name_end = strrchr(table_name_start, '.'); + if (!table_name_end) + table_name_end = table_name_start + strlen(table_name_start); + ptrdiff_t table_name_len = table_name_end - table_name_end; + + for (const char **suffix = fts_common_tables; *suffix; ++suffix) + if (!strncmp(table_name_start, *suffix, table_name_len)) + return true; + for (size_t i = 0; fts_index_selector[i].suffix; ++i) + if (!strncmp(table_name_start, fts_index_selector[i].suffix, + table_name_len)) + return true; + + return false; +} /* This callback is called if DDL operation is detected, @@ -985,8 +1205,9 @@ static void backup_file_op_fail(uint32_t space_id, int type, break; case FILE_DELETE: fail = !check_if_skip_table( - filename_to_spacename(name, len).c_str()); - msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name); + filename_to_spacename(name, len).c_str()) + && !check_if_fts_table(reinterpret_cast<const char*>(name)); + msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name); break; default: ut_ad(0); @@ -1112,6 +1333,7 @@ enum options_xtrabackup OPT_INNODB_LOG_FILE_BUFFERING, #endif OPT_INNODB_LOG_FILE_SIZE, + OPT_INNODB_LOG_FILES_IN_GROUP, OPT_INNODB_OPEN_FILES, OPT_XTRA_DEBUG_SYNC, OPT_INNODB_CHECKSUM_ALGORITHM, @@ -1129,9 +1351,9 @@ enum options_xtrabackup OPT_NO_LOCK, OPT_SAFE_SLAVE_BACKUP, OPT_RSYNC, + OPT_NO_BACKUP_LOCKS, OPT_FORCE_NON_EMPTY_DIRS, OPT_NO_VERSION_CHECK, - OPT_NO_BACKUP_LOCKS, OPT_DECOMPRESS, OPT_INCREMENTAL_HISTORY_NAME, OPT_INCREMENTAL_HISTORY_UUID, @@ -1343,8 +1565,10 @@ struct my_option xb_client_options[]= { 0, 0, 0, 0, 0, 0}, {"no-lock", OPT_NO_LOCK, - "Use this option to disable table lock " - "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + "This option should not be used as " + "mariadb-backup now is using BACKUP LOCKS, which minimizes the " + "lock time. ALTER TABLE can run in parallel with BACKUP LOCKS." + "Use the --no-lock option it only if ALL your " "tables are InnoDB and you DO NOT CARE about the binary log " "position of the backup. This option shouldn't be used if there " "are any DDL statements being executed or if any updates are " @@ -1373,14 +1597,12 @@ struct my_option xb_client_options[]= { GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"rsync", OPT_RSYNC, - "Uses the rsync utility to optimize local file " - "transfers. When this option is specified, " XB_TOOL_NAME " uses rsync " - "to copy all non-InnoDB files instead of spawning a separate cp for " - "each file, which can be much faster for servers with a large number " - "of databases or tables. This option cannot be used together with " - "--stream.", - (uchar *) &opt_rsync, (uchar *) &opt_rsync, 0, GET_BOOL, NO_ARG, 0, 0, 0, - 0, 0, 0}, + "Obsolete depricated option", + &ignored_option, &ignored_option, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, + "Obsolete depricated option", + &ignored_option, &ignored_option, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " @@ -1398,15 +1620,6 @@ struct my_option xb_client_options[]= { (uchar *) &opt_noversioncheck, (uchar *) &opt_noversioncheck, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"no-backup-locks", OPT_NO_BACKUP_LOCKS, - "This option controls if " - "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " - "on the backup stage. The option has no effect when backup locks are " - "not supported by the server. This option is enabled by default, " - "disable with --no-backup-locks.", - (uchar *) &opt_no_backup_locks, (uchar *) &opt_no_backup_locks, 0, - GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " "extension in a backup previously made with the --compress option. " @@ -1483,11 +1696,10 @@ struct my_option xb_client_options[]= { (uchar *) &opt_remove_original, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, - "This option specifies which types of queries are allowed to complete " - "before " XB_TOOL_NAME " will issue the global lock. Default is all.", - (uchar *) &opt_lock_wait_query_type, (uchar *) &opt_lock_wait_query_type, - &query_type_typelib, GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, - 0}, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, &query_type_typelib, GET_ENUM, + REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, "This option specifies which types of queries should be killed to " @@ -1504,32 +1716,31 @@ struct my_option xb_client_options[]= { NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, - "This option specifies the number of seconds " XB_TOOL_NAME " waits " - "between starting FLUSH TABLES WITH READ LOCK and killing those " - "queries that block it. Default is 0 seconds, which means " - XB_TOOL_NAME " will not attempt to kill any queries.", - (uchar *) &opt_kill_long_queries_timeout, - (uchar *) &opt_kill_long_queries_timeout, 0, GET_UINT, REQUIRED_ARG, 0, 0, + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, - "This option specifies time in seconds that " XB_TOOL_NAME " should wait " - "for queries that would block FTWRL before running it. If there are " - "still such queries when the timeout expires, " XB_TOOL_NAME " terminates " - "with an error. Default is 0, in which case " XB_TOOL_NAME " does not " - "wait for queries to complete and starts FTWRL immediately.", - (uchar *) &opt_lock_wait_timeout, (uchar *) &opt_lock_wait_timeout, 0, - GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + "Alias for startup-wait-timeout", + (uchar*) &opt_lock_wait_timeout, (uchar*) &opt_lock_wait_timeout, + 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"startup-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that mariadb-backup should wait for " + "BACKUP STAGE START to complete. BACKUP STAGE START has to wait until all " + "currently running queries using explicite LOCK TABLES has ended. " + "If there are still such queries when the timeout expires, mariadb-backup " + "terminates with an error. Default is 0, in which case mariadb-backup waits " + "indefinitely for BACKUP STAGE START to finish", + (uchar*) &opt_lock_wait_timeout, (uchar*) &opt_lock_wait_timeout, + 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, - "This option specifies the query run time threshold which is used by " - XB_TOOL_NAME " to detect long-running queries with a non-zero value " - "of --ftwrl-wait-timeout. FTWRL is not started until such " - "long-running queries exist. This option has no effect if " - "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", - (uchar *) &opt_lock_wait_threshold, (uchar *) &opt_lock_wait_threshold, 0, - GET_UINT, REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, - + "Old disabled option which has no effect anymore (not needed " + "with BACKUP LOCKS)", + (uchar*) 0, (uchar*) 0, 0, GET_UINT, + REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, "How many seconds --safe-slave-backup should wait for " @@ -1595,7 +1806,7 @@ struct my_option xb_server_options[] = {"parallel", OPT_XTRA_PARALLEL, "Number of threads to use for parallel datafiles transfer. " "The default value is 1.", - (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT, + (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_UINT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION, @@ -1677,8 +1888,8 @@ struct my_option xb_server_options[] = {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE, "Redo log buffer size in bytes.", (G_PTR*) &log_sys.buf_size, (G_PTR*) &log_sys.buf_size, 0, - IF_WIN(GET_ULL,GET_ULONG), REQUIRED_ARG, 2U << 20, - 2U << 20, SIZE_T_MAX, 0, 4096, 0}, + GET_UINT, REQUIRED_ARG, 2U << 20, + 2U << 20, log_sys.buf_size_max, 0, 4096, 0}, #if defined __linux__ || defined _WIN32 {"innodb_log_file_buffering", OPT_INNODB_LOG_FILE_BUFFERING, "Whether the file system cache for ib_logfile0 is enabled during --backup", @@ -1754,17 +1965,17 @@ struct my_option xb_server_options[] = 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"plugin-dir", OPT_PLUGIN_DIR, - "Server plugin directory. Used to load plugins during 'prepare' phase." - "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", - &xb_plugin_dir, &xb_plugin_dir, - 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + "Server plugin directory. Used to load plugins during 'prepare' phase." + "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", + &xb_plugin_dir, &xb_plugin_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, {"aria_log_dir_path", OPT_ARIA_LOG_DIR_PATH, "Path to individual files and their sizes.", &aria_log_dir_path, &aria_log_dir_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " + {"open_files_limit", 0, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0}, @@ -1885,7 +2096,7 @@ static int prepare_export() IF_WIN("\"","") "\"%s\" --mysqld \"%s\"" " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." " --innodb --innodb-fast-shutdown=0 --loose-partition" - " --innodb-buffer-pool-size=%llu" + " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu" " --console --skip-log-error --skip-log-bin --bootstrap %s< " BOOTSTRAP_FILENAME IF_WIN("\"",""), mariabackup_exe, @@ -1899,7 +2110,7 @@ static int prepare_export() IF_WIN("\"","") "\"%s\" --mysqld" " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." " --innodb --innodb-fast-shutdown=0 --loose-partition" - " --innodb-buffer-pool-size=%llu" + " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu" " --console --log-error= --skip-log-bin --bootstrap %s< " BOOTSTRAP_FILENAME IF_WIN("\"",""), mariabackup_exe, @@ -1948,7 +2159,8 @@ static void usage(void) puts("Open source backup tool for InnoDB and XtraDB\n\ \n\ Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\ -Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\ +Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy.\n\ +Portions Copyright (C) 2017-2023 MariaDB Corporation / MariaDB Plc.\n\ \n\ This program is free software; you can redistribute it and/or\n\ modify it under the terms of the GNU General Public License\n\ @@ -2021,6 +2233,10 @@ xb_get_one_option(const struct my_option *opt, ADD_PRINT_PARAM_OPT(srv_log_group_home_dir); break; + case OPT_INNODB_LOG_FILES_IN_GROUP: + case OPT_INNODB_LOG_FILE_SIZE: + break; + case OPT_INNODB_FLUSH_METHOD: #ifdef _WIN32 /* From: storage/innobase/handler/ha_innodb.cc:innodb_init_params */ @@ -2133,6 +2349,11 @@ xb_get_one_option(const struct my_option *opt, } } break; + case OPT_RSYNC: + case OPT_NO_BACKUP_LOCKS: + if (my_handle_options_init_variables) + fprintf(stderr, "Obsolete option: %s. Ignored\n", opt->name); + break; #define MYSQL_CLIENT #include "sslopt-case.h" #undef MYSQL_CLIENT @@ -2428,7 +2649,12 @@ static bool innodb_init() os_file_delete_if_exists_func(ib_logfile0.c_str(), nullptr); os_file_t file= os_file_create_func(ib_logfile0.c_str(), OS_FILE_CREATE, OS_FILE_NORMAL, - OS_DATA_FILE_NO_O_DIRECT, false, &ret); +#if defined _WIN32 || defined O_DIRECT + OS_DATA_FILE_NO_O_DIRECT, +#else + OS_DATA_FILE, +#endif + false, &ret); if (!ret) { invalid_log: @@ -3026,12 +3252,7 @@ static my_bool xtrabackup_copy_datafile(ds_ctxt *ds_data, goto skip; } - if (!changed_page_bitmap) { - read_filter = &rf_pass_through; - } - else { - read_filter = &rf_bitmap; - } + read_filter = &rf_pass_through; res = xb_fil_cur_open(&cursor, read_filter, node, thread_n, ULLONG_MAX); if (res == XB_FIL_CUR_SKIP) { @@ -3355,50 +3576,22 @@ To use this facility, you need to 3. start mariabackup with --dbug=+d,debug_mariabackup_events */ void dbug_mariabackup_event(const char *event, - const fil_space_t::name_type key) + const fil_space_t::name_type key, + bool need_lock) { + static std::mutex dbug_mariabackup_event_mutex; char *sql = dbug_mariabackup_get_val(event, key); if (sql && *sql) { msg("dbug_mariabackup_event : executing '%s'", sql); - xb_mysql_query(mysql_connection, sql, false, true); - } + if (need_lock) { + std::lock_guard<std::mutex> lock(dbug_mariabackup_event_mutex); + xb_mysql_query(mysql_connection, sql, false, true); + } else + xb_mysql_query(mysql_connection, sql, false, true); + } } #endif // DBUG_OFF -/** Datafiles copying thread.*/ -static void data_copy_thread_func(data_thread_ctxt_t *ctxt) /* thread context */ -{ - uint num = ctxt->num; - fil_node_t* node; - ut_ad(ctxt->corrupted_pages); - - /* - Initialize mysys thread-specific memory so we can - use mysys functions in this thread. - */ - my_thread_init(); - - while ((node = datafiles_iter_next(ctxt->it)) != NULL) { - DBUG_MARIABACKUP_EVENT("before_copy", node->space->name()); - DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", - node->space->name(), - backup_wait_for_lsn(get_current_lsn(mysql_connection));); - /* copy the datafile */ - if (xtrabackup_copy_datafile(ctxt->datasinks->m_data, - ctxt->datasinks->m_meta, node, num, NULL, - xtrabackup_incremental ? wf_incremental : wf_write_through, - *ctxt->corrupted_pages)) - die("failed to copy datafile."); - - DBUG_MARIABACKUP_EVENT("after_copy", node->space->name()); - } - - pthread_mutex_lock(ctxt->count_mutex); - (*ctxt->count)--; - pthread_mutex_unlock(ctxt->count_mutex); - - my_thread_end(); -} /************************************************************************ Initialize the appropriate datasink(s). Both local backups and streaming in the @@ -3553,6 +3746,11 @@ static void xb_load_single_table_tablespace(const char *dirname, } if (file->open_read_only(true) != DB_SUCCESS) { + // Ignore FTS tables, as they can be removed for intermediate tables, + // this code must be executed under stronger or equal to BLOCK_DDL lock, + // so there must not be errors for non-intermediate FTS tables. + if (check_if_fts_table(filname)) + return; die("Can't open datafile %s", name); } @@ -4558,7 +4756,6 @@ bool Backup_datasinks::backup_low() if (failed_ids.size() > 0) { return false; } - if (!xtrabackup_incremental) { safe_strcpy(metadata_type, sizeof(metadata_type), "full-backuped"); @@ -4597,16 +4794,441 @@ bool Backup_datasinks::backup_low() return true; } +class InnodbDataCopier { +public: + InnodbDataCopier(Backup_datasinks &backup_datasinks, + CorruptedPages &corrupted_pages, + ThreadPool &thread_pool) : + m_backup_datasinks(backup_datasinks), + m_corrupted_pages(corrupted_pages), + m_tasks(thread_pool) {} + + ~InnodbDataCopier() { + DBUG_ASSERT(m_tasks.is_finished()); + } + + bool start() { + DBUG_ASSERT(m_tasks.is_finished()); + m_tasks.push_task( + std::bind(&InnodbDataCopier::scan_job, this, std::placeholders::_1)); + return true; + } + + bool wait_for_finish() { + return m_tasks.wait_for_finish(); + } + +private: + void scan_job(unsigned thread_num) { + datafiles_iter_t it; + fil_node_t* node; + while ((node = datafiles_iter_next(&it)) != nullptr) { + m_tasks.push_task( + std::bind(&InnodbDataCopier::copy_job, this, node, + std::placeholders::_1)); + } + m_tasks.finish_task(1); + } + + void copy_job(fil_node_t *node, unsigned thread_num) { + DBUG_ASSERT(node); + // TODO: this came from the old code, where it was not thread-safe + // too, use separate mysql connection per thread here + DBUG_MARIABACKUP_EVENT("before_copy", node->space->name()); + DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", + node->space->name(), + backup_wait_for_lsn( + get_current_lsn(mysql_connection));); + /* copy the datafile */ + if(xtrabackup_copy_datafile(m_backup_datasinks.m_data, + m_backup_datasinks.m_meta, + node, thread_num, NULL, + xtrabackup_incremental + ? wf_incremental : wf_write_through, + m_corrupted_pages)) + die("mariabackup: Error: failed to copy datafile."); + // TODO: this came from the old code, where it was not thread-safe + // too, use separate mysql connection per thread here + DBUG_MARIABACKUP_EVENT("after_copy", node->space->name()); + m_tasks.finish_task(1); + } + + Backup_datasinks &m_backup_datasinks; + CorruptedPages &m_corrupted_pages; + TasksGroup m_tasks; +}; + + +class BackupStages { + + public: + + BackupStages(ds_ctxt_t *ds_data) : + m_bs_con(nullptr), + m_aria_backup(fil_path_to_mysql_datadir, + aria_log_dir_path, + ds_data, m_con_pool, m_thread_pool), + m_common_backup(fil_path_to_mysql_datadir, ds_data, m_con_pool, + m_thread_pool) {} + + ~BackupStages() { destroy(); } + + bool init() { + if ((m_bs_con = xb_mysql_connect()) == nullptr) + return false; + + while(m_con_pool.size() < xtrabackup_parallel) { + MYSQL *con = xb_mysql_connect(); + if (con == nullptr) + return false; + m_con_pool.push_back(con); + } + + if (!m_thread_pool.start(xtrabackup_parallel)) + return false; + if (!m_aria_backup.init()) + return false; + m_aria_backup.set_post_copy_table_hook( + std::bind(&BackupStages::store_table_version, this, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + m_common_backup.set_post_copy_table_hook( + std::bind(&BackupStages::store_table_version, this, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + return true; + } + + void destroy() { + m_thread_pool.stop(); + while (!m_con_pool.empty()) { + MYSQL *con = m_con_pool.back(); + m_con_pool.pop_back(); + mysql_close(con); + } + if (m_bs_con) + mysql_close(m_bs_con); + m_bs_con = nullptr; + } + + bool stage_start(Backup_datasinks &backup_datasinks, + CorruptedPages &corrupted_pages) { + msg("BACKUP STAGE START"); + if (!opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + + history_lock_time = time(NULL); + + if (!lock_for_backup_stage_start(m_bs_con)) { + msg("Error on BACKUP STAGE START query execution"); + return(false); + } + } + + InnodbDataCopier innodb_data_copier(backup_datasinks, + corrupted_pages, + m_thread_pool); + // Start InnoDB data files copy in background + if (!innodb_data_copier.start()) { + msg("Error on starting InnoDB data files backup"); + return false; + } + // Start online non-stats-log Aria tables copying in background + if (!m_aria_backup.start(opt_no_lock)) { + msg("Error on starting Aria data files backup"); + innodb_data_copier.wait_for_finish(); + return false; + } + + // Wait for all innodb data files copy finish + if(!innodb_data_copier.wait_for_finish()) { + msg("InnoDB data files backup process is finished with error"); + return false; + } + // Wait for online non-stats-log Aria tables copy finish + if (!m_aria_backup.wait_for_finish()) { + msg("Aria data files backup process is finished with error"); + return false; + } + + DBUG_MARIABACKUP_EVENT_LOCK("after_aria_background", {}); + + return true; + } + + bool stage_flush() { + msg("BACKUP STAGE FLUSH"); + if (!opt_no_lock && !lock_for_backup_stage_flush(m_bs_con)) { + msg("Error on BACKUP STAGE FLUSH query execution"); + return false; + } + auto tables_in_use = get_tables_in_use(mysql_connection); + // Copy non-stats-log non-in-use tables of non-InnoDB-Aria-RocksDB engines + // in background + if (!m_common_backup.scan(tables_in_use, + &m_copied_common_tables, opt_no_lock, true)) { + msg("Error on scan data directory for common engines"); + return false; + } + // Copy Aria offline non-stats-log non-in-use tables in background + if (!m_aria_backup.copy_offline_tables(&tables_in_use, opt_no_lock, + false)) { + msg("Error on start Aria tables backup"); + return false; + } + + if (!m_aria_backup.copy_log_tail()) { + msg("Error on Aria log tail copy"); + return false; + }; + + // Wait for Aria tables copy finish + if (!m_aria_backup.wait_for_finish()) { + msg("Aria data files backup process is finished with error"); + return false; + } + // Wait for non-InnoDB-Aria-RocksDB engines copy finish + if (!m_common_backup.wait_for_finish()) { + msg("Data files backup process is finished with error"); + return false; + } + + DBUG_EXECUTE_IF("emulate_ddl_on_intermediate_table", + dbug_emulate_ddl_on_intermediate_table_thread = + dbug_start_query_thread( + "SET debug_sync='copy_data_between_tables_after_set_backup_lock " + "SIGNAL copy_started';" + "SET debug_sync='copy_data_between_tables_before_reset_backup_lock " + "SIGNAL before_backup_lock_reset WAIT_FOR backup_lock_reset';" + "SET debug_sync='alter_table_after_temp_table_drop " + "SIGNAL temp_table_dropped';" + "SET SESSION lock_wait_timeout = 1;" + "ALTER TABLE test.t1 ADD COLUMN col1_copy INT, ALGORITHM = COPY;", + NULL, 0, 0); + xb_mysql_query(mysql_connection, + "SET debug_sync='now WAIT_FOR copy_started'", false, true); + ); + + return true; + } + + bool stage_block_ddl(Backup_datasinks &backup_datasinks, + CorruptedPages &corrupted_pages) { + if (!opt_no_lock) { + if (!lock_for_backup_stage_block_ddl(m_bs_con)) { + msg("BACKUP STAGE BLOCK_DDL"); + return false; + } + if (have_galera_enabled) + { + xb_mysql_query(mysql_connection, "SET SESSION wsrep_sync_wait=0", false); + } + } + + ulonglong server_lsn_after_lock = get_current_lsn(mysql_connection); + + // Copy the rest of non-stats-lognon-InnoDB-Aria-RocksDB tables + // Do not execute BACKUP LOCK under BLOCK_DDL stage + if (!m_common_backup.scan(m_copied_common_tables, &m_copied_common_tables, + true, false)) { + msg("Error on scan data directory for common engines"); + return false; + } + // Copy log tables tail + if (!m_common_backup.copy_log_tables(false)) { + msg("Error on copy system tables"); + return false; + } + + // Copy the rest of non-stats Aria tables in background + if (!m_aria_backup.copy_offline_tables(nullptr, true, false)) { + msg("Error on start Aria tables backup"); + return false; + } + + // Copy .frm, .trn and other files + if (!backup_files(backup_datasinks.m_data, + fil_path_to_mysql_datadir)) { + msg("Backup files error"); + return false; + } + + msg("Waiting for log copy thread to read lsn %llu", + server_lsn_after_lock); + backup_wait_for_lsn(server_lsn_after_lock); + corrupted_pages.backup_fix_ddl(backup_datasinks.m_data, + backup_datasinks.m_meta); + + if (!m_aria_backup.copy_log_tail()) { + msg("Error on Aria log tail copy"); + return false; + } + + // Wait for Aria tables copy finish + if (!m_aria_backup.wait_for_finish()) { + msg("Aria data files backup process is finished with error"); + return false; + } + // Wait for non-InnoDB-Aria-RocksDB engines copy finish + if (!m_common_backup.wait_for_finish()) { + msg("Data files backup process is finished with error"); + return false; + } + + ddl_log::backup(fil_path_to_mysql_datadir, + backup_datasinks.m_data, m_tables); + + DBUG_MARIABACKUP_EVENT_LOCK("after_stage_block_ddl", {}); + + return true; + } + + bool stage_block_commit(Backup_datasinks &backup_datasinks) { + msg("BACKUP STAGE BLOCK_COMMIT"); + if (!opt_no_lock && !lock_for_backup_stage_commit(m_bs_con)) { + msg("Error on BACKUP STAGE BLOCK_COMMIT query execution"); + return false; + } + + // Copy log tables tail + if (!m_common_backup.copy_log_tables(true)) { + msg("Error on copy log tables"); + return false; + } + + // Copy stats tables + if (!m_common_backup.copy_stats_tables()) { + msg("Error on copy stats tables"); + return false; + } + + // Copy system Aria files + if (!m_aria_backup.finalize()) { + msg("Error on finalize Aria tables backup"); + return false; + } + + if (!m_common_backup.wait_for_finish()) { + msg("Error on finish common engines backup"); + return false; + } + + if (!m_common_backup.close_log_tables()) { + msg("Error on close log tables"); + return false; + } + + if (!backup_files_from_datadir(backup_datasinks.m_data, + fil_path_to_mysql_datadir, + "aws-kms-key")) { + msg("Error on root data dir files backup"); + return false; + } + + if (has_rocksdb_plugin()) { + rocksdb_create_checkpoint(); + } + + // There is no need to stop slave thread before coping non-Innodb data when + // --no-lock option is used because --no-lock option requires that no DDL or + // DML to non-transaction tables can occur. + if (opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + } + + if (opt_slave_info) { + if (!write_slave_info(backup_datasinks.m_data, + mysql_connection)) { + return(false); + } + } + + /* The only reason why Galera/binlog info is written before + wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup + binary will start streamig a temporary copy of REDO log to stdout and + thus, any streaming from innobackupex would interfere. The only way to + avoid that is to have a single process, i.e. merge innobackupex and + xtrabackup. */ + if (opt_galera_info) { + if (!write_galera_info(backup_datasinks.m_data, + mysql_connection)) { + return(false); + } + } + + bool with_binlogs = opt_binlog_info == BINLOG_INFO_ON; + + if (with_binlogs || opt_galera_info) { + if (!write_binlog_info(backup_datasinks.m_data, + mysql_connection)) { + return(false); + } + } + + if (!opt_no_lock) { + msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS..."); + xb_mysql_query(mysql_connection, + "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false); + } + + return backup_datasinks.backup_low(); + } + + bool stage_end(Backup_datasinks &backup_datasinks) { + msg("BACKUP STAGE END"); + /* release all locks */ + if (!opt_no_lock) { + unlock_all(m_bs_con); + history_lock_time = 0; + } else { + history_lock_time = time(NULL) - history_lock_time; + } + backup_release(); + DBUG_EXECUTE_IF("check_mdl_lock_works", + pthread_join(dbug_alter_thread, nullptr); + ); + + DBUG_EXECUTE_IF("emulate_ddl_on_intermediate_table", + pthread_join( + dbug_emulate_ddl_on_intermediate_table_thread, + nullptr); + ); + + backup_finish(backup_datasinks.m_data); + return true; + } + + void store_table_version( + std::string db, std::string table, std::string table_version) { + auto tk = table_key(db, table); + std::lock_guard<std::mutex> lock(m_tables_mutex); + m_tables[std::move(tk)] = std::move(table_version); + } + + private: + Backup_datasinks *backup_datasinks; + MYSQL *m_bs_con; + ThreadPool m_thread_pool; + std::vector<MYSQL *> m_con_pool; + std::mutex m_tables_mutex; + ddl_log::tables_t m_tables; + aria::Backup m_aria_backup; + common_engine::Backup m_common_backup; + std::unordered_set<table_key_t> m_copied_common_tables; +}; + /** Implement --backup @return whether the operation succeeded */ static bool xtrabackup_backup_func() { MY_STAT stat_info; - uint i; - uint count; - pthread_mutex_t count_mutex; CorruptedPages corrupted_pages; - data_thread_ctxt_t *data_threads; Backup_datasinks backup_datasinks; pthread_cond_init(&scanned_lsn_cond, NULL); @@ -4622,7 +5244,7 @@ static bool xtrabackup_backup_func() return(false); } msg("cd to %s", mysql_real_data_home); - xb_plugin_backup_init(mysql_connection); + encryption_plugin_backup_init(mysql_connection); msg("open files limit requested %lu, set to %lu", xb_open_files_limit, xb_set_max_open_files(xb_open_files_limit)); @@ -4663,22 +5285,6 @@ fail: return(false); } - if (srv_buf_pool_size >= 1000 * 1024 * 1024) { - /* Here we still have srv_pool_size counted - in kilobytes (in 4.0 this was in bytes) - srv_boot() converts the value to - pages; if buffer pool is less than 1000 MB, - assume fewer threads. */ - srv_max_n_threads = 50000; - - } else if (srv_buf_pool_size >= 8 * 1024 * 1024) { - - srv_max_n_threads = 10000; - } else { - srv_max_n_threads = 1000; /* saves several MB of memory, - especially in 64-bit - computers */ - } srv_thread_pool_init(); /* Reset the system variables in the recovery module. */ trx_pool_init(); @@ -4698,9 +5304,10 @@ fail: } /* get current checkpoint_lsn */ { + log_sys.latch.wr_lock(SRW_LOCK_CALL); mysql_mutex_lock(&recv_sys.mutex); - dberr_t err = recv_sys.find_checkpoint(); + log_sys.latch.wr_unlock(); if (err != DB_SUCCESS) { msg("Error: cannot read redo log header"); @@ -4796,11 +5403,6 @@ fail: std::thread(log_copying_thread).detach(); - /* FLUSH CHANGED_PAGE_BITMAPS call */ - if (!flush_changed_page_bitmaps()) { - goto fail; - } - ut_a(xtrabackup_parallel > 0); if (xtrabackup_parallel > 1) { @@ -4812,71 +5414,36 @@ fail: mdl_lock_all(); DBUG_EXECUTE_IF("check_mdl_lock_works", - dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int", + dbug_alter_thread = + dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int", "Waiting for table metadata lock", 0, 0);); } - datafiles_iter_t it; - - /* Create data copying threads */ - data_threads = (data_thread_ctxt_t *) - malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); - count = xtrabackup_parallel; - pthread_mutex_init(&count_mutex, NULL); - - for (i = 0; i < (uint) xtrabackup_parallel; i++) { - data_threads[i].it = ⁢ - data_threads[i].num = i+1; - data_threads[i].count = &count; - data_threads[i].count_mutex = &count_mutex; - data_threads[i].corrupted_pages = &corrupted_pages; - data_threads[i].datasinks= &backup_datasinks; - std::thread(data_copy_thread_func, data_threads + i).detach(); - } + BackupStages stages(backup_datasinks.m_data); - /* Wait for threads to exit */ - while (1) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - pthread_mutex_lock(&count_mutex); - bool stop = count == 0; - pthread_mutex_unlock(&count_mutex); - if (stop) { - break; - } - } - - pthread_mutex_destroy(&count_mutex); - free(data_threads); - - DBUG_ASSERT(backup_datasinks.m_data); - DBUG_ASSERT(backup_datasinks.m_meta); - bool ok = backup_start(backup_datasinks.m_data, - backup_datasinks.m_meta, corrupted_pages); + if (!stages.init()) + goto fail; - if (ok) { - ok = backup_datasinks.backup_low(); + if (!stages.stage_start(backup_datasinks, corrupted_pages)) + goto fail; - backup_release(); + if (!stages.stage_flush()) + goto fail; - DBUG_EXECUTE_IF("check_mdl_lock_works", - pthread_join(dbug_alter_thread, nullptr);); + if (!stages.stage_block_ddl(backup_datasinks, corrupted_pages)) + goto fail; - if (ok) { - backup_finish(backup_datasinks.m_data); - } - } + if (!stages.stage_block_commit(backup_datasinks)) + goto fail; - if (opt_log_innodb_page_corruption) - ok = corrupted_pages.print_to_file(backup_datasinks.m_data, - MB_CORRUPTED_PAGES_FILE); + if (!stages.stage_end(backup_datasinks)) + goto fail; - if (!ok) { + if (opt_log_innodb_page_corruption + && !corrupted_pages.print_to_file(backup_datasinks.m_data, + MB_CORRUPTED_PAGES_FILE)) goto fail; - } - if (changed_page_bitmap) { - xb_page_bitmap_deinit(changed_page_bitmap); - } backup_datasinks.destroy(); msg("Redo log (from LSN " LSN_PF " to " LSN_PF ") was copied.", @@ -4938,6 +5505,12 @@ void CorruptedPages::backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta) DBUG_MARIABACKUP_EVENT("backup_fix_ddl", {}); + DBUG_EXECUTE_IF("emulate_ddl_on_intermediate_table", + xb_mysql_query(mysql_connection, + "SET debug_sync='now SIGNAL backup_lock_reset " + "WAIT_FOR temp_table_dropped'", false, true); + ); + for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin(); iter != ddl_tracker.tables_in_backup.end(); iter++) { @@ -5082,6 +5655,7 @@ void CorruptedPages::backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta) } } + /* ================= prepare ================= */ /*********************************************************************** @@ -5568,7 +6142,7 @@ std::string change_extension(std::string filename, std::string new_ext) { } -static void rename_file(const char *from,const char *to) { +void rename_file(const char *from,const char *to) { msg("Renaming %s to %s\n", from, to); if (my_rename(from, to, MY_WME)) { die("Can't rename %s to %s errno %d", from, to, errno); @@ -5590,7 +6164,7 @@ typedef ibool (*handle_datadir_entry_func_t)( void* arg); /*!<in: caller-provided data */ /** Rename, and replace destination file, if exists */ -static void rename_force(const char *from, const char *to) { +void rename_force(const char *from, const char *to) { if (access(to, R_OK) == 0) { msg("Removing %s", to); if (my_delete(to, MYF(MY_WME))) { @@ -5884,7 +6458,7 @@ static std::string read_file_as_string(const std::string file) { } /** Delete file- Provide verbose diagnostics and exit, if operation fails. */ -static void delete_file(const std::string& file, bool if_exists = false) { +void delete_file(const std::string& file, bool if_exists = false) { if (if_exists && !file_exists(file)) return; if (my_delete(file.c_str(), MYF(MY_WME))) { @@ -6025,7 +6599,7 @@ static bool xtrabackup_prepare_func(char** argv) } int argc; for (argc = 0; argv[argc]; argc++) {} - xb_plugin_prepare_init(argc, argv, xtrabackup_incremental_dir); + encryption_plugin_prepare_init(argc, argv); xtrabackup_target_dir= mysql_data_home_buff; xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here @@ -6068,7 +6642,6 @@ static bool xtrabackup_prepare_func(char** argv) return(false); } - srv_max_n_threads = 1000; srv_n_purge_threads = 1; xb_filters_init(); @@ -6237,6 +6810,8 @@ error: if (ok && xtrabackup_export) ok= (prepare_export() == 0); + if (ok) ok = aria::prepare(xtrabackup_target_dir); + cleanup: xb_filters_free(); return ok && !ib::error::was_logged() && corrupted_pages.empty(); @@ -6374,7 +6949,7 @@ static bool check_all_privileges() int check_result = PRIVILEGE_OK; - /* FLUSH TABLES WITH READ LOCK */ + /* BACKUP LOCKS */ if (!opt_no_lock) { check_result |= check_privilege( @@ -6442,21 +7017,6 @@ xb_init() return(false); } - if (xtrabackup_backup && opt_rsync) - { - if (xtrabackup_stream_fmt) - { - msg("Error: --rsync doesn't work with --stream\n"); - return(false); - } - bool have_rsync = IF_WIN(false, (system("rsync --version > /dev/null 2>&1") == 0)); - if (!have_rsync) - { - msg("Error: rsync executable not found, cannot run backup with --rsync\n"); - return false; - } - } - n_mixed_options = 0; if (opt_decompress) { @@ -6493,6 +7053,7 @@ xb_init() if (opt_check_privileges && !check_all_privileges()) { return(false); } + history_start_time = time(NULL); } diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index d091c474..38d7e5fd 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -24,8 +24,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include <my_getopt.h> #include "datasink.h" #include "xbstream.h" -#include "changed_page_bitmap.h" +#include "fil0fil.h" #include <set> +#include "handler.h" + +#include <utility> +#include <vector> +#include <tuple> +#include <functional> + #define XB_TOOL_NAME "mariadb-backup" #define XB_HISTORY_TABLE "mysql.mariadb_backup_history" @@ -84,8 +91,6 @@ extern my_bool xb_backup_rocksdb; extern uint opt_protocol; -extern xb_page_bitmap *changed_page_bitmap; - extern char *xtrabackup_incremental; extern my_bool xtrabackup_incremental_force_scan; @@ -112,7 +117,7 @@ extern my_bool xtrabackup_decrypt_decompress; extern char *innobase_data_file_path; extern longlong innobase_page_size; -extern int xtrabackup_parallel; +extern uint xtrabackup_parallel; extern my_bool xb_close_files; extern const char *xtrabackup_compress_alg; @@ -131,7 +136,6 @@ extern my_bool opt_galera_info; extern my_bool opt_slave_info; extern my_bool opt_no_lock; extern my_bool opt_safe_slave_backup; -extern my_bool opt_rsync; extern my_bool opt_force_non_empty_dirs; extern my_bool opt_noversioncheck; extern my_bool opt_no_backup_locks; @@ -288,15 +292,40 @@ fil_file_readdir_next_file( os_file_stat_t* info); /*!< in/out: buffer where the info is returned */ -#ifndef DBUG_OFF -#include <fil0fil.h> -extern void dbug_mariabackup_event(const char *event, - const fil_space_t::name_type key); +const char *convert_dst(const char *dst); -#define DBUG_MARIABACKUP_EVENT(A, B) \ - DBUG_EXECUTE_IF("mariabackup_events", dbug_mariabackup_event(A, B);) -#else -#define DBUG_MARIABACKUP_EVENT(A, B) /* empty */ -#endif // DBUG_OFF +std::string get_table_version_from_image(const std::vector<uchar> &frm_image); +std::pair<bool, legacy_db_type> + get_table_engine_from_image(const std::vector<uchar> &frm_image); +std::string read_table_version_id(File file); + +std::string convert_tablename_to_filepath( + const char *data_dir_path, const std::string &db, const std::string &table); + +std::tuple<std::string, std::string, std::string> +convert_filepath_to_tablename(const char *filepath); + +typedef std::string table_key_t; + +inline table_key_t table_key(const std::string &db, const std::string &table) { + return std::string(db).append(".").append(table); +}; + +inline table_key_t table_key(const char *db, const char *table) { + return std::string(db).append(".").append(table); +}; + +typedef std::function<void(std::string, std::string, std::string)> + post_copy_table_hook_t; + +my_bool +check_if_skip_table( +/******************/ + const char* name); /*!< in: path to the table */ + +bool is_log_table(const char *dbname, const char *tablename); +bool is_stats_table(const char *dbname, const char *tablename); +extern my_bool xtrabackup_copy_back; +extern my_bool xtrabackup_move_back; #endif /* XB_XTRABACKUP_H */ |