diff options
Diffstat (limited to 'storage/rocksdb/mysql-test/rocksdb_hotbackup/include')
13 files changed, 271 insertions, 0 deletions
diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/clean_tmpfiles.sh b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/clean_tmpfiles.sh new file mode 100755 index 00000000..98a1fecc --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/clean_tmpfiles.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +COPY_LOG="${MYSQL_TMP_DIR}/myrocks_hotbackup_copy_log" +SIGNAL_FILE=${MYSQL_TMP_DIR}/myrocks_hotbackup_signal +MOVEBACK_LOG="${MYSQL_TMP_DIR}/myrocks_hotbackup_moveback_log" +rm -f $COPY_LOG +rm -f $SIGNAL_FILE +rm -f $MOVEBACK_LOG diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/cleanup.inc b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/cleanup.inc new file mode 100644 index 00000000..947bf027 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/cleanup.inc @@ -0,0 +1,3 @@ + +--source include/rpl_end.inc + diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/create_slocket_socket.sh b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/create_slocket_socket.sh new file mode 100755 index 00000000..db470f52 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/create_slocket_socket.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +src_data_dir="${MYSQLTEST_VARDIR}/mysqld.1/data/" +python -c "import socket as s; sock = s.socket(s.AF_UNIX); sock.bind('${src_data_dir}/slocket')" diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/create_table.sh b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/create_table.sh new file mode 100755 index 00000000..2004caca --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/create_table.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +COPY_LOG=$1 +SIGNAL_FILE=$2 +# Creating a table after myrocks_hotbackup reaches waiting loop + +done=0 +while : ; do + wait=`tail -1 $COPY_LOG | grep 'Waiting until' | wc -l` + if [ "$wait" -eq "1" ]; then + break + fi + sleep 1 +done +$MYSQL --defaults-group-suffix=.1 db1 -e "create table r10 (id int primary key ) engine=rocksdb" +touch $SIGNAL_FILE diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/load_data.sh b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/load_data.sh new file mode 100755 index 00000000..80f1a5e2 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/load_data.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +# Insert 100 batches of 100 records each to a table with following schema: +# create table db1.t1 ( +# `id` int(10) not null auto_increment, +# `k` int(10), +# `data` varchar(2048), +# primary key (`id`), +# key (`k`) +# ) engine=innodb; + +MAX_INSERTS=100 +MAX_ROWS_PER_INSERT=100 + +insertData() { + for ((i=1; i<=$MAX_INSERTS; i++)); + do + stmt='INSERT INTO db1.t1 values' + for ((j=1; j<=$MAX_ROWS_PER_INSERT; j++)); + do + k=$RANDOM + data=$(head -c 2048 /dev/urandom|tr -cd 'a-zA-Z0-9') + stmt=$stmt' (NULL, '$k', "'$data'")' + if [ $j -lt $MAX_ROWS_PER_INSERT ]; then + stmt=$stmt',' + fi + done + stmt=$stmt';' + $MYSQL --defaults-group-suffix=.1 -e "$stmt" + done +} + +NUM_PARALLEL_INSERTS=25 +pids=() +for ((k=1; k<=$NUM_PARALLEL_INSERTS; k++)); +do + insertData & + pids+=($!) +done +for ((k=1; k<=$NUM_PARALLEL_INSERTS; k++)); +do + wait ${pids[k]} +done diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/load_data_and_run.sh b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/load_data_and_run.sh new file mode 100755 index 00000000..a8e6fc44 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/load_data_and_run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +# Initially loads a chunk of data. +# Then start loading another chunk of data, +# while simultaneously running a backup + +suite/rocksdb_hotbackup/include/load_data.sh 2>&1 +suite/rocksdb_hotbackup/include/load_data.sh 2>&1 & +suite/rocksdb_hotbackup/include/stream_run.sh 2>&1 diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/load_data_slocket.sh b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/load_data_slocket.sh new file mode 100755 index 00000000..036d6866 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/load_data_slocket.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +# Insert 10 batches of 10 records each to a table with following schema: +# create table slocket.t1 ( +# `id` int(10) not null auto_increment, +# `k` int(10), +# `data` varchar(2048), +# primary key (`id`), +# key (`k`) +# ) engine=innodb; + +MAX_INSERTS=10 +MAX_ROWS_PER_INSERT=10 + +insertData() { + for ((i=1; i<=$MAX_INSERTS; i++)); + do + stmt='INSERT INTO slocket.t1 values' + for ((j=1; j<=$MAX_ROWS_PER_INSERT; j++)); + do + k=$RANDOM + data=$(head -c 2048 /dev/urandom|tr -cd 'a-zA-Z0-9') + stmt=$stmt' (NULL, '$k', "'$data'")' + if [ $j -lt $MAX_ROWS_PER_INSERT ]; then + stmt=$stmt',' + fi + done + stmt=$stmt';' + $MYSQL --defaults-group-suffix=.1 -e "$stmt" + done +} + +NUM_PARALLEL_INSERTS=25 +pids=() +for ((k=1; k<=$NUM_PARALLEL_INSERTS; k++)); +do + insertData & + pids+=($!) +done +for ((k=1; k<=$NUM_PARALLEL_INSERTS; k++)); +do + wait ${pids[k]} +done diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/remove_slocket_socket.sh b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/remove_slocket_socket.sh new file mode 100755 index 00000000..9114629b --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/remove_slocket_socket.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +src_data_dir="${MYSQLTEST_VARDIR}/mysqld.1/data/" +rm "${src_data_dir}/slocket" diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup.inc b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup.inc new file mode 100644 index 00000000..26c3f2ce --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup.inc @@ -0,0 +1,16 @@ +--let $rpl_server_count= 2 +--let $rpl_topology= none +--source include/rpl_init.inc +--source include/rpl_default_connections.inc + +connection server_1; +create database db1; + +create table db1.t1 ( + `id` int(10) not null auto_increment, + `k` int(10), + `data` varchar(2048), + primary key (`id`), + key (`k`) +) engine=rocksdb; + diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup_replication_gtid.sh b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup_replication_gtid.sh new file mode 100755 index 00000000..3c95068a --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup_replication_gtid.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +binlog_line=($(grep -o "Last binlog file position [0-9]*, file name .*\.[0-9]*" ${MYSQLTEST_VARDIR}/log/mysqld.2.err | tail -1)) +binlog_pos=${binlog_line[4]%?} +binlog_file=${binlog_line[7]} + +sql="show gtid_executed in '$binlog_file' from $binlog_pos" +result=($($MYSQL --defaults-group-suffix=.1 -e "$sql")) +gtid_executed=${result[1]} + +sql="reset master;" +sql="$sql reset slave;" +sql="$sql change master to master_host='127.0.0.1', master_port=${MASTER_MYPORT}, master_user='root', master_auto_position=1, master_connect_retry=1;" +sql="$sql set global gtid_purged='$gtid_executed';" +sql="$sql start slave;" +sql="$sql stop slave;" +sql="$sql change master to master_auto_position=0;" +sql="$sql start slave;" +$MYSQL --defaults-group-suffix=.2 -e "$sql" +echo "$sql" > ${MYSQL_TMP_DIR}/gtid_stmt diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup_replication_gtid_and_sync.inc b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup_replication_gtid_and_sync.inc new file mode 100644 index 00000000..75dc3196 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup_replication_gtid_and_sync.inc @@ -0,0 +1,4 @@ +--exec suite/rocksdb_hotbackup/include/setup_replication_gtid.sh + +let $slave_sync_timeout = 1800; +source include/wait_for_slave_to_sync_with_master.inc; diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup_slocket.inc b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup_slocket.inc new file mode 100644 index 00000000..ce889164 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/setup_slocket.inc @@ -0,0 +1,10 @@ +connection server_1; +create database slocket; + +create table slocket.t1 ( + `id` int(10) not null auto_increment, + `k` int(10), + `data` varchar(2048), + primary key (`id`), + key (`k`) +) engine=rocksdb; diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/stream_run.sh b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/stream_run.sh new file mode 100755 index 00000000..6108cfbb --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_hotbackup/include/stream_run.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +. suite/rocksdb_hotbackup/include/clean_tmpfiles.sh + +if [ "$STREAM_TYPE" == 'wdt' ]; then + which wdt >/dev/null 2>&1 + if [ $? -ne 0 ]; then + # change to tar if wdt is not installed + STREAM_TYPE='tar' + fi +fi + +set -e +set -o pipefail + +# Takes a full backup from server_1 to server_2 +# using myrocks_hotbackup streaming + +checkpoint_dir="${MYSQLTEST_VARDIR}/checkpoint" +backup_dir="${MYSQLTEST_VARDIR}/backup" +dest_data_dir="${MYSQLTEST_VARDIR}/mysqld.2/data/" + +mysql_dir=$(echo $MYSQL | awk '{print $1}' | xargs dirname) +PATH=$mysql_dir:$PATH + +mkdir -p $checkpoint_dir +rm -rf $checkpoint_dir/* + +mkdir -p $backup_dir +rm -rf $backup_dir/* +# delete and recreate the dest dir to make sure all hidden files +# and directories (such as .rocksdb) are blown away +rm -rf $dest_data_dir/ +mkdir $dest_data_dir + + +SIGNAL_CONDITION="" + +if [ "$FRM" == '1' ]; then + suite/rocksdb_hotbackup/include/create_table.sh $COPY_LOG $SIGNAL_FILE 2>&1 & +fi + +if [ "$DEBUG_SIGNAL" == '1' ]; then + SIGNAL_CONDITION="--debug_signal_file=$SIGNAL_FILE" +fi + +if [ "$STREAM_TYPE" == 'tar' ]; then + BACKUP_CMD="$MYSQL_MYROCKS_HOTBACKUP --user='root' --port=${MASTER_MYPORT} \ + --stream=tar --checkpoint_dir=$checkpoint_dir $SIGNAL_CONDITION 2> \ + $COPY_LOG | tar -xi -C $backup_dir" +elif [ "$STREAM_TYPE" == 'wdt' ]; then + BACKUP_CMD="$MYSQL_MYROCKS_HOTBACKUP --user='root' --stream=wdt \ + --port=${MASTER_MYPORT} --destination=localhost --backup_dir=$backup_dir \ + --avg_mbytes_per_sec=10 --interval=5 $SIGNAL_CONDITION \ + --extra_wdt_sender_options='--block_size_mbytes=1' \ + --checkpoint_dir=$checkpoint_dir 2> \ + $COPY_LOG" +elif [ "$STREAM_TYPE" == "xbstream_socket" ]; then + BACKUP_CMD="$MYSQL_MYROCKS_HOTBACKUP --user='root' --socket=${MASTER_MYSOCK} \ + --stream=xbstream --checkpoint_dir=$checkpoint_dir $SIGNAL_CONDITION 2> \ + $COPY_LOG | xbstream -x \ + --directory=$backup_dir" +else + BACKUP_CMD="$MYSQL_MYROCKS_HOTBACKUP --user='root' --port=${MASTER_MYPORT} \ + --stream=xbstream --checkpoint_dir=$checkpoint_dir $SIGNAL_CONDITION 2> \ + $COPY_LOG | xbstream -x \ + --directory=$backup_dir" +fi + +echo "myrocks_hotbackup copy phase" +eval "$BACKUP_CMD" + +mkdir ${backup_dir}/test # TODO: Fix skipping empty directories + + +echo "myrocks_hotbackup move-back phase" +$MYSQL_MYROCKS_HOTBACKUP --move_back --datadir=$dest_data_dir \ + --rocksdb_datadir=$dest_data_dir/\#rocksdb \ + --rocksdb_waldir=$dest_data_dir/\#rocksdb \ + --backup_dir=$backup_dir > $MOVEBACK_LOG 2>&1 + |