summaryrefslogtreecommitdiffstats
path: root/storage/tokudb/PerconaFT/src/tests/recover-update-multiple.cc
diff options
context:
space:
mode:
Diffstat (limited to 'storage/tokudb/PerconaFT/src/tests/recover-update-multiple.cc')
-rw-r--r--storage/tokudb/PerconaFT/src/tests/recover-update-multiple.cc507
1 files changed, 507 insertions, 0 deletions
diff --git a/storage/tokudb/PerconaFT/src/tests/recover-update-multiple.cc b/storage/tokudb/PerconaFT/src/tests/recover-update-multiple.cc
new file mode 100644
index 00000000..f2f5d09f
--- /dev/null
+++ b/storage/tokudb/PerconaFT/src/tests/recover-update-multiple.cc
@@ -0,0 +1,507 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License, version 3,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+#include "test.h"
+
+// verify recovery of some update multiple operations
+
+static const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE;
+
+static int
+get_num_new_keys(int i, int dbnum) {
+ if (dbnum == 0) return 1;
+ if (i & (1<<4)) {
+ dbnum++; // Shift every once in a while.
+ }
+ return (i + dbnum) % 3; // 0, 1, or 2
+}
+
+static int
+get_num_keys(int i, int dbnum) {
+ if (dbnum == 0) return 1;
+ return (i + dbnum) % 3; // 0, 1, or 2
+}
+
+static int
+get_total_secondary_rows(int num_primary) {
+ assert(num_primary % 3 == 0);
+ return num_primary / 3 * (0 + 1 + 2);
+}
+
+static int
+get_total_num_keys(int i, int num_dbs) {
+ int sum = 0;
+ for (int db = 1; db < num_dbs; ++db) {
+ sum += get_num_keys(i, db);
+ }
+ return sum;
+}
+
+static int
+get_total_num_new_keys(int i, int num_dbs) {
+ int sum = 0;
+ for (int db = 1; db < num_dbs; ++db) {
+ sum += get_num_new_keys(i, db);
+ }
+ return sum;
+}
+
+static int
+get_key(int i, int dbnum, int which) {
+ assert(i < INT16_MAX / 2);
+ assert(which >= 0);
+ assert(which < get_num_keys(i, dbnum));
+ assert(which < 4);
+ assert(dbnum < 16);
+ if (dbnum == 0) {
+ assert(which == 0);
+ return htonl((2*i) << 16);
+ } else {
+ return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1));
+ }
+}
+
+static int
+get_new_key(int i, int dbnum, int which) {
+ assert(which >= 0);
+ assert(which < get_num_new_keys(i, dbnum));
+ assert(which < 4);
+ assert(dbnum < 16);
+
+ if (dbnum == 0) {
+ assert(which == 0);
+ return htonl((2*i+1) << 16);
+ } else if ((i+dbnum+which) & (1<<5)) {
+ return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1)); // no change from original
+ } else {
+ return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1) + 1);
+ }
+}
+
+static void
+get_data(int *v, int i, int ndbs) {
+ int index = 0;
+ for (int dbnum = 1; dbnum < ndbs; dbnum++) {
+ for (int which = 0; which < get_num_keys(i, dbnum); ++which) {
+ v[index++] = get_key(i, dbnum, which);
+ }
+ }
+}
+
+static void
+get_new_data(int *v, int i, int ndbs) {
+ int index = 0;
+ for (int dbnum = 1; dbnum < ndbs; dbnum++) {
+ for (int which = 0; which < get_num_new_keys(i, dbnum); ++which) {
+ v[index++] = get_new_key(i, dbnum, which);
+ if (which > 0) {
+ assert(index >= 2);
+ assert(memcmp(&v[index-2], &v[index-1], sizeof(v[0])) < 0);
+ }
+ }
+ }
+}
+
+
+static int
+put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
+ (void)src_val;
+ assert(src_db != dest_db);
+ assert(src_db);
+ int dbnum;
+ assert(dest_db->descriptor->dbt.size == sizeof dbnum);
+ memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
+ assert(dbnum > 0);
+
+ int pri_key = *(int *) src_key->data;
+ int* pri_val = (int*) src_val->data;
+
+ bool is_new = (ntohl(pri_key) >> 16) % 2 == 1;
+ int i = (ntohl(pri_key) >> 16) / 2;
+
+ int num_keys = is_new ? get_num_new_keys(i, dbnum) : get_num_keys(i, dbnum);
+
+ toku_dbt_array_resize(dest_key_arrays, num_keys);
+
+ if (dest_val_arrays) {
+ toku_dbt_array_resize(dest_val_arrays, num_keys);
+ }
+
+ int index = 0;
+
+ for (int idb = 1; idb < dbnum; idb++) {
+ index += is_new ? get_num_new_keys(i, idb) : get_num_keys(i, idb);
+ }
+ assert(src_val->size % sizeof(int) == 0);
+ assert((int)src_val->size / 4 >= index + num_keys);
+
+ for (int which = 0; which < num_keys; which++) {
+ DBT *dest_key = &dest_key_arrays->dbts[which];
+ DBT *dest_val = NULL;
+
+ assert(dest_key->flags == DB_DBT_REALLOC);
+ if (dest_key->ulen < sizeof(int)) {
+ dest_key->data = toku_xrealloc(dest_key->data, sizeof(int));
+ dest_key->ulen = sizeof(int);
+ }
+ dest_key->size = sizeof(int);
+ if (dest_val_arrays) {
+ dest_val = &dest_val_arrays->dbts[which];
+ assert(dest_val->flags == DB_DBT_REALLOC);
+ dest_val->size = 0;
+ }
+ int new_key = is_new ? get_new_key(i, dbnum, which) : get_key(i, dbnum, which);
+ assert(new_key == pri_val[index + which]);
+ *(int*)dest_key->data = new_key;
+ }
+ return 0;
+}
+
+static int
+del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT *src_data) {
+ return put_callback(dest_db, src_db, dest_key_arrays, NULL, src_key, src_data);
+}
+
+static void
+update_diagonal(DB_ENV *env, DB_TXN *txn, DB *db[], int ndbs, int nrows) {
+ assert(ndbs > 0);
+ int r;
+
+ int narrays = 2 * ndbs;
+ DBT_ARRAY keys[narrays];
+ DBT_ARRAY vals[narrays];
+ for (int i = 0; i < narrays; i++) {
+ toku_dbt_array_init(&keys[i], 1);
+ toku_dbt_array_init(&vals[i], 1);
+ }
+
+ for (int i = 0; i < nrows; i++) {
+
+ // update the data i % ndbs col from x to x+1
+
+ int old_k = get_key(i, 0, 0);
+ DBT old_key; dbt_init(&old_key, &old_k, sizeof old_k);
+ int new_k = get_new_key(i, 0, 0);
+ DBT new_key; dbt_init(&new_key, &new_k, sizeof new_k);
+
+ int num_old_keys = get_total_num_keys(i, ndbs);
+ int v[num_old_keys]; get_data(v, i, ndbs);
+ DBT old_data; dbt_init(&old_data, &v[0], sizeof v);
+
+ int num_new_keys = get_total_num_new_keys(i, ndbs);
+ int newv[num_new_keys]; get_new_data(newv, i, ndbs);
+ DBT new_data; dbt_init(&new_data, &newv[0], sizeof newv);
+
+ uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array));
+
+ r = env->update_multiple(env, db[0], txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, narrays, keys, narrays, vals);
+ assert_zero(r);
+ }
+ for (int i = 0; i < narrays; i++) {
+ toku_dbt_array_destroy(&keys[i]);
+ toku_dbt_array_destroy(&vals[i]);
+ }
+
+}
+
+static void
+populate_primary(DB_ENV *env, DB *db, int ndbs, int nrows) {
+ int r;
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+
+ // populate
+ for (int i = 0; i < nrows; i++) {
+ int k = get_key(i, 0, 0);
+ int secondary_keys = get_total_num_keys(i, ndbs);
+ int v[secondary_keys]; get_data(v, i, ndbs);
+ DBT key; dbt_init(&key, &k, sizeof k);
+ DBT val; dbt_init(&val, &v[0], sizeof v);
+ r = db->put(db, txn, &key, &val, 0); assert_zero(r);
+ }
+
+ r = txn->commit(txn, 0); assert_zero(r);
+}
+
+static void
+populate_secondary(DB_ENV *env, DB *db, int dbnum, int nrows) {
+ int r;
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+
+ // populate
+ for (int i = 0; i < nrows; i++) {
+ for (int which = 0; which < get_num_keys(i, dbnum); which++) {
+ int k = get_key(i, dbnum, which);
+ DBT key; dbt_init(&key, &k, sizeof k);
+ DBT val; dbt_init(&val, NULL, 0);
+ r = db->put(db, txn, &key, &val, 0); assert_zero(r);
+ }
+ }
+
+ r = txn->commit(txn, 0); assert_zero(r);
+}
+
+static void
+verify_pri_seq(DB_ENV *env, DB *db, int ndbs, int nrows) {
+ const int dbnum = 0;
+ int r;
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+
+ DBC *cursor = NULL;
+ r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
+ int i;
+ for (i = 0; ; i++) {
+ DBT key; memset(&key, 0, sizeof key);
+ DBT val; memset(&val, 0, sizeof val);
+ r = cursor->c_get(cursor, &key, &val, DB_NEXT);
+ if (r != 0)
+ break;
+ int k;
+ int expectk = get_new_key(i, dbnum, 0);
+
+ assert(key.size == sizeof k);
+ memcpy(&k, key.data, key.size);
+ assert(k == expectk);
+
+ int num_keys = get_total_num_new_keys(i, ndbs);
+ assert(val.size == num_keys*sizeof(int));
+ int v[num_keys]; get_new_data(v, i, ndbs);
+ assert(memcmp(val.data, v, val.size) == 0);
+ }
+ assert(i == nrows); // if (i != nrows) printf("%s:%d %d %d\n", __FUNCTION__, __LINE__, i, nrows); // assert(i == nrows);
+ r = cursor->c_close(cursor); assert_zero(r);
+ r = txn->commit(txn, 0); assert_zero(r);
+}
+
+static void
+verify_sec_seq(DB_ENV *env, DB *db, int dbnum, int nrows) {
+ assert(dbnum > 0);
+ int r;
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+
+ DBC *cursor = NULL;
+ r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
+ int i;
+ int rows_found = 0;
+
+ for (i = 0; ; i++) {
+ int num_keys = get_num_new_keys(i, dbnum);
+ for (int which = 0; which < num_keys; ++which) {
+ DBT key; memset(&key, 0, sizeof key);
+ DBT val; memset(&val, 0, sizeof val);
+ r = cursor->c_get(cursor, &key, &val, DB_NEXT);
+ if (r != 0) {
+ CKERR2(r, DB_NOTFOUND);
+ goto done;
+ }
+ rows_found++;
+ int k;
+ int expectk = get_new_key(i, dbnum, which);
+
+ assert(key.size == sizeof k);
+ memcpy(&k, key.data, key.size);
+ int got_i = (ntohl(k) >> 16) / 2;
+ if (got_i < i) {
+ // Will fail. Too many old i's
+ assert(k == expectk);
+ } else if (got_i > i) {
+ // Will fail. Too few in previous i.
+ assert(k == expectk);
+ }
+
+ if (k != expectk && which < get_num_keys(i, dbnum) && k == get_key(i, dbnum, which)) {
+ // Will fail, never got updated.
+ assert(k == expectk);
+ }
+ assert(k == expectk);
+ assert(val.size == 0);
+ }
+ }
+done:
+ assert(rows_found == get_total_secondary_rows(nrows));
+ r = cursor->c_close(cursor); assert_zero(r);
+ r = txn->commit(txn, 0); assert_zero(r);
+}
+
+static void
+run_test(int ndbs, int nrows) {
+ int r;
+ toku_os_recursive_delete(TOKU_TEST_FILENAME);
+ r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+ DB_ENV *env;
+ r = db_env_create(&env, 0); assert_zero(r);
+ r = env->set_generate_row_callback_for_put(env, put_callback); assert_zero(r);
+ r = env->set_generate_row_callback_for_del(env, del_callback); assert_zero(r);
+ r = env->open(env, TOKU_TEST_FILENAME, envflags, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+ DB *db[ndbs];
+ for (int dbnum = 0; dbnum < ndbs; dbnum++) {
+ r = db_create(&db[dbnum], env, 0);
+ assert_zero(r);
+ DBT dbt_dbnum; dbt_init(&dbt_dbnum, &dbnum, sizeof dbnum);
+ assert_zero(r);
+ char dbname[32]; sprintf(dbname, "%d.tdb", dbnum);
+ r = db[dbnum]->open(db[dbnum], NULL, dbname, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666);
+ assert_zero(r);
+ IN_TXN_COMMIT(env, NULL, txn_desc, 0, {
+ { int chk_r = db[dbnum]->change_descriptor(db[dbnum], txn_desc, &dbt_dbnum, 0); CKERR(chk_r); }
+ });
+ }
+
+ r = env->txn_checkpoint(env, 0, 0, 0); assert_zero(r);
+
+ for (int dbnum = 0; dbnum < ndbs; dbnum++) {
+ if (dbnum == 0)
+ populate_primary(env, db[dbnum], ndbs, nrows);
+ else
+ populate_secondary(env, db[dbnum], dbnum, nrows);
+ }
+
+ r = env->txn_checkpoint(env, 0, 0, 0); assert_zero(r);
+
+ // update multiple key0
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+
+ update_diagonal(env, txn, db, ndbs, nrows);
+
+ r = txn->commit(txn, 0); assert_zero(r);
+ for (int dbnum = 0; dbnum < ndbs; dbnum++) {
+ if (dbnum == 0) {
+ verify_pri_seq(env, db[0], ndbs, nrows);
+ } else {
+ verify_sec_seq(env, db[dbnum], dbnum, nrows);
+ }
+ }
+
+ toku_hard_crash_on_purpose();
+}
+
+
+static void
+verify_all(DB_ENV *env, int ndbs, int nrows) {
+ int r;
+ for (int dbnum = 0; dbnum < ndbs; dbnum++) {
+ DB *db = NULL;
+ r = db_create(&db, env, 0);
+ assert_zero(r);
+ char dbname[32]; sprintf(dbname, "%d.tdb", dbnum);
+ r = db->open(db, NULL, dbname, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666);
+ assert_zero(r);
+ if (dbnum == 0) {
+ verify_pri_seq(env, db, ndbs, nrows);
+ } else {
+ verify_sec_seq(env, db, dbnum, nrows);
+ }
+ r = db->close(db, 0);
+ assert_zero(r);
+ }
+}
+
+static void
+run_recover(int ndbs, int nrows) {
+ int r;
+
+ DB_ENV *env;
+ r = db_env_create(&env, 0); assert_zero(r);
+ r = env->set_generate_row_callback_for_put(env, put_callback); assert_zero(r);
+ r = env->set_generate_row_callback_for_del(env, del_callback); assert_zero(r);
+ r = env->open(env, TOKU_TEST_FILENAME, envflags|DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+ verify_all(env, ndbs, nrows);
+ r = env->close(env, 0); assert_zero(r);
+}
+
+static int
+usage(void) {
+ return 1;
+}
+
+int
+test_main (int argc, char * const argv[]) {
+ bool do_test = false;
+ bool do_recover = false;
+ int ndbs = 2;
+ int nrows = 3*(1<<5)*4;
+
+ for (int i = 1; i < argc; i++) {
+ char * const arg = argv[i];
+ if (strcmp(arg, "-v") == 0) {
+ verbose++;
+ continue;
+ }
+ if (strcmp(arg, "-q") == 0) {
+ verbose--;
+ if (verbose < 0)
+ verbose = 0;
+ continue;
+ }
+ if (strcmp(arg, "--test") == 0) {
+ do_test = true;
+ continue;
+ }
+ if (strcmp(arg, "--recover") == 0) {
+ do_recover = true;
+ continue;
+ }
+ if (strcmp(arg, "--ndbs") == 0 && i+1 < argc) {
+ ndbs = atoi(argv[++i]);
+ continue;
+ }
+ if (strcmp(arg, "--nrows") == 0 && i+1 < argc) {
+ nrows = atoi(argv[++i]);
+ continue;
+ }
+ if (strcmp(arg, "--help") == 0) {
+ return usage();
+ }
+ }
+ while (nrows % (3*(1<<5)) != 0) {
+ nrows++;
+ }
+
+ if (do_test)
+ run_test(ndbs, nrows);
+ if (do_recover)
+ run_recover(ndbs, nrows);
+
+ return 0;
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-20 13:26:06 +0000