From 2cda63bc01c725f8b230dbe8a6fb6ef922fb8664 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Tue, 16 Apr 2013 23:57:59 -0400 Subject: [PATCH] merge -c 14118 tokudb.1960 to main [t:1960] git-svn-id: file:///svn/toku/tokudb@14119 c7de825b-a66e-492c-adef-691d508d4ae1 --- newbrt/recover.c | 17 ++- newbrt/varray.c | 7 +- newbrt/varray.h | 3 + src/tests/x1-nested2.c | 241 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 262 insertions(+), 6 deletions(-) create mode 100644 src/tests/x1-nested2.c diff --git a/newbrt/recover.c b/newbrt/recover.c index aeee7c5f5f3..1e5b143b02f 100644 --- a/newbrt/recover.c +++ b/newbrt/recover.c @@ -410,12 +410,13 @@ static int toku_recover_backward_xstillopen (struct logtype_xstillopen *l, struc static int toku_recover_xbegin (LSN lsn, TXNID parent_xid) { int r; - // TODO do something with the parent - assert(parent_xid == 0); + // lookup the parent TOKUTXN parent = NULL; + r = toku_txnid2txn(recover_logger, parent_xid, &parent); + assert(r == 0); // create a transaction and bind it to the transaction id - TOKUTXN txn; + TOKUTXN txn = NULL; r = toku_txn_begin_with_xid(parent, &txn, recover_logger, lsn.lsn); assert(r == 0); return 0; @@ -538,6 +539,13 @@ static int tokudb_needs_recovery(const char *log_dir) { return needs_recovery; } +// Sort the transactions in reverse order of transaction id +static int compare_txn(const void *a, const void *b) { + TOKUTXN atxn = (TOKUTXN) * (void **) a; + TOKUTXN btxn = (TOKUTXN) * (void **) b; + return - (atxn->txnid64 - btxn->txnid64); +} + // Append a transaction to the set of live transactions static int append_live_txn(OMTVALUE v, u_int32_t UU(i), void *extra) { TOKUTXN txn = (TOKUTXN) v; @@ -618,11 +626,12 @@ static int really_do_recovery(const char *data_dir, const char *log_dir) { // restart logging toku_logger_restart(recover_logger, lastlsn); - // abort all of the remaining live transactions + // abort all of the remaining live transactions in reverse transaction id order struct varray *live_txns = NULL; r = varray_create(&live_txns, 1); assert(r == 0); toku_omt_iterate(recover_logger->live_txns, append_live_txn, live_txns); + varray_sort(live_txns, compare_txn); varray_iterate(live_txns, abort_live_txn, NULL); varray_destroy(&live_txns); diff --git a/newbrt/varray.c b/newbrt/varray.c index ecde107eb01..a32e2210b86 100644 --- a/newbrt/varray.c +++ b/newbrt/varray.c @@ -50,9 +50,12 @@ int varray_append(struct varray *va, void *p) { va->a[va->c++] = p; return 0; } + +void varray_sort(struct varray *va, int (*compare)(const void *a, const void *b)) { + qsort(va->a, va->c, sizeof (void *), compare); +} void varray_iterate(struct varray *va, void (*f)(void *p, void *extra), void *extra) { - int i; - for (i=0; ic; i++) + for (int i = 0; i < va->c; i++) f(va->a[i], extra); } diff --git a/newbrt/varray.h b/newbrt/varray.h index 5ebae24df59..4954777b8e8 100644 --- a/newbrt/varray.h +++ b/newbrt/varray.h @@ -22,6 +22,9 @@ void varray_destroy(struct varray **vap); // element is the new element // Returns: 0 if success int varray_append(struct varray *va, void *p); + +// Sort the array elements +void varray_sort(struct varray *va, int (*)(const void *, const void *)); // Apply a function to all of the elements in the array // Effect: The function f is called for every element in the array, with diff --git a/src/tests/x1-nested2.c b/src/tests/x1-nested2.c new file mode 100644 index 00000000000..c2cbb2afd99 --- /dev/null +++ b/src/tests/x1-nested2.c @@ -0,0 +1,241 @@ +/* Transaction consistency: + * fork a process: + * Open two tables, T1 and T2 + * begin transaction + * store A in T1 + * checkpoint + * store B in T2 + * commit (or abort) + * signal to end the process abruptly + * wait for the process to finish + * open the environment doing recovery + * check to see if both A and B are present (or absent) + */ +#include +#include +#include "test.h" + +const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN |DB_RECOVER; +char *namea="a.db"; +char *nameb="b.db"; + + +static void +do_x1_shutdown (BOOL do_commit, BOOL do_abort) { + int r; + system("rm -rf " ENVDIR); + toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); + DB_ENV *env; + DB *dba, *dbb; + r = db_env_create(&env, 0); CKERR(r); + r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); + r = db_create(&dba, env, 0); CKERR(r); + r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); + r = db_create(&dbb, env, 0); CKERR(r); + r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); + DB_TXN *txn0; + r = env->txn_begin(env, NULL, &txn0, 0); CKERR(r); + DB_TXN *txn; + r = env->txn_begin(env, txn0, &txn, 0); CKERR(r); + { + DBT a={.data="a", .size=2}; + DBT b={.data="b", .size=2}; + r = dba->put(dba, txn, &a, &b, 0); CKERR(r); + r = env->txn_checkpoint(env, 0, 0, 0); CKERR(r); + r = dbb->put(dbb, txn, &b, &a, 0); CKERR(r); + } + //printf("opened\n"); + if (do_commit) { + r = txn->commit(txn, 0); CKERR(r); + r = txn->commit(txn0, 0); CKERR(r); + } else if (do_abort) { + r = txn->abort(txn); CKERR(r); + r = txn->abort(txn0); CKERR(r); + + // force an fsync of the log + r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); + r = txn->commit(txn, 0); CKERR(r); + } + //printf("shutdown\n"); + abort(); +} + +static void +do_x1_recover (BOOL did_commit) { + DB_ENV *env; + DB *dba, *dbb; + int r; + r = db_env_create(&env, 0); CKERR(r); + r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); + r = db_create(&dba, env, 0); CKERR(r); + r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); + r = db_create(&dbb, env, 0); CKERR(r); + r = dba->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); + DBT aa={.size=0}, ab={.size=0}; + DBT ba={.size=0}, bb={.size=0}; + DB_TXN *txn; + DBC *ca,*cb; + r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); + r = dba->cursor(dba, txn, &ca, 0); CKERR(r); + r = dbb->cursor(dbb, txn, &cb, 0); CKERR(r); + int ra = ca->c_get(ca, &aa, &ab, DB_FIRST); CKERR(r); + int rb = cb->c_get(cb, &ba, &bb, DB_FIRST); CKERR(r); + if (did_commit) { + assert(ra==0); + assert(rb==0); + // verify key-value pairs + assert(aa.size==2); + assert(ab.size==2); + assert(ba.size==2); + assert(bb.size==2); + const char a[2] = "a"; + const char b[2] = "b"; + assert(memcmp(aa.data, &a, 2)==0); + assert(memcmp(ab.data, &b, 2)==0); + assert(memcmp(ab.data, &b, 2)==0); + assert(memcmp(bb.data, &a, 2)==0); + // make sure no other entries in DB + assert(ca->c_get(ca, &aa, &ab, DB_NEXT) == DB_NOTFOUND); + assert(cb->c_get(cb, &ba, &bb, DB_NEXT) == DB_NOTFOUND); + fprintf(stderr, "Both verified. Yay!\n"); + } else { + // It wasn't committed (it also wasn't aborted), but a checkpoint happened. + assert(ra==DB_NOTFOUND); + assert(rb==DB_NOTFOUND); + fprintf(stderr, "Neither present. Yay!\n"); + } + r = ca->c_close(ca); CKERR(r); + r = cb->c_close(cb); CKERR(r); + r = txn->commit(txn, 0); CKERR(r); + r = dba->close(dba, 0); CKERR(r); + r = dbb->close(dbb, 0); CKERR(r); + r = env->close(env, 0); CKERR(r); + exit(0); +} + +static void +do_x1_recover_only (void) { + DB_ENV *env; + int r; + + r = db_env_create(&env, 0); CKERR(r); + r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); + r = env->close(env, 0); CKERR(r); + exit(0); +} + +const char *cmd; + +static void +do_test_internal (BOOL commit) { + pid_t pid; + if (0 == (pid=fork())) { + int r=execl(cmd, verbose ? "-v" : "-q", commit ? "--commit" : "--abort", NULL); + assert(r==-1); + printf("execl failed: %d (%s)\n", errno, strerror(errno)); + assert(0); + } + { + int r; + int status; + r = waitpid(pid, &status, 0); + //printf("signaled=%d sig=%d\n", WIFSIGNALED(status), WTERMSIG(status)); + assert(WIFSIGNALED(status) && WTERMSIG(status)==SIGABRT); + } + // Now find out what happend + + if (0 == (pid = fork())) { + int r=execl(cmd, verbose ? "-v" : "-q", commit ? "--recover-committed" : "--recover-aborted", NULL); + assert(r==-1); + printf("execl failed: %d (%s)\n", errno, strerror(errno)); + assert(0); + } + { + int r; + int status; + r = waitpid(pid, &status, 0); + //printf("recovery exited=%d\n", WIFEXITED(status)); + assert(WIFEXITED(status) && WEXITSTATUS(status)==0); + } +} + +static void +do_test (void) { + do_test_internal(TRUE); + do_test_internal(FALSE); +} + +BOOL do_commit=FALSE, do_abort=FALSE, do_explicit_abort=FALSE, do_recover_committed=FALSE, do_recover_aborted=FALSE, do_recover_only=FALSE; + +static void +x1_parse_args (int argc, char *argv[]) { + int resultcode; + cmd = argv[0]; + argc--; argv++; + while (argc>0) { + if (strcmp(argv[0], "-v") == 0) { + verbose++; + } else if (strcmp(argv[0],"-q")==0) { + verbose--; + if (verbose<0) verbose=0; + } else if (strcmp(argv[0], "--commit")==0) { + do_commit=1; + } else if (strcmp(argv[0], "--abort")==0) { + do_abort=1; + } else if (strcmp(argv[0], "--explicit-abort")==0) { + do_explicit_abort=1; + } else if (strcmp(argv[0], "--recover-committed")==0) { + do_recover_committed=1; + } else if (strcmp(argv[0], "--recover-aborted")==0) { + do_recover_aborted=1; + } else if (strcmp(argv[0], "--recover-only") == 0) { + do_recover_only=1; + } else if (strcmp(argv[0], "-h")==0) { + resultcode=0; + do_usage: + fprintf(stderr, "Usage:\n%s [-v|-q]* [-h] {--commit | --abort | --explicit-abort | --recover-committed | --recover-aborted } \n", cmd); + exit(resultcode); + } else { + fprintf(stderr, "Unknown arg: %s\n", argv[0]); + resultcode=1; + goto do_usage; + } + argc--; + argv++; + } + { + int n_specified=0; + if (do_commit) n_specified++; + if (do_abort) n_specified++; + if (do_explicit_abort) n_specified++; + if (do_recover_committed) n_specified++; + if (do_recover_aborted) n_specified++; + if (do_recover_only) n_specified++; + if (n_specified>1) { + printf("Specify only one of --commit or --abort or --recover-committed or --recover-aborted\n"); + resultcode=1; + goto do_usage; + } + } +} + +int +test_main (int argc, char *argv[]) { + x1_parse_args(argc, argv); + if (do_commit) { + do_x1_shutdown (TRUE, FALSE); + } else if (do_abort) { + do_x1_shutdown (FALSE, FALSE); + } else if (do_explicit_abort) { + do_x1_shutdown(FALSE, TRUE); + } else if (do_recover_committed) { + do_x1_recover(TRUE); + } else if (do_recover_aborted) { + do_x1_recover(FALSE); + } else if (do_recover_only) { + do_x1_recover_only(); + } else { + do_test(); + } + return 0; +}