mirror of
https://github.com/MariaDB/server.git
synced 2025-01-22 23:04:20 +01:00
Addresses #2227 refs[t:2227] Add test case that reproduces crash reliably
Closes #2225 closes[t:2225] Add fileops unit tests (recovery version). git-svn-id: file:///svn/toku/tokudb@16251 c7de825b-a66e-492c-adef-691d508d4ae1
This commit is contained in:
Yoni Fogel
parent
3d64b79078
commit
d1bd4159a3
3 changed files with 565 additions and 1 deletions
|
|
@ -105,6 +105,7 @@ BDB_DONTRUN_TESTS = \
|
|||
checkpoint_callback \
|
||||
recovery_stress \
|
||||
recovery_fileops_stress \
|
||||
recovery_fileops_unit \
|
||||
test_txn_nested1 \
|
||||
test_txn_nested2 \
|
||||
test_txn_nested3 \
|
||||
|
|
@ -504,6 +505,58 @@ recover_stress.tdbrun: checkpoint_stress.tdb$(BINSUF) $(PTHREAD_LOCAL)
|
|||
test `grep -l 'HAPPY CRASH' dir.checkpoint_stress.c.tdb/error.* |wc -l` = $$(($(STRESS_RUNS)-1)) \
|
||||
) $(MAYBEINVERTER) $(SUMMARIZE_CMD)
|
||||
|
||||
recovery_fileops_unit.tdbrun: SHELL=/bin/bash
|
||||
recovery_fileops_unit.tdbrun: recovery_fileops_unit.tdb$(BINSUF) $(PTHREAD_LOCAL)
|
||||
rm -rf dir.$@.errors
|
||||
mkdir dir.$@.errors
|
||||
( \
|
||||
Oset="c d r"; \
|
||||
aset="0 1"; \
|
||||
bset="0 1"; \
|
||||
cset="0 1 2"; \
|
||||
fset="0 1"; \
|
||||
count=0; \
|
||||
for O in $$Oset; do \
|
||||
if test $$O = c ; then \
|
||||
gset="0"; \
|
||||
hset="0"; \
|
||||
else \
|
||||
gset="0 1 2 3 4 5"; \
|
||||
hset="0 1"; \
|
||||
fi; \
|
||||
for a in $$aset; do \
|
||||
for b in $$bset; do \
|
||||
if test $$b -eq 0 ; then \
|
||||
dset="0"; \
|
||||
eset="0"; \
|
||||
else \
|
||||
dset="0 1"; \
|
||||
eset="0 1"; \
|
||||
fi; \
|
||||
for c in $$cset; do \
|
||||
for d in $$dset; do \
|
||||
for e in $$eset; do \
|
||||
for f in $$fset; do \
|
||||
for g in $$gset; do \
|
||||
for h in $$hset; do \
|
||||
errorfile=dir.$@.errors/crash.$$O.$$a.$$b.$$c.$$d.$$e.$$f.$$g.$$h; \
|
||||
combination="-O $$O -A $$a -B $$b -C $$c -D $$d -E $$e -F $$f -G $$g -H $$"; \
|
||||
./$< $(VERBVERBOSE) -c $$combination 2> $$errorfile && { cat dir.$@.errors/crash; echo Error: no crash in $$errorfile ; exit 1; }; \
|
||||
grep 'HAPPY CRASH' $$errorfile >/dev/null || { cat $$errorfile; echo Error: incorrect crash in $$errorfile ; exit 1; }; \
|
||||
$(TDBVGRIND) ./$< $(VERBVERBOSE) -r $$combination || { cat $$errorfile ; echo Error: during recovery in $$errorfile ; exit 1; }; \
|
||||
count=$$(($$count + 1)); \
|
||||
done; \
|
||||
done; \
|
||||
done; \
|
||||
done; \
|
||||
done; \
|
||||
done; \
|
||||
done; \
|
||||
done; \
|
||||
done; \
|
||||
echo $$count tests passed; \
|
||||
) $(SUMMARIZE_CMD)
|
||||
|
||||
# helgrind1 is supposed to fail.
|
||||
helgrind1.tdbrun: TDBVGRIND=$(HGRIND) --log-file=helgrind1.tdb.deleteme
|
||||
helgrind1.tdbrun: MAYBEINVERTER=$(INVERTER)
|
||||
|
|
|
|||
|
|
@ -463,7 +463,8 @@ verify_sequential_rows(DB* compare_db, int64_t firstkey, int64_t numkeys) {
|
|||
static void UU()
|
||||
snapshot(DICTIONARY d, int do_checkpoint) {
|
||||
if (do_checkpoint) {
|
||||
env->txn_checkpoint(env, 0, 0, 0);
|
||||
int r = env->txn_checkpoint(env, 0, 0, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
else {
|
||||
db_shutdown(d);
|
||||
|
|
|
|||
510
src/tests/recovery_fileops_unit.c
Normal file
510
src/tests/recovery_fileops_unit.c
Normal file
|
|
@ -0,0 +1,510 @@
|
|||
/* -*- mode: C; c-basic-offset: 4 -*- */
|
||||
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
|
||||
#ident "$Id$"
|
||||
|
||||
#include "test.h"
|
||||
#include "toku_pthread.h"
|
||||
#include <db.h>
|
||||
#include <sys/stat.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
static int do_recover;
|
||||
static int do_crash;
|
||||
static char fileop;
|
||||
static int choices['H'-'A'+1];
|
||||
const int num_choices = sizeof(choices)/sizeof(choices[0]);
|
||||
static DB_TXN *txn;
|
||||
const char *oldname = "oldfoo";
|
||||
const char *newname = "newfoo";
|
||||
DB_ENV *env;
|
||||
DB *db;
|
||||
static int crash_during_checkpoint;
|
||||
static char *cmd;
|
||||
|
||||
#define usage() assert(FALSE)
|
||||
#if 0
|
||||
static void
|
||||
usage(void) {
|
||||
fprintf(stderr, "Usage:\n%s [-v|-q]* [-h] (-c|-r) -O fileop -A# -B# -C# -D# -E# [-F# -G#]\n"
|
||||
" fileop = c/r/d (create/rename/delete)\n"
|
||||
" Where # is a single digit number > 0.\n"
|
||||
" A-F are required for fileop=create\n"
|
||||
" A-H are required for fileop=delete, fileop=rename\n", cmd);
|
||||
exit(1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
enum { CLOSE_TXN_COMMIT, CLOSE_TXN_ABORT, CLOSE_TXN_NONE };
|
||||
enum {CREATE_CREATE, CREATE_CHECKPOINT, CREATE_COMMIT_NEW,
|
||||
CREATE_COMMIT_NEW_CHECKPOINT, CREATE_COMMIT_CHECKPOINT_NEW,
|
||||
CREATE_CHECKPOINT_COMMIT_NEW};
|
||||
static int
|
||||
get_x_choice(char c, int possibilities) {
|
||||
assert(c < 'A' + num_choices);
|
||||
assert(c >= 'A');
|
||||
int choice = choices[c-'A'];
|
||||
if (choice >= possibilities)
|
||||
usage();
|
||||
return choice;
|
||||
}
|
||||
|
||||
//return 0 or 1
|
||||
static int
|
||||
get_bool_choice(char c) {
|
||||
return get_x_choice(c, 2);
|
||||
}
|
||||
|
||||
static int
|
||||
get_choice_first_create_unrelated_txn(void) {
|
||||
return get_bool_choice('A');
|
||||
}
|
||||
|
||||
static int
|
||||
get_choice_do_checkpoint_after_fileop(void) {
|
||||
return get_bool_choice('B');
|
||||
}
|
||||
|
||||
static int
|
||||
get_choice_txn_close_type(void) {
|
||||
return get_x_choice('C', 3);
|
||||
}
|
||||
|
||||
static int
|
||||
get_choice_close_txn_before_checkpoint(void) {
|
||||
int choice = get_bool_choice('D');
|
||||
//Can't do checkpoint related thing without checkpoint
|
||||
if (choice)
|
||||
assert(get_choice_do_checkpoint_after_fileop());
|
||||
return choice;
|
||||
}
|
||||
|
||||
static int
|
||||
get_choice_crash_checkpoint_in_callback(void) {
|
||||
int choice = get_bool_choice('E');
|
||||
//Can't do checkpoint related thing without checkpoint
|
||||
if (choice)
|
||||
assert(get_choice_do_checkpoint_after_fileop());
|
||||
return choice;
|
||||
}
|
||||
|
||||
static int
|
||||
get_choice_flush_log_before_crash(void) {
|
||||
return get_bool_choice('F');
|
||||
}
|
||||
|
||||
static int
|
||||
get_choice_create_type(void) {
|
||||
return get_x_choice('G', 6);
|
||||
}
|
||||
|
||||
static int
|
||||
get_choice_txn_does_open_close_before_fileop(void) {
|
||||
return get_bool_choice('H');
|
||||
}
|
||||
|
||||
static void
|
||||
do_args(int argc, char *argv[]) {
|
||||
cmd = argv[0];
|
||||
int i;
|
||||
//Clear
|
||||
for (i = 0; i < num_choices; i++) {
|
||||
choices[i] = -1;
|
||||
}
|
||||
|
||||
char c;
|
||||
while ((c = getopt(argc, argv, "vqhcrO:A:B:C:D:E:F:G:H:")) != -1) {
|
||||
switch(c) {
|
||||
case 'v':
|
||||
verbose++;
|
||||
break;
|
||||
case 'q':
|
||||
verbose--;
|
||||
if (verbose<0) verbose=0;
|
||||
break;
|
||||
case 'h':
|
||||
case '?':
|
||||
usage();
|
||||
break;
|
||||
case 'c':
|
||||
do_crash = 1;
|
||||
break;
|
||||
case 'r':
|
||||
do_recover = 1;
|
||||
break;
|
||||
case 'O':
|
||||
if (fileop != '\0')
|
||||
usage();
|
||||
fileop = optarg[0];
|
||||
switch (fileop) {
|
||||
case 'c':
|
||||
case 'r':
|
||||
case 'd':
|
||||
break;
|
||||
default:
|
||||
usage();
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case 'A':
|
||||
case 'B':
|
||||
case 'C':
|
||||
case 'D':
|
||||
case 'E':
|
||||
case 'F':
|
||||
case 'G':
|
||||
case 'H':
|
||||
if (fileop == '\0')
|
||||
usage();
|
||||
int num = atoi(optarg);
|
||||
if (num < 0 || num > 9)
|
||||
usage();
|
||||
choices[c - 'A'] = num;
|
||||
break;
|
||||
default:
|
||||
usage();
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (argc!=optind) { usage(); exit(1); }
|
||||
|
||||
for (i = 0; i < num_choices; i++) {
|
||||
if (i >= 'G' - 'A' && fileop == 'c')
|
||||
break;
|
||||
if (choices[i] == -1)
|
||||
usage();
|
||||
}
|
||||
assert(!do_recover || !do_crash);
|
||||
assert(do_recover || do_crash);
|
||||
}
|
||||
|
||||
static void UU() crash_it(void) {
|
||||
int r;
|
||||
if (get_choice_flush_log_before_crash()) {
|
||||
r = env->log_flush(env, NULL); //TODO: USe a real DB_LSN* instead of NULL
|
||||
CKERR(r);
|
||||
}
|
||||
fprintf(stderr, "HAPPY CRASH\n");
|
||||
fflush(stdout);
|
||||
fflush(stderr);
|
||||
int zero = 0;
|
||||
int divide_by_zero = 1/zero;
|
||||
printf("force use of %d\n", divide_by_zero);
|
||||
fflush(stdout);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
static void checkpoint_callback_maybe_crash(void * UU(extra)) {
|
||||
if (crash_during_checkpoint)
|
||||
crash_it();
|
||||
}
|
||||
|
||||
static void env_startup(void) {
|
||||
int r;
|
||||
int recover_flag = do_crash ? 0 : DB_RECOVER;
|
||||
if (do_crash) {
|
||||
db_env_set_checkpoint_callback(checkpoint_callback_maybe_crash, NULL);
|
||||
r = system("rm -rf " ENVDIR); CKERR(r);
|
||||
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
|
||||
}
|
||||
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | recover_flag;
|
||||
r = db_env_create(&env, 0);
|
||||
CKERR(r);
|
||||
env->set_errfile(env, stderr);
|
||||
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO);
|
||||
CKERR(r);
|
||||
//Disable auto-checkpointing.
|
||||
r = env->checkpointing_set_period(env, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
|
||||
static void
|
||||
env_shutdown(void) {
|
||||
int r;
|
||||
r = env->close(env, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
|
||||
static void
|
||||
maybe_make_oldest_living_txn(void) {
|
||||
if (get_choice_first_create_unrelated_txn()) {
|
||||
// create a txn that never closes, forcing recovery to run from the beginning of the log
|
||||
DB_TXN *oldest_living_txn;
|
||||
int r;
|
||||
r = env->txn_begin(env, NULL, &oldest_living_txn, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
make_txn(void) {
|
||||
int r;
|
||||
r = env->txn_begin(env, NULL, &txn, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
|
||||
static void
|
||||
fcreate(void) {
|
||||
int r;
|
||||
r = db_create(&db, env, 0);
|
||||
CKERR(r);
|
||||
r = db->open(db, txn, oldname, NULL, DB_BTREE, DB_CREATE|DB_EXCL, 0666);
|
||||
CKERR(r);
|
||||
r = db->close(db, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
|
||||
static void
|
||||
fdelete(void) {
|
||||
int r;
|
||||
r = env->dbremove(env, txn, oldname, NULL, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
|
||||
static void
|
||||
frename(void) {
|
||||
int r;
|
||||
r = env->dbrename(env, txn, oldname, NULL, newname, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
|
||||
static void
|
||||
close_txn(int type) {
|
||||
int r;
|
||||
assert(txn);
|
||||
if (type==CLOSE_TXN_COMMIT) {
|
||||
//commit
|
||||
r = txn->commit(txn, 0);
|
||||
CKERR(r);
|
||||
txn = NULL;
|
||||
}
|
||||
else if (type == CLOSE_TXN_ABORT) {
|
||||
//abort
|
||||
r = txn->abort(txn);
|
||||
CKERR(r);
|
||||
txn = NULL;
|
||||
}
|
||||
else
|
||||
assert(type == CLOSE_TXN_NONE);
|
||||
}
|
||||
|
||||
static void
|
||||
checkpoint(void) {
|
||||
int r;
|
||||
r = env->txn_checkpoint(env, 0, 0, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
|
||||
static void
|
||||
create_and_crash(void) {
|
||||
//Make txn
|
||||
make_txn();
|
||||
//fcreate
|
||||
fcreate();
|
||||
|
||||
if (get_choice_do_checkpoint_after_fileop()) {
|
||||
crash_during_checkpoint = get_choice_crash_checkpoint_in_callback();
|
||||
if (get_choice_close_txn_before_checkpoint())
|
||||
close_txn(get_choice_txn_close_type());
|
||||
checkpoint();
|
||||
if (!get_choice_close_txn_before_checkpoint())
|
||||
close_txn(get_choice_txn_close_type());
|
||||
}
|
||||
else {
|
||||
crash_during_checkpoint = get_choice_crash_checkpoint_in_callback();
|
||||
assert(!crash_during_checkpoint);
|
||||
close_txn(get_choice_txn_close_type());
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
create_and_maybe_checkpoint_and_or_close_after_create(void) {
|
||||
fcreate();
|
||||
switch (get_choice_create_type()) {
|
||||
case (CREATE_CREATE): //Just create
|
||||
break;
|
||||
case (CREATE_CHECKPOINT): //Create then checkpoint
|
||||
checkpoint();
|
||||
break;
|
||||
case (CREATE_COMMIT_NEW): //Create then commit
|
||||
close_txn(CLOSE_TXN_COMMIT);
|
||||
make_txn();
|
||||
break;
|
||||
case (CREATE_COMMIT_NEW_CHECKPOINT): //Create then commit then create new txn then checkpoint
|
||||
close_txn(CLOSE_TXN_COMMIT);
|
||||
make_txn();
|
||||
checkpoint();
|
||||
break;
|
||||
case (CREATE_COMMIT_CHECKPOINT_NEW): //Create then commit then checkpoint then create new txn
|
||||
close_txn(CLOSE_TXN_COMMIT);
|
||||
checkpoint();
|
||||
make_txn();
|
||||
break;
|
||||
case (CREATE_CHECKPOINT_COMMIT_NEW): //Create then checkpoint then commit then create new txn
|
||||
checkpoint();
|
||||
close_txn(CLOSE_TXN_COMMIT);
|
||||
make_txn();
|
||||
break;
|
||||
default:
|
||||
assert(FALSE);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
maybe_open_and_close_file_again_before_fileop(void) {
|
||||
if (get_choice_txn_does_open_close_before_fileop()) {
|
||||
int r;
|
||||
r = db_create(&db, env, 0);
|
||||
CKERR(r);
|
||||
r = db->open(db, txn, oldname, NULL, DB_BTREE, 0, 0666);
|
||||
CKERR(r);
|
||||
r = db->close(db, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
delete_and_crash(void) {
|
||||
//Make txn
|
||||
make_txn();
|
||||
//fcreate
|
||||
create_and_maybe_checkpoint_and_or_close_after_create();
|
||||
|
||||
maybe_open_and_close_file_again_before_fileop();
|
||||
|
||||
fdelete();
|
||||
if (get_choice_do_checkpoint_after_fileop()) {
|
||||
crash_during_checkpoint = get_choice_crash_checkpoint_in_callback();
|
||||
if (get_choice_close_txn_before_checkpoint())
|
||||
close_txn(get_choice_txn_close_type());
|
||||
checkpoint();
|
||||
if (!get_choice_close_txn_before_checkpoint())
|
||||
close_txn(get_choice_txn_close_type());
|
||||
}
|
||||
else {
|
||||
crash_during_checkpoint = get_choice_crash_checkpoint_in_callback();
|
||||
assert(!crash_during_checkpoint);
|
||||
close_txn(get_choice_txn_close_type());
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
rename_and_crash(void) {
|
||||
//Make txn
|
||||
make_txn();
|
||||
//fcreate
|
||||
create_and_maybe_checkpoint_and_or_close_after_create();
|
||||
|
||||
maybe_open_and_close_file_again_before_fileop();
|
||||
|
||||
frename();
|
||||
if (get_choice_do_checkpoint_after_fileop()) {
|
||||
crash_during_checkpoint = get_choice_crash_checkpoint_in_callback();
|
||||
if (get_choice_close_txn_before_checkpoint())
|
||||
close_txn(get_choice_txn_close_type());
|
||||
checkpoint();
|
||||
if (!get_choice_close_txn_before_checkpoint())
|
||||
close_txn(get_choice_txn_close_type());
|
||||
}
|
||||
else {
|
||||
crash_during_checkpoint = get_choice_crash_checkpoint_in_callback();
|
||||
assert(!crash_during_checkpoint);
|
||||
close_txn(get_choice_txn_close_type());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void
|
||||
execute_and_crash(void) {
|
||||
maybe_make_oldest_living_txn();
|
||||
//split into create/delete/rename
|
||||
if (fileop=='c')
|
||||
create_and_crash();
|
||||
else if (fileop == 'd')
|
||||
delete_and_crash();
|
||||
else {
|
||||
assert(fileop == 'r');
|
||||
rename_and_crash();
|
||||
}
|
||||
crash_it();
|
||||
}
|
||||
|
||||
static int
|
||||
did_create_commit_early(void) {
|
||||
int r;
|
||||
switch (get_choice_create_type()) {
|
||||
case (CREATE_CREATE): //Just create
|
||||
case (CREATE_CHECKPOINT): //Create then checkpoint
|
||||
r = 0;
|
||||
break;
|
||||
case (CREATE_COMMIT_NEW): //Create then commit
|
||||
case (CREATE_COMMIT_NEW_CHECKPOINT): //Create then commit then create new txn then checkpoint
|
||||
case (CREATE_COMMIT_CHECKPOINT_NEW): //Create then commit then checkpoint then create new txn
|
||||
case (CREATE_CHECKPOINT_COMMIT_NEW): //Create then checkpoint then commit then create new txn
|
||||
r = 1;
|
||||
break;
|
||||
default:
|
||||
assert(FALSE);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
static void
|
||||
verify_file_exists(const char *name, int should_exist) {
|
||||
int r;
|
||||
r = db_create(&db, env, 0);
|
||||
CKERR(r);
|
||||
r = db->open(db, NULL, name, NULL, DB_BTREE, 0, 0666);
|
||||
if (should_exist)
|
||||
CKERR(r);
|
||||
else
|
||||
CKERR2(r, ENOENT);
|
||||
r = db->close(db, 0);
|
||||
CKERR(r);
|
||||
}
|
||||
|
||||
static int
|
||||
fileop_did_commit(void) {
|
||||
return get_choice_txn_close_type() == CLOSE_TXN_COMMIT &&
|
||||
(!get_choice_do_checkpoint_after_fileop() ||
|
||||
!get_choice_crash_checkpoint_in_callback() ||
|
||||
get_choice_close_txn_before_checkpoint());
|
||||
}
|
||||
|
||||
static void
|
||||
recover_and_verify(void) {
|
||||
//Recovery was done during env_startup
|
||||
int expect_old_name = 0;
|
||||
int expect_new_name = 0;
|
||||
if (fileop=='c') {
|
||||
expect_old_name = fileop_did_commit();
|
||||
}
|
||||
else if (fileop == 'd') {
|
||||
expect_old_name = did_create_commit_early() && !fileop_did_commit();
|
||||
}
|
||||
else {
|
||||
//Wrong? if checkpoint AND crash during checkpoint
|
||||
if (fileop_did_commit())
|
||||
expect_new_name = 1;
|
||||
else if (did_create_commit_early())
|
||||
expect_old_name = 1;
|
||||
}
|
||||
verify_file_exists(oldname, expect_old_name);
|
||||
verify_file_exists(newname, expect_new_name);
|
||||
env_shutdown();
|
||||
}
|
||||
|
||||
int
|
||||
test_main(int argc, char *argv[]) {
|
||||
crash_during_checkpoint = 0; //Do not crash during checkpoint (possibly during recovery).
|
||||
do_args(argc, argv);
|
||||
env_startup();
|
||||
if (do_crash)
|
||||
execute_and_crash();
|
||||
else
|
||||
recover_and_verify();
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
Loading…
Add table
Reference in a new issue