From 57ddbdd9f3e799d6027969454682389867d3454e Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Tue, 16 Apr 2013 23:58:00 -0400 Subject: [PATCH] some changes related to recovery and the log. close[t:1959] git-svn-id: file:///svn/toku/tokudb@14328 c7de825b-a66e-492c-adef-691d508d4ae1 --- newbrt/recover.c | 63 +++++++++++++++++++++------- newbrt/recover.h | 6 ++- src/tests/bug1381.c | 4 +- src/tests/create-datadir.c | 79 +++++++++++++++++++++++++++++++++++ src/tests/test1324.c | 7 ++++ src/tests/test_txn_recover3.c | 5 +++ src/ydb.c | 32 +++++++++++--- 7 files changed, 171 insertions(+), 25 deletions(-) create mode 100644 src/tests/create-datadir.c diff --git a/newbrt/recover.c b/newbrt/recover.c index 61aa449f1d1..5c7110e3d0c 100644 --- a/newbrt/recover.c +++ b/newbrt/recover.c @@ -747,26 +747,56 @@ static int do_recovery(RECOVER_ENV env, const char *data_dir, const char *log_di return rr; } +static int recover_lock(const char *lock_dir, int *lockfd) { + const char fname[] = "/__recoverylock_dont_delete_me"; + int namelen=strlen(lock_dir); + char lockfname[namelen+sizeof(fname)]; + + int l = snprintf(lockfname, sizeof(lockfname), "%s%s", lock_dir, fname); + assert(l+1 == (signed)(sizeof(lockfname))); + *lockfd = toku_os_lock_file(lockfname); + if (*lockfd < 0) { + int e = errno; + printf("Couldn't run recovery because some other process holds the recovery lock %s\n", lockfname); + return e; + } + return 0; +} + +static int recover_unlock(int lockfd) { + int r = toku_os_unlock_file(lockfd); + if (r != 0) + return errno; + return 0; +} + +int tokudb_recover_delete_rolltmp_files(const char *UU(data_dir), const char *log_dir) { + int lockfd = -1; + + int r = recover_lock(log_dir, &lockfd); + if (r != 0) + return r; + + r = toku_delete_rolltmp_files(log_dir); + + int rr = recover_unlock(lockfd); + if (r == 0 && rr != 0) + r = rr; + + return r; +} + int tokudb_recover(const char *data_dir, const char *log_dir, brt_compare_func bt_compare, brt_compare_func dup_compare) { int r; - int lockfd; + int lockfd = -1; - { - const char fname[] = "/__recoverylock_dont_delete_me"; - int namelen=strlen(data_dir); - char lockfname[namelen+sizeof(fname)]; - - int l = snprintf(lockfname, sizeof(lockfname), "%s%s", data_dir, fname); - assert(l+1 == (signed)(sizeof(lockfname))); - lockfd = toku_os_lock_file(lockfname); - if (lockfd<0) { - printf("Couldn't run recovery because some other process holds the recovery lock %s\n", lockfname); - return errno; } - } + r = recover_lock(log_dir, &lockfd); + if (r != 0) + return r; r = toku_delete_rolltmp_files(log_dir); if (r != 0) { - toku_os_unlock_file(lockfd); + (void) recover_unlock(lockfd); return r; } @@ -781,8 +811,9 @@ int tokudb_recover(const char *data_dir, const char *log_dir, brt_compare_func b recover_env_cleanup(&renv); } - r = toku_os_unlock_file(lockfd); - if (r != 0) return errno; + r = recover_unlock(lockfd); + if (r != 0) + return r; return rr; } diff --git a/newbrt/recover.h b/newbrt/recover.h index 3651d1938ca..10331daa895 100644 --- a/newbrt/recover.h +++ b/newbrt/recover.h @@ -15,7 +15,7 @@ #include "x1764.h" // Run tokudb recovery from the log -// Returns: 0 if successfull +// Returns 0 if success int tokudb_recover(const char *datadir, const char *logdir, brt_compare_func bt_compare, brt_compare_func dup_compare); // Effect: Check the tokudb logs to determine whether or not we need to run recovery. @@ -24,4 +24,8 @@ int tokudb_recover(const char *datadir, const char *logdir, brt_compare_func bt_ // Returns: TRUE if we need recovery, otherwise FALSE. int tokudb_needs_recovery(const char *logdir, BOOL ignore_empty_log); +// Delete the rolltmp files +// Ruturns 0 if success +int tokudb_recover_delete_rolltmp_files(const char *datadir, const char *logdir); + #endif // TOKURECOVER_H diff --git a/src/tests/bug1381.c b/src/tests/bug1381.c index 832ab28643c..007797487cb 100644 --- a/src/tests/bug1381.c +++ b/src/tests/bug1381.c @@ -20,7 +20,7 @@ static void do_1381_maybe_lock (int do_table_lock, u_int64_t *raw_count) { DB_ENV *env; DB *db; - const int envflags = DB_CREATE|DB_INIT_MPOOL|DB_INIT_TXN|DB_INIT_LOCK|DB_INIT_LOG |DB_THREAD |DB_PRIVATE | DB_RECOVER; + const int envflags = DB_CREATE|DB_INIT_MPOOL|DB_INIT_TXN|DB_INIT_LOCK|DB_THREAD|DB_PRIVATE; r = db_env_create(&env, 0); CKERR(r); r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); @@ -35,7 +35,7 @@ static void do_1381_maybe_lock (int do_table_lock, u_int64_t *raw_count) { { DB_ENV *env; DB *db; - const int envflags = DB_CREATE|DB_INIT_MPOOL|DB_INIT_TXN|DB_INIT_LOCK|DB_INIT_LOG |DB_THREAD |DB_PRIVATE; + const int envflags = DB_CREATE|DB_INIT_MPOOL|DB_INIT_TXN|DB_INIT_LOCK|DB_THREAD |DB_PRIVATE; r = db_env_create(&env, 0); CKERR(r); r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); diff --git a/src/tests/create-datadir.c b/src/tests/create-datadir.c new file mode 100644 index 00000000000..243d674cd63 --- /dev/null +++ b/src/tests/create-datadir.c @@ -0,0 +1,79 @@ +// test data directories + +#include +#include +#include "test.h" + +const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN; + +char *namea="a.db"; +char *nameb="b.db"; + +static void run_test (void) { + int r; + system("rm -rf " ENVDIR); + toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); + + DB_ENV *env; + r = db_env_create(&env, 0); CKERR(r); + r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); + + DB *dba; + r = db_create(&dba, env, 0); CKERR(r); + r = dba->open(dba, NULL, "a.db", NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); + r = dba->close(dba, 0); CKERR(r); + + DB *dbb; + r = db_create(&dbb, env, 0); CKERR(r); + r = dbb->open(dbb, NULL, "bdir/b.db", NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); assert(r != 0); + r = toku_os_mkdir(ENVDIR "/bdir", 0777); assert(r == 0); + r = dbb->open(dbb, NULL, "bdir/b.db", NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); + r = dbb->close(dbb, 0); CKERR(r); + + r = env->close(env, 0); CKERR(r); + + r = toku_os_mkdir(ENVDIR "/cdir", 0777); assert(r == 0); + r = db_env_create(&env, 0); CKERR(r); + r = env->set_data_dir(env, "cdir"); CKERR(r); + r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); + + DB *dbc; + r = db_create(&dbc, env, 0); CKERR(r); + r = dbc->open(dbc, NULL, "c.db", NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); + r = dbc->close(dbc, 0); CKERR(r); + + r = env->close(env, 0); CKERR(r); +} + +const char *cmd; + +static void test_parse_args (int argc, char *argv[]) { + int resultcode; + cmd = argv[0]; + argc--; argv++; + while (argc>0) { + if (strcmp(argv[0], "-v") == 0) { + verbose++; + } else if (strcmp(argv[0],"-q")==0) { + verbose--; + if (verbose<0) verbose=0; + } else if (strcmp(argv[0], "-h")==0) { + resultcode=0; + do_usage: + fprintf(stderr, "Usage:\n%s [-v|-q]* [-h] {--test | --recover } \n", cmd); + exit(resultcode); + } else { + fprintf(stderr, "Unknown arg: %s\n", argv[0]); + resultcode=1; + goto do_usage; + } + argc--; + argv++; + } +} + +int test_main (int argc, char *argv[]) { + test_parse_args(argc, argv); + run_test(); + return 0; +} diff --git a/src/tests/test1324.c b/src/tests/test1324.c index 90326facbb3..33dceb4d153 100644 --- a/src/tests/test1324.c +++ b/src/tests/test1324.c @@ -35,6 +35,13 @@ do_1324 (int moreflags) DB_ENV *env; int r; + if (moreflags & DB_INIT_LOG) { + // create the log + r = db_env_create(&env, 0); CKERR(r); + r = env->open(env, ENVDIR, envflags & ~DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); + r = env->close(env, 0); CKERR(r); + } + r = db_env_create(&env, 0); CKERR(r); r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); diff --git a/src/tests/test_txn_recover3.c b/src/tests/test_txn_recover3.c index 9e887ecf2ee..8badb45d37f 100644 --- a/src/tests/test_txn_recover3.c +++ b/src/tests/test_txn_recover3.c @@ -27,6 +27,11 @@ test_txn_recover3 (int nrows) { const char * const sname = "t.tokudb/status.brt"; int r; + r = db_env_create(&env, 0); assert(r == 0); + env->set_errfile(env, stderr); + r = env->open(env, ENVDIR, DB_CREATE|DB_INIT_MPOOL|DB_INIT_TXN|DB_INIT_LOCK|DB_INIT_LOG |DB_THREAD |DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); + r = env->close(env, 0); CKERR(r); + r = db_env_create(&env, 0); assert(r == 0); env->set_errfile(env, stderr); r = env->open(env, ENVDIR, DB_CREATE|DB_INIT_MPOOL|DB_INIT_TXN|DB_INIT_LOCK|DB_INIT_LOG |DB_THREAD |DB_PRIVATE | DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); diff --git a/src/ydb.c b/src/ydb.c index 8fb8d1986cb..57edc341b05 100644 --- a/src/ydb.c +++ b/src/ydb.c @@ -293,6 +293,19 @@ static int toku_c_close(DBC * c); /* misc */ static char *construct_full_name(const char *dir, const char *fname); + +static int delete_rolltmp_files(DB_ENV *env) { + const char *datadir=env->i->dir; + char *logdir; + if (env->i->lg_dir) { + logdir = construct_full_name(env->i->dir, env->i->lg_dir); + } else { + logdir = toku_strdup(env->i->dir); + } + int r = tokudb_recover_delete_rolltmp_files(datadir, logdir); + toku_free(logdir); + return r; +} static int do_recovery (DB_ENV *env) { const char *datadir=env->i->dir; @@ -385,15 +398,22 @@ static int toku_env_open(DB_ENV * env, const char *home, u_int32_t flags, int mo env->i->open_flags = flags; env->i->open_mode = mode; - unused_flags &= ~DB_INIT_TXN & ~DB_INIT_LOG; + unused_flags &= ~DB_INIT_TXN & ~DB_INIT_LOG; - if (flags&DB_RECOVER) { - r = do_recovery(env); - if (r != 0) return r; - } else { - r = needs_recovery(env); + if (flags & DB_INIT_TXN) { + r = delete_rolltmp_files(env); if (r != 0) return r; } + + if (flags & DB_INIT_LOG) { + if (flags & DB_RECOVER) { + r = do_recovery(env); + if (r != 0) return r; + } else { + r = needs_recovery(env); + if (r != 0) return r; + } + } if (flags & (DB_INIT_TXN | DB_INIT_LOG)) { char* full_dir = NULL;