From 0dfc3aa50dcafbf4afee911eac588b133f51f6aa Mon Sep 17 00:00:00 2001 From: "Bradley C. Kuszmaul" Date: Tue, 16 Apr 2013 23:57:35 -0400 Subject: [PATCH] Some fixes for recovery (#1364). Also some temporary fixes for #1277. Addresses #1364, #1277. Delete the 1364 branch, so I can restart it. On the 1343 branch, I did: {{{ svn merge -r 8586:8595 https://svn.tokutek.com/tokudb/toku/tokudb.1032b+1343+1364 }}} git-svn-id: file:///svn/toku/tokudb.1032b+1343@8596 c7de825b-a66e-492c-adef-691d508d4ae1 --- newbrt/brt.c | 2 +- newbrt/leafentry.c | 1 - newbrt/log.c | 6 +++--- newbrt/log.h | 2 +- newbrt/logformat.c | 2 -- newbrt/recover.c | 7 +++++-- newbrt/roll.c | 7 ++++--- newbrt/threadpool.c | 2 +- src/tests/Makefile | 13 +++++++------ src/tests/test_db_remove.c | 13 +++++++++---- windows/memory.c | 2 +- 11 files changed, 32 insertions(+), 25 deletions(-) diff --git a/newbrt/brt.c b/newbrt/brt.c index 05ce8c5582c..9564c2171a5 100644 --- a/newbrt/brt.c +++ b/newbrt/brt.c @@ -2856,7 +2856,7 @@ int toku_brt_open(BRT t, const char *fname, const char *fname_in_env, const char if (r != 0) goto died0a; if (did_create) { mode_t mode = S_IRWXU|S_IRWXG|S_IRWXO; - r = toku_logger_log_fcreate(txn, fname_in_env, toku_cachefile_filenum(t->cf), mode); + r = toku_logger_log_fcreate(txn, fname_in_env, mode); if (r != 0) goto died_after_open; } r = toku_logger_log_fopen(txn, fname_in_env, toku_cachefile_filenum(t->cf)); diff --git a/newbrt/leafentry.c b/newbrt/leafentry.c index eebdd281682..5d9c2e34eaf 100644 --- a/newbrt/leafentry.c +++ b/newbrt/leafentry.c @@ -162,7 +162,6 @@ u_int32_t toku_logsizeof_LEAFENTRY (LEAFENTRY le) { } int toku_fread_LEAFENTRY(FILE *f, LEAFENTRY *le, struct x1764 *checksum, u_int32_t *len) { - assert(0); u_int8_t state; int r = toku_fread_u_int8_t (f, &state, checksum, len); if (r!=0) return r; TXNID xid; diff --git a/newbrt/log.c b/newbrt/log.c index 1ace95e2087..23195eaa71c 100644 --- a/newbrt/log.c +++ b/newbrt/log.c @@ -531,13 +531,13 @@ int toku_logger_txn_begin (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER return 0; } -int toku_logger_log_fcreate (TOKUTXN txn, const char *fname, FILENUM filenum, int mode) { +int toku_logger_log_fcreate (TOKUTXN txn, const char *fname, int mode) { if (txn==0) return 0; if (txn->logger->is_panicked) return EINVAL; BYTESTRING bs = { .len=strlen(fname), .data = toku_strdup_in_rollback(txn, fname) }; - int r = toku_log_fcreate (txn->logger, (LSN*)0, 0, toku_txn_get_txnid(txn), filenum, bs, mode); + int r = toku_log_fcreate (txn->logger, (LSN*)0, 0, toku_txn_get_txnid(txn), bs, mode); if (r!=0) return r; - r = toku_logger_save_rollback_fcreate(txn, toku_txn_get_txnid(txn), filenum, bs); + r = toku_logger_save_rollback_fcreate(txn, toku_txn_get_txnid(txn), bs); return r; } diff --git a/newbrt/log.h b/newbrt/log.h index 7450c9c7bc2..ca67bc12ffd 100644 --- a/newbrt/log.h +++ b/newbrt/log.h @@ -43,7 +43,7 @@ int toku_logger_commit (TOKUTXN txn, int no_sync); int toku_logger_txn_begin (TOKUTXN /*parent*/,TOKUTXN *, TOKULOGGER /*logger*/); -int toku_logger_log_fcreate (TOKUTXN, const char */*fname*/, FILENUM /*filenum*/, int /*mode*/); +int toku_logger_log_fcreate (TOKUTXN, const char */*fname*/, int /*mode*/); int toku_logger_log_fopen (TOKUTXN, const char * /*fname*/, FILENUM); diff --git a/newbrt/logformat.c b/newbrt/logformat.c index 484a9c4afa4..2d82937f003 100644 --- a/newbrt/logformat.c +++ b/newbrt/logformat.c @@ -41,7 +41,6 @@ int logformat_version_number = 0; const struct logtype rollbacks[] = { {"fcreate", 'F', FA{{"TXNID", "xid", 0}, - {"FILENUM", "filenum", 0}, {"BYTESTRING", "fname", 0}, NULLFIELD}}, // cmdinsert is used to insert a key-value pair into a NODUP DB. For rollback we don't need the data. @@ -96,7 +95,6 @@ const struct logtype logtypes[] = { NULLFIELD}}, #endif {"fcreate", 'F', FA{{"TXNID", "txnid", 0}, - {"FILENUM", "filenum", 0}, {"BYTESTRING", "fname", 0}, {"u_int32_t", "mode", "0%o"}, NULLFIELD}}, diff --git a/newbrt/recover.c b/newbrt/recover.c index 585d3075bf7..582cb4ca486 100644 --- a/newbrt/recover.c +++ b/newbrt/recover.c @@ -83,13 +83,15 @@ create_dir_from_file (const char *fname) { } static void -toku_recover_fcreate (LSN UU(lsn), TXNID UU(txnid), FILENUM UU(filenum), BYTESTRING fname,u_int32_t mode) { +toku_recover_fcreate (LSN UU(lsn), TXNID UU(txnid), BYTESTRING fname,u_int32_t mode) { char *fixed_fname = fixup_fname(&fname); create_dir_from_file(fixed_fname); int fd = open(fixed_fname, O_CREAT+O_TRUNC+O_WRONLY+O_BINARY, mode); assert(fd>=0); toku_free(fixed_fname); toku_free_BYTESTRING(fname); + int r = close(fd); + assert(r==0); } static int @@ -130,6 +132,7 @@ static void toku_recover_fheader (LSN UU(lsn), TXNID UU(txnid),FILENUM filenum,L struct brt_header *MALLOC(h); assert(h); h->dirty=0; + XMALLOC(h->flags_array); h->flags_array[0] = header.flags; h->nodesize = header.nodesize; h->free_blocks = header.free_blocks; @@ -203,7 +206,7 @@ toku_recover_newbrtnode (LSN lsn, FILENUM filenum, BLOCKNUM blocknum,u_int32_t h // Now put it in the cachetable u_int32_t fullhash = toku_cachetable_hash(pair->cf, blocknum); n->fullhash = fullhash; - toku_cachetable_put(pair->cf, blocknum, fullhash, n, toku_serialize_brtnode_size(n), toku_brtnode_flush_callback, toku_brtnode_fetch_callback, 0); + toku_cachetable_put(pair->cf, blocknum, fullhash, n, toku_serialize_brtnode_size(n), toku_brtnode_flush_callback, toku_brtnode_fetch_callback, pair->brt->h); VERIFY_COUNTS(n); diff --git a/newbrt/roll.c b/newbrt/roll.c index 64fffadfb69..f28318fde74 100644 --- a/newbrt/roll.c +++ b/newbrt/roll.c @@ -12,14 +12,12 @@ #define TOKU_DO_COMMIT_CMD_DELETE_BOTH 1 int toku_commit_fcreate (TXNID UU(xid), - FILENUM UU(filenum), BYTESTRING UU(bs_fname), TOKUTXN UU(txn)) { return 0; } int toku_rollback_fcreate (TXNID xid __attribute__((__unused__)), - FILENUM filenum, BYTESTRING bs_fname, TOKUTXN txn __attribute__((__unused__))) { char *fname = fixup_fname(&bs_fname); @@ -28,6 +26,8 @@ int toku_rollback_fcreate (TXNID xid __attribute__((__unused__)), char full_fname[full_len]; int l = snprintf(full_fname,full_len, "%s/%s", directory, fname); assert(l<=full_len); +#if 0 + // I don't think this is right. fcreate simply creates the file, and doesn't put it in the cache table. //Remove reference to the fd in the cachetable CACHEFILE cf; int r = toku_cachefile_of_filenum(txn->logger->ct, filenum, &cf); @@ -35,7 +35,8 @@ int toku_rollback_fcreate (TXNID xid __attribute__((__unused__)), r = toku_cachefile_redirect_nullfd(cf); assert(r==0); } - r = unlink(full_fname); +#endif + int r = unlink(full_fname); assert(r==0); toku_free(fname); return 0; diff --git a/newbrt/threadpool.c b/newbrt/threadpool.c index f869dfca644..f62cdc98ccd 100644 --- a/newbrt/threadpool.c +++ b/newbrt/threadpool.c @@ -1,7 +1,7 @@ #include "includes.h" // use gcc builtin fetch_and_add 0->no 1->yes -#define DO_ATOMIC_FETCH_AND_ADD 0 +#define DO_ATOMIC_FETCH_AND_ADD 1 struct threadpool { int max_threads; diff --git a/src/tests/Makefile b/src/tests/Makefile index 4e7a46f8667..83c9a66921d 100644 --- a/src/tests/Makefile +++ b/src/tests/Makefile @@ -43,6 +43,7 @@ BDB_DONTRUN_TESTS = \ test938c \ helgrind1 \ helgrind2 \ + helgrind3 \ #\ ends prev line BDB_TESTS = $(patsubst %.c,%.bdb$(BINSUF),$(filter-out $(patsubst %,%.c,$(BDB_DONTRUN_TESTS)),$(SRCS))) @@ -58,8 +59,6 @@ TDB_TESTS_THAT_SHOULD_FAIL= \ test_truncate_subdb \ test_txn_nested_abort3 \ test_txn_nested_abort4 \ - helgrind2 \ - helgrind3 \ #\ ends prev line TDB_TESTS_THAT_SHOULD_FAIL_LIT= \ @@ -123,12 +122,14 @@ SHOULD_FAIL = $(TDB_TESTS_THAT_SHOULD_FAIL_LIT) $(patsubst %,%.tdbrun,$(TDB_TEST $(SHOULD_FAIL): MAYBEINVERTER=$(INVERTER) $(SHOULD_FAIL): SUMMARIZE_CMD=$(SUMMARIZE_SHOULD_FAIL) +TDBVGRIND=$(VGRIND) + # Use -s on the command line to make things quiet. # Use -s on the command line to make things quiet. %.bdbrun: %.bdb$(BINSUF) $(DEPEND_COMPILE) $(DEPEND_LINK) $(BDBVGRIND) ./$< $(VERBVERBOSE) $(SUMMARIZE_CMD) %.tdbrun: %.tdb$(BINSUF) $(DEPEND_COMPILE) $(DEPEND_LINK) $(notdir $(LIBTDB)) - $(VGRIND) ./$< $(VERBVERBOSE) $(MAYBEINVERTER) $(SUMMARIZE_CMD) + $(TDBVGRIND) ./$< $(VERBVERBOSE) $(MAYBEINVERTER) $(SUMMARIZE_CMD) %.recover: %.tdb$(BINSUF) $(VGRIND) ./$< && \ @@ -273,11 +274,11 @@ tgbr_%_c.tdbrun: test_get_both_range.tdb$(BINSUF) $(VGRIND) ./$< $(VERBVERBOSE) -i $* -c $(MAYBEINVERTER) $(SUMMARIZE_CMD) # helgrind1 is supposed to fail. -helgrind1.tdbrun: VGRIND=$(HGRIND) --log-file=helgrind1.tdb.deleteme +helgrind1.tdbrun: TDBVGRIND=$(HGRIND) --log-file=helgrind1.tdb.deleteme helgrind1.tdbrun: MAYBEINVERTER=$(INVERTER) -helgrind2.tdbrun: VGRIND=$(HGRIND) +helgrind2.tdbrun: TDBVGRIND=$(HGRIND) helgrind2.bdbrun: BDBVGRIND=$(HGRIND) -helgrind3.tdbrun: VGRIND=$(HGRIND) +helgrind3.tdbrun: TDBVGRIND=$(HGRIND) helgrind3.bdbrun: BDBVGRIND=$(HGRIND) dumpit: diff --git a/src/tests/test_db_remove.c b/src/tests/test_db_remove.c index 5e3377410ee..59545bbf747 100644 --- a/src/tests/test_db_remove.c +++ b/src/tests/test_db_remove.c @@ -14,12 +14,17 @@ void test_db_remove (void) { r=toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); assert(r==0); // create the DB - r = db_create(&db1, null_env, 0); assert(r == 0); + r = db_create(&db1, null_env, 0); assert(r == 0); r = db1->open(db1, null_txn, fname, 0, DB_BTREE, DB_CREATE, 0666); assert(r == 0); - // Now remove it. - r = db_create(&db2, null_env, 0); assert(r==0); - r = db2->remove(db2, fname, 0, 0); assert(r==0); + // Now remove it, while it is open. + r = db_create(&db2, null_env, 0); assert(r==0); + r = db2->remove(db2, fname, 0, 0); +#ifdef USE_TDB + assert(r!=0); +#else + assert(r==0); +#endif r = db1->close(db1, 0); assert(r==0); } diff --git a/windows/memory.c b/windows/memory.c index c440c63ded0..9ace0e51b25 100644 --- a/windows/memory.c +++ b/windows/memory.c @@ -76,7 +76,7 @@ toku_realloc(void *p, size_t size) void toku_free(void* p) { - toku_free_counter++; + (void)__sync_fetch_and_add(&toku_free_counter, 1); if (t_free) t_free(p); else