mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-02-01 19:41:47 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

FT-548 support upgrade after dirty shutdown of versions 25 through 27

Browse source
This commit is contained in:
Rich Prohaska 2014-09-02 09:55:24 -04:00
parent 6827d3bd51
commit ed9df111a5
10 changed files with 662 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
ft/logger/log-internal.h
View file

@ -188,6 +188,7 @@ struct tokulogger {
int toku_logger_find_next_unused_log_file(const char *directory, long long *result);
int toku_logger_find_logfiles (const char *directory, char ***resultp, int *n_logfiles);
void toku_logger_free_logfiles (char **logfiles, int n_logfiles);
static inline int
txn_has_current_rollback_log(TOKUTXN txn) {

16
ft/logger/log_upgrade.cc
View file

@ -209,10 +209,7 @@ cleanup:
r = toku_logcursor_destroy(&cursor);
assert(r == 0);
cleanup_no_logcursor:
for(int i=0;i<n_logfiles;i++) {
toku_free(logfiles[i]);
}
toku_free(logfiles);
toku_logger_free_logfiles(logfiles, n_logfiles);
FOOTPRINTCAPTURE;
return rval;
}
@ -227,10 +224,6 @@ verify_clean_shutdown_of_log_version(const char *log_dir, uint32_t version, LSN
if (version < TOKU_LOG_VERSION) {
FOOTPRINT(1);
r = verify_clean_shutdown_of_log_version_old(log_dir, last_lsn, last_xid, version);
if (r != 0) {
fprintf(stderr, "Cannot upgrade TokuFT version %d database.", version);
fprintf(stderr, " Previous improper shutdown detected.\n");
}
}
else {
FOOTPRINT(2);
@ -325,6 +318,13 @@ toku_maybe_upgrade_log(const char *env_dir, const char *log_dir, LSN * lsn_of_cl
TXNID last_xid = TXNID_NONE;
r = verify_clean_shutdown_of_log_version(log_dir, version_of_logs_on_disk, &last_lsn, &last_xid);
if (r != 0) {
if (TOKU_LOG_VERSION_25 <= version_of_logs_on_disk && version_of_logs_on_disk <= TOKU_LOG_VERSION_27
&& TOKU_LOG_VERSION_27 == TOKU_LOG_VERSION) {
r = 0; // can do recovery on dirty shutdown
} else {
fprintf(stderr, "Cannot upgrade TokuFT version %d database.", version_of_logs_on_disk);
fprintf(stderr, " Previous improper shutdown detected.\n");
}
goto cleanup;
}
FOOTPRINT(5);

6
ft/logger/logcursor.cc
View file

@ -277,11 +277,7 @@ int toku_logcursor_destroy(TOKULOGCURSOR *lc) {
(*lc)->entry_valid = false;
}
r = lc_close_cur_logfile(*lc);
int lf;
for(lf=0;lf<(*lc)->n_logfiles;lf++) {
if ( (*lc)->logfiles[lf] ) toku_free((*lc)->logfiles[lf]);
}
if ( (*lc)->logfiles ) toku_free((*lc)->logfiles);
toku_logger_free_logfiles((*lc)->logfiles, (*lc)->n_logfiles);
if ( (*lc)->logdir ) toku_free((*lc)->logdir);
if ( (*lc)->buffer ) toku_free((*lc)->buffer);
toku_free(*lc);

5
ft/logger/logfilemgr.cc
View file

@ -186,10 +186,7 @@ int toku_logfilemgr_init(TOKULOGFILEMGR lfm, const char *log_dir, TXNID *last_xi
toku_logfilemgr_add_logfile_info(lfm, lf_info);
toku_logcursor_destroy(&cursor);
}
for(int i=0;i<n_logfiles;i++) {
toku_free(logfiles[i]);
}
toku_free(logfiles);
toku_logger_free_logfiles(logfiles, n_logfiles);
*last_xid_if_clean_shutdown = last_xid;
return 0;
}

30
ft/logger/logger.cc
View file

@ -421,7 +421,7 @@ wait_till_output_available (TOKULOGGER logger)
// Implementation hint: Use a pthread_cond_wait.
// Entry: Holds the output_condition_lock (but not the inlock)
// Exit: Holds the output_condition_lock and logger->output_is_available
//
//
{
tokutime_t t0 = toku_time_now();
while (!logger->output_is_available) {
@ -490,7 +490,7 @@ release_output (TOKULOGGER logger, LSN fsynced_lsn)
toku_cond_broadcast(&logger->output_condition);
toku_mutex_unlock(&logger->output_condition_lock);
}
static void
swap_inbuf_outbuf (TOKULOGGER logger)
// Effect: Swap the inbuf and outbuf
@ -693,7 +693,7 @@ int toku_logger_find_logfiles (const char *directory, char ***resultp, int *n_lo
while ((de=readdir(d))) {
uint64_t thisl;
uint32_t version_ignore;
if ( !(is_a_logfile_any_version(de->d_name, &thisl, &version_ignore)) ) continue; //#2424: Skip over files that don't match the exact logfile template
if ( !(is_a_logfile_any_version(de->d_name, &thisl, &version_ignore)) ) continue; //#2424: Skip over files that don't match the exact logfile template
if (n_results+1>=result_limit) {
result_limit*=2;
XREALLOC_N(result_limit, result);
@ -707,7 +707,7 @@ int toku_logger_find_logfiles (const char *directory, char ***resultp, int *n_lo
// which are one character longer than old log file names ("xxx.tokulog2"). The comparison function
// won't look beyond the terminating NUL, so an extra character in the comparison string doesn't matter.
// Allow room for terminating NUL after "xxx.tokulog13" even if result[0] is of form "xxx.tokulog2."
int width = sizeof(result[0]+2);
int width = sizeof(result[0]+2);
qsort(result, n_results, width, logfilenamecompare);
*resultp = result;
*n_logfiles = n_results;
@ -715,6 +715,12 @@ int toku_logger_find_logfiles (const char *directory, char ***resultp, int *n_lo
return d ? closedir(d) : 0;
}
void toku_logger_free_logfiles(char **logfiles, int n_logfiles) {
for (int i = 0; i < n_logfiles; i++)
toku_free(logfiles[i]);
toku_free(logfiles);
}
static int open_logfile (TOKULOGGER logger)
// Entry and Exit: This thread has permission to modify the output.
{
@ -723,7 +729,7 @@ static int open_logfile (TOKULOGGER logger)
snprintf(fname, fnamelen, "%s/log%012lld.tokulog%d", logger->directory, logger->next_log_file_number, TOKU_LOG_VERSION);
long long index = logger->next_log_file_number;
if (logger->write_log_files) {
logger->fd = open(fname, O_CREAT+O_WRONLY+O_TRUNC+O_EXCL+O_BINARY, S_IRUSR+S_IWUSR);
logger->fd = open(fname, O_CREAT+O_WRONLY+O_TRUNC+O_EXCL+O_BINARY, S_IRUSR+S_IWUSR);
if (logger->fd==-1) {
return get_error_errno();
}
@ -741,7 +747,7 @@ static int open_logfile (TOKULOGGER logger)
if ( logger->write_log_files ) {
TOKULOGFILEINFO XMALLOC(lf_info);
lf_info->index = index;
lf_info->maxlsn = logger->written_lsn;
lf_info->maxlsn = logger->written_lsn;
lf_info->version = TOKU_LOG_VERSION;
toku_logfilemgr_add_logfile_info(logger->logfilemgr, lf_info);
}
@ -770,7 +776,7 @@ void toku_logger_maybe_trim_log(TOKULOGGER logger, LSN trim_lsn)
int n_logfiles = toku_logfilemgr_num_logfiles(lfm);
TOKULOGFILEINFO lf_info = NULL;
if ( logger->write_log_files && logger->trim_log_files) {
while ( n_logfiles > 1 ) { // don't delete current logfile
uint32_t log_version;
@ -850,7 +856,7 @@ void toku_logger_maybe_fsync(TOKULOGGER logger, LSN lsn, int do_fsync, bool hold
}
static void
logger_write_buffer(TOKULOGGER logger, LSN *fsynced_lsn)
logger_write_buffer(TOKULOGGER logger, LSN *fsynced_lsn)
// Entry: Holds the input lock and permission to modify output.
// Exit: Holds only the permission to modify output.
// Effect: Write the buffers to the output. If DO_FSYNC is true, then fsync.
@ -878,7 +884,7 @@ int toku_logger_restart(TOKULOGGER logger, LSN lastlsn)
// close the log file
if ( logger->write_log_files) { // fsyncs don't work to /dev/null
toku_file_fsync_without_accounting(logger->fd);
toku_file_fsync_without_accounting(logger->fd);
}
r = close(logger->fd); assert(r == 0);
logger->fd = -1;
@ -901,7 +907,7 @@ void toku_logger_log_fcreate (TOKUTXN txn, const char *fname, FILENUM filenum, u
if (txn) {
BYTESTRING bs_fname = { .len = (uint32_t) strlen(fname), .data = (char *) fname };
// fsync log on fcreate
toku_log_fcreate (txn->logger, (LSN*)0, 1, txn, toku_txn_get_txnid(txn), filenum,
toku_log_fcreate (txn->logger, (LSN*)0, 1, txn, toku_txn_get_txnid(txn), filenum,
bs_fname, mode, treeflags, nodesize, basementnodesize, compression_method);
}
}
@ -1339,7 +1345,7 @@ int toku_logger_log_archive (TOKULOGGER logger, char ***logs_p, int flags) {
for (i=all_n_logs-2; i>=0; i--) { // start at all_n_logs-2 because we never archive the most recent log
r = peek_at_log(logger, all_logs[i], &earliest_lsn_in_logfile);
if (r!=0) continue; // In case of error, just keep going
if (earliest_lsn_in_logfile.lsn <= save_lsn.lsn) {
break;
}
@ -1428,7 +1434,7 @@ toku_logger_get_status(TOKULOGGER logger, LOGGER_STATUS statp) {
//////////////////////////////////////////////////////////////////////////////////////////////////////
// Used for upgrade:
// Used for upgrade:
// if any valid log files exist in log_dir, then
// set *found_any_logs to true and set *version_found to version number of latest log
int

4
ft/logger/logger.h
View file

@ -103,6 +103,10 @@ enum {
TOKU_LOG_VERSION_2 = 2,
//After 2 we linked the log version to the FT_LAYOUT VERSION.
//So it went from 2 to 13 (3-12 do not exist)
TOKU_LOG_VERSION_24 = 24,
TOKU_LOG_VERSION_25 = 25, // change rollinclude rollback log entry
TOKU_LOG_VERSION_26 = 26, // no change from 25
TOKU_LOG_VERSION_27 = 27, // no change from 26
TOKU_LOG_VERSION = FT_LAYOUT_VERSION,
TOKU_LOG_MIN_SUPPORTED_VERSION = FT_LAYOUT_MIN_SUPPORTED_VERSION,
};

11
ft/tests/CMakeLists.txt
View file

@ -101,6 +101,17 @@ if(BUILD_TESTING OR BUILD_FT_TESTS)
set_property(TEST ft/upgrade_test_simple APPEND PROPERTY ENVIRONMENT "TOKUDB_DATA=${TOKUDB_DATA}")
# should be a file GLOB and a loop
declare_custom_tests(test-upgrade-recovery-logs)
add_ft_test_aux(test-upgrade-recovery-logs-24-clean test-upgrade-recovery-logs ${TOKUDB_DATA}/upgrade-recovery-logs-24-clean)
add_ft_test_aux(test-upgrade-recovery-logs-24-dirty test-upgrade-recovery-logs ${TOKUDB_DATA}/upgrade-recovery-logs-24-dirty)
add_ft_test_aux(test-upgrade-recovery-logs-25-clean test-upgrade-recovery-logs ${TOKUDB_DATA}/upgrade-recovery-logs-25-clean)
add_ft_test_aux(test-upgrade-recovery-logs-25-dirty test-upgrade-recovery-logs ${TOKUDB_DATA}/upgrade-recovery-logs-25-dirty)
add_ft_test_aux(test-upgrade-recovery-logs-26-clean test-upgrade-recovery-logs ${TOKUDB_DATA}/upgrade-recovery-logs-26-clean)
add_ft_test_aux(test-upgrade-recovery-logs-26-dirty test-upgrade-recovery-logs ${TOKUDB_DATA}/upgrade-recovery-logs-26-dirty)
add_ft_test_aux(test-upgrade-recovery-logs-27-clean test-upgrade-recovery-logs ${TOKUDB_DATA}/upgrade-recovery-logs-27-clean)
add_ft_test_aux(test-upgrade-recovery-logs-27-dirty test-upgrade-recovery-logs ${TOKUDB_DATA}/upgrade-recovery-logs-27-dirty)
## give some tests, that time out normally, 1 hour to complete
set(long_tests
ft/ftloader-test-extractor-3a

151
ft/tests/generate-upgrade-recovery-logs.cc Normal file
View file

@ -0,0 +1,151 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuFT, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// Generate a recovery log with a checkpoint and an optional shutdown log entry.
// These logs will be used later to test recovery.
#include "test.h"
static void generate_recovery_log(const char *testdir, bool do_shutdown) {
int r;
// setup the test dir
toku_os_recursive_delete(testdir);
r = toku_os_mkdir(testdir, S_IRWXU);
CKERR(r);
// open the log
TOKULOGGER logger;
r = toku_logger_create(&logger);
CKERR(r);
r = toku_logger_open(testdir, logger);
CKERR(r);
// log checkpoint
LSN beginlsn;
toku_log_begin_checkpoint(logger, &beginlsn, false, 0, 0);
toku_log_end_checkpoint(logger, nullptr, false, beginlsn, 0, 0, 0);
// log shutdown
if (do_shutdown) {
toku_log_shutdown(logger, nullptr, true, 0, 0);
}
r = toku_logger_close(&logger);
CKERR(r);
}
int test_main(int argc, const char *argv[]) {
bool do_shutdown = true;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-v") == 0) {
verbose++;
continue;
}
if (strcmp(argv[i], "-q") == 0) {
if (verbose > 0)
verbose--;
continue;
}
if (strcmp(argv[i], "--clean") == 0) {
do_shutdown = true;
continue;
}
if (strcmp(argv[i], "--dirty") == 0) {
do_shutdown = false;
continue;
}
}
char testdir[256];
sprintf(testdir, "upgrade-recovery-logs-%d-%s", TOKU_LOG_VERSION, do_shutdown ? "clean" : "dirty");
generate_recovery_log(testdir, do_shutdown);
return 0;
}

192
ft/tests/test-upgrade-recovery-logs.cc Normal file
View file

@ -0,0 +1,192 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuFT, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// Test that recovery works correctly on a recovery log in a log directory.
#include "test.h"
static void run_recovery(const char *testdir) {
int r;
int log_version;
char shutdown[32+1];
r = sscanf(testdir, "upgrade-recovery-logs-%d-%32s", &log_version, shutdown);
assert(r == 2);
char **logfiles = nullptr;
int n_logfiles = 0;
r = toku_logger_find_logfiles(testdir, &logfiles, &n_logfiles);
CKERR(r);
assert(n_logfiles > 0);
FILE *f = fopen(logfiles[n_logfiles-1], "r");
assert(f);
uint32_t real_log_version;
r = toku_read_logmagic(f, &real_log_version);
CKERR(r);
assert((uint32_t)log_version == (uint32_t)real_log_version);
r = fclose(f);
CKERR(r);
toku_logger_free_logfiles(logfiles, n_logfiles);
// test needs recovery
r = tokuft_needs_recovery(testdir, false);
if (strcmp(shutdown, "clean") == 0) {
CKERR(r); // clean does not need recovery
} else if (strcmp(shutdown, "dirty") == 0) {
CKERR2(r, 1); // dirty needs recovery
} else {
CKERR(EINVAL);
}
// test maybe upgrade log
LSN lsn_of_clean_shutdown;
bool upgrade_in_progress;
r = toku_maybe_upgrade_log(testdir, testdir, &lsn_of_clean_shutdown, &upgrade_in_progress);
if (strcmp(shutdown, "dirty") == 0 && log_version <= 24) {
CKERR2(r, TOKUDB_UPGRADE_FAILURE); // we dont support dirty upgrade from versions <= 24
return;
} else {
CKERR(r);
}
if (!verbose) {
// redirect stderr
int devnul = open(DEV_NULL_FILE, O_WRONLY);
assert(devnul >= 0);
int rr = toku_dup2(devnul, fileno(stderr));
assert(rr == fileno(stderr));
rr = close(devnul);
assert(rr == 0);
}
// run recovery
if (r == 0) {
r = tokuft_recover(NULL,
NULL_prepared_txn_callback,
NULL_keep_cachetable_callback,
NULL_logger, testdir, testdir, 0, 0, 0, NULL, 0);
CKERR(r);
}
}
int test_main(int argc, const char *argv[]) {
int i = 0;
for (i = 1; i < argc; i++) {
if (strcmp(argv[i], "-v") == 0) {
verbose++;
continue;
}
if (strcmp(argv[i], "-q") == 0) {
if (verbose > 0)
verbose--;
continue;
}
break;
}
if (i < argc) {
const char *full_test_dir = argv[i];
const char *test_dir = basename(full_test_dir);
if (strcmp(full_test_dir, test_dir) != 0) {
int r;
char cmd[32 + strlen(full_test_dir) + strlen(test_dir)];
sprintf(cmd, "rm -rf %s", test_dir);
r = system(cmd);
CKERR(r);
sprintf(cmd, "cp -r %s %s", full_test_dir, test_dir);
r = system(cmd);
CKERR(r);
}
run_recovery(test_dir);
}
return 0;
}

275
src/tests/recover-rollinclude.cc Normal file
View file

@ -0,0 +1,275 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
This software is covered by US Patent No. 8,489,638.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// Create a rollback log with a rollinclude log entry, crash after the txn commits and before the last checkpoint.
// Recovery crashes 7.1.0, should succeed.
#include "test.h"
#include <endian.h>
// Insert max_rows key/val pairs into the db
// We want to force a rollinclude so we use a child transaction and insert enough rows so that it spills.
// It spills at about 144K and 289K rows.
static void do_inserts(DB_ENV *env, DB *db, uint64_t max_rows, size_t val_size) {
char val_data[val_size]; memset(val_data, 0, val_size);
int r;
DB_TXN *parent = nullptr;
r = env->txn_begin(env, nullptr, &parent, 0);
CKERR(r);
DB_TXN *child = nullptr;
r = env->txn_begin(env, parent, &child, 0);
CKERR(r);
for (uint64_t i = 0; i < max_rows; i++) {
// pick a sequential key but it does not matter for this test.
uint64_t k[2] = {
htonl(i), random64(),
};
DBT key = { .data = k, .size = sizeof k };
DBT val = { .data = val_data, .size = (uint32_t) val_size };
r = db->put(db, child, &key, &val, 0);
CKERR(r);
if (i == max_rows-1) {
r = child->commit(child, 0);
CKERR(r);
r = env->txn_checkpoint(env, 0, 0, 0);
CKERR(r);
}
}
r = parent->commit(parent, 0);
CKERR(r);
}
static void run_test(uint64_t num_rows, size_t val_size, bool do_crash) {
int r;
DB_ENV *env = nullptr;
r = db_env_create(&env, 0);
CKERR(r);
r = env->set_cachesize(env, 8, 0, 1);
CKERR(r);
r = env->open(env, TOKU_TEST_FILENAME,
DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE,
S_IRWXU+S_IRWXG+S_IRWXO);
CKERR(r);
DB *db = nullptr;
r = db_create(&db, env, 0);
CKERR(r);
r = db->open(db, nullptr, "foo.db", 0, DB_BTREE, DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO);
CKERR(r);
r = env->txn_checkpoint(env, 0, 0, 0);
CKERR(r);
do_inserts(env, db, num_rows, val_size);
if (do_crash)
assert(0); // crash on purpose
r = db->close(db, 0);
CKERR(r);
r = env->close(env, 0);
CKERR(r);
}
static void do_verify(DB_ENV *env, DB *db, uint64_t num_rows, size_t val_size UU()) {
int r;
DB_TXN *txn = nullptr;
r = env->txn_begin(env, nullptr, &txn, 0);
CKERR(r);
DBC *c = nullptr;
r = db->cursor(db, txn, &c, 0);
CKERR(r);
uint64_t i = 0;
while (1) {
DBT key = {};
DBT val = {};
r = c->c_get(c, &key, &val, DB_NEXT);
if (r == DB_NOTFOUND)
break;
CKERR(r);
assert(key.size == 16);
uint64_t k[2];
memcpy(k, key.data, key.size);
assert(htonl(k[0]) == i);
assert(val.size == val_size);
i++;
}
assert(i == num_rows);
r = c->c_close(c);
CKERR(r);
r = txn->commit(txn, 0);
CKERR(r);
}
static void run_recover(uint64_t num_rows, size_t val_size) {
int r;
DB_ENV *env = nullptr;
r = db_env_create(&env, 0);
CKERR(r);
r = env->set_cachesize(env, 8, 0, 1);
CKERR(r);
r = env->open(env, TOKU_TEST_FILENAME,
DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE | DB_RECOVER,
S_IRWXU+S_IRWXG+S_IRWXO);
CKERR(r);
DB *db = nullptr;
r = db_create(&db, env, 0);
CKERR(r);
r = db->open(db, nullptr, "foo.db", 0, DB_BTREE, 0, S_IRWXU+S_IRWXG+S_IRWXO);
CKERR(r);
do_verify(env, db, num_rows, val_size);
r = db->close(db, 0);
CKERR(r);
r = env->close(env, 0);
CKERR(r);
}
int test_main (int argc, char *const argv[]) {
bool do_test = false;
bool do_recover = false;
bool do_crash = true;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-v") == 0) {
verbose++;
continue;
}
if (strcmp(argv[i], "-q") == 0) {
if (verbose > 0) verbose--;
continue;
}
if (strcmp(argv[i], "--test") == 0) {
do_test = true;
continue;
}
if (strcmp(argv[i], "--recover") == 0) {
do_recover = true;
continue;
}
if (strcmp(argv[i], "--crash") == 0 && i+1 < argc) {
do_crash = atoi(argv[++i]);
continue;
}
}
uint64_t num_rows = 300000;
size_t val_size = 1;
if (do_test) {
// init the env directory
toku_os_recursive_delete(TOKU_TEST_FILENAME);
int r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO);
CKERR(r);
run_test(num_rows, val_size, do_crash);
}
if (do_recover) {
run_recover(num_rows, val_size);
}
return 0;
}