diff --git a/src/loader.c b/src/loader.c index 322b736c8fb..25eaff6ebe8 100644 --- a/src/loader.c +++ b/src/loader.c @@ -158,7 +158,7 @@ int toku_loader_create_loader(DB_ENV *env, loader->i->loader_flags = loader_flags; loader->i->temp_file_template = (char *)toku_malloc(MAX_FILE_SIZE); - int n = snprintf(loader->i->temp_file_template, MAX_FILE_SIZE, "%s/%s%s", env->i->real_data_dir, loader_temp_prefix, loader_temp_suffix); + int n = snprintf(loader->i->temp_file_template, MAX_FILE_SIZE, "%s/%s%s", env->i->real_tmp_dir, loader_temp_prefix, loader_temp_suffix); if ( !(n>0 && ni->brt_loader, - loader->i->env->i->cachetable, - loader->i->env->i->generate_row_for_put, - src_db, - N, - dbs, - descriptors, - (const char **)new_inames_in_env, - compare_functions, - loader->i->temp_file_template, - load_lsn); + loader->i->env->i->cachetable, + loader->i->env->i->generate_row_for_put, + src_db, + N, + dbs, + descriptors, + (const char **)new_inames_in_env, + compare_functions, + loader->i->temp_file_template, + load_lsn); if ( r!=0 ) { toku_free(new_inames_in_env); toku_free(descriptors); diff --git a/src/tests/loader-stress-test.c b/src/tests/loader-stress-test.c index 501d310a8d7..9b6245b3392 100644 --- a/src/tests/loader-stress-test.c +++ b/src/tests/loader-stress-test.c @@ -346,6 +346,7 @@ static void test_loader(DB **dbs) char *free_me = NULL; char *env_dir = ENVDIR; // the default env_dir. +char *tmp_subdir = "tmp.subdir"; static void run_test(void) { @@ -358,8 +359,15 @@ static void run_test(void) r = system(syscmd); CKERR(r); } r = toku_os_mkdir(env_dir, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); - - r = db_env_create(&env, 0); CKERR(r); + { + char len = strlen(env_dir) + strlen(tmp_subdir) + 10; + char tmpdir[len]; + r = snprintf(tmpdir, len, "%s/%s", env_dir, tmp_subdir); + assert(rset_tmp_dir(env, tmp_subdir); CKERR(r); + } r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r); r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r); if ( verbose ) printf("CACHESIZE = %d MB\n", CACHESIZE); diff --git a/src/tests/loader-stress-test.cilk b/src/tests/loader-stress-test.cilk deleted file mode 100644 index f25180f73c1..00000000000 --- a/src/tests/loader-stress-test.cilk +++ /dev/null @@ -1,313 +0,0 @@ -/* -*- mode: C; c-basic-offset: 4 -*- */ -// /usr/local/cilk/bin/cilk++ -Wall -Wextra -Wcast-align -Wno-missing-noreturn -Werror -Wmissing-format-attribute -fvisibility=hidden -fPIC -Wshadow -O0 -g3 -ggdb3 -I. -I../ -I../../newbrt -I../range_tree -I../lock_tree -I../../linux -I../../toku_include -DBRT_LEVEL_STRADDLE_CALLBACK_LOGIC_NOT_READY=1 -D_SVID_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -I../../include -DENVDIR=\"dir.loader-stress-test.cilk\" loader-stress-test.cilk -L.. -ltokudb -Wl,-rpath,.. -o loader-stress-test - -#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved." -#ident "$Id$" - -#include "test.h" -#include "toku_pthread.h" -#include -#include - -DB_ENV *env; -enum {MAX_NAME=128}; -enum {MAX_DBS=256}; -int NUM_DBS=5; -int NUM_ROWS=100000; -int CHECK_RESULTS=0; -enum {MAGIC=311}; - -// -// Functions to create unique key/value pairs, extractors, checkers, ... for each of NUM_DBS -// - -// a is the bit-wise permute table. For DB[i], permute bits as described in a[i] using 'twiddle32' -// inv is the inverse bit-wise permute of a[]. To get the original value from a twiddled value, twiddle32 (again) with inv[] -int a[MAX_DBS][32]; -int inv[MAX_DBS][32]; - -// extract_functions is an array of extract functions. In our case, each entry is extract_i -#if defined(__cilkplusplus) || defined (__cplusplus) -extern "C" { -#endif -int (*extract_functions[MAX_DBS])(int i, DB *db, DBT *primary_key, DBT *primary_value, DBT *secondary_key, DBT *secondary_value, void *extra); - -// rotate right and left functions -static inline unsigned int rotr32(const unsigned int x, const unsigned int num) { - const unsigned int n = num % 32; - return (x >> n) | ( x << (32 - n)); -} -static inline unsigned int rotl32(const unsigned int x, const unsigned int num) { - const unsigned int n = num % 32; - return (x << n) | ( x >> (32 - n)); -} - -static void generate_permute_tables(void) { - int i, j, tmp; - for(int db=0;db> i ) & 1) << a[db][i]; - } - return b; -} - -// permute bits of x based on inverse permute table bitmap -static unsigned int inv_twiddle32(unsigned int x, int db) -{ - unsigned int b = 0; - for(int i=0;i<32;i++) { - b |= (( x >> i ) & 1) << inv[db][i]; - } - return b; -} - -// generate val from key, index -static unsigned int generate_val(int key, int i) { - return rotl32((key + MAGIC), i); -} -static unsigned int pkey_for_val(int key, int i) { - return rotr32(key, i) - MAGIC; -} - -// primary table requires identity function -static int extract_primary(int i, DB *db, - DBT *primary_key, DBT *primary_value, - DBT *secondary_key, DBT *secondary_value, - void *extra) -{ - i = i; - db = db; - extra = extra; - - dbt_init(secondary_key, primary_key->data, primary_key->size); - dbt_init(secondary_value, primary_value->data, primary_value->size); - - return 0; // as per spec -} - -// for DB[i], extract key/value pairs from primary key -static int extract_i(int i, DB *db, - DBT *primary_key, DBT *primary_value, - DBT *secondary_key, DBT *secondary_value, - void *extra) -{ - if ( i == 0 ) { - extract_primary(i,db,primary_key,primary_value,secondary_key,secondary_value,extra); - } - else - { - unsigned int *new_key = (unsigned int *)toku_malloc(sizeof(unsigned int)); - unsigned int *new_val = (unsigned int *)toku_malloc(sizeof(unsigned int)); - - *new_key = twiddle32(*(unsigned int*)primary_key->data, i); - *new_val = generate_val(*(unsigned int*)primary_key->data, i); - - dbt_init(secondary_key, new_key, sizeof(unsigned int)); - dbt_init(secondary_value, new_val, sizeof(unsigned int)); - } - return 0; // as per spec -} -#if defined(__cilkplusplus) || defined(__cplusplus) -} // extern "C" -#endif - -// -// function to generate extract maps -// -static void generate_extract_funcs(void) { - generate_permute_tables(); - for(int i=0;itxn_begin(env, NULL, &txn, 0); - CKERR(r); - - DBC *cursor; - r = dbs[j]->cursor(dbs[j], txn, &cursor, 0); - CKERR(r); - for(int i=0;ic_get(cursor, &key, &val, DB_NEXT); - CKERR(r); - k = *(unsigned int*)key.data; - pkey_for_db_key = (j == 0) ? k : inv_twiddle32(k, j); - v = *(unsigned int*)val.data; - // test that we have the expected keys and values - assert((unsigned int)pkey_for_db_key == (unsigned int)pkey_for_val(v, j)); -// printf(" DB[%d] key = %10u, val = %10u, pkey_for_db_key = %10u, pkey_for_val=%10d\n", j, v, k, pkey_for_db_key, pkey_for_val(v, j)); - } - {printf("."); fflush(stdout);} - r = cursor->c_close(cursor); - CKERR(r); - r = txn->commit(txn, 0); - CKERR(r); - } - printf("\nCheck OK\n"); -} - -static void test_loader(DB **dbs) -{ - int r; - DB_TXN *txn; - DB_LOADER *loader; - - // create and initialize loader - r = env->txn_begin(env, NULL, &txn, 0); - CKERR(r); - r = env->create_loader(env, txn, &loader, NUM_DBS, dbs, extract_functions, NULL); - CKERR(r); - r = loader->set_duplicate_callback(loader, NULL); - CKERR(r); - r = loader->set_poll_function(loader, NULL); - CKERR(r); - - // using loader->put, put values into DB - DBT key, val; - unsigned int k, v; - for(int i=1;i<=NUM_ROWS;i++) { - k = i; - v = generate_val(i, 0); - dbt_init(&key, &k, sizeof(unsigned int)); - dbt_init(&val, &v, sizeof(unsigned int)); - r = loader->put(loader, &key, &val); - CKERR(r); - if ( CHECK_RESULTS) { if((i%10000) == 0){printf("."); fflush(stdout);} } - } - if( CHECK_RESULTS ) {printf("\n"); fflush(stdout);} - - // close the loader - r = loader->close(loader); - CKERR(r); - - r = txn->commit(txn, 0); - CKERR(r); - - // verify the DBs - if ( CHECK_RESULTS ) { - check_results(dbs); - } -} - -static void run_test(void) -{ - int r; - r = system("rm -rf " ENVDIR); CKERR(r); - r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); - - r = db_env_create(&env, 0); CKERR(r); - r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r); - r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r); - int envflags = DB_INIT_LOCK | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE; - r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); - env->set_errfile(env, stderr); - //Disable auto-checkpointing - r = env->checkpointing_set_period(env, 0); CKERR(r); - - DBT desc; - dbt_init(&desc, "foo", sizeof("foo")); - char name[MAX_NAME*2]; - - DB **dbs = (DB**)toku_malloc(sizeof(DB*) * NUM_DBS); - assert(dbs != NULL); - for(int i=0;iset_descriptor(dbs[i], 1, &desc, abort_on_upgrade); CKERR(r); - snprintf(name, sizeof(name), "db_%04x", i); - r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r); - } - - generate_extract_funcs(); - -// printf("running test_loader()\n"); - // -------------------------- // - test_loader(dbs); - // -------------------------- // -// printf("done test_loader()\n"); - - for(int i=0;iclose(dbs[i], 0); CKERR(r); - dbs[i] = NULL; - } - int x[5]; - cilk_for(int j=0;jclose(env, 0); CKERR(r); - toku_free(dbs); - -} - -// ------------ infrastructure ---------- -static void do_args(int argc, char *argv[]); - -int test_main(int argc, char **argv) { - do_args(argc, argv); - run_test(); - return 0; -} - -static void do_args(int argc, char *argv[]) { - int resultcode; - char *cmd = argv[0]; - argc--; argv++; - while (argc>0) { - if (strcmp(argv[0], "-h")==0) { - resultcode=0; - do_usage: - fprintf(stderr, "Usage: -h -c -d -r \n%s\n", cmd); - exit(resultcode); - } else if (strcmp(argv[0], "-d")==0) { - argc--; argv++; - NUM_DBS = atoi(argv[0]); - if ( NUM_DBS > MAX_DBS ) { - fprintf(stderr, "max value for -d field is %d\n", MAX_DBS); - resultcode=1; - goto do_usage; - } - } else if (strcmp(argv[0], "-r")==0) { - argc--; argv++; - NUM_ROWS = atoi(argv[0]); - } else if (strcmp(argv[0], "-c")==0) { - CHECK_RESULTS = 1; - } else { - fprintf(stderr, "Unknown arg: %s\n", argv[0]); - resultcode=1; - goto do_usage; - } - argc--; - argv++; - } -} diff --git a/src/ydb-internal.h b/src/ydb-internal.h index cb2ffe4e43c..d8810956404 100644 --- a/src/ydb-internal.h +++ b/src/ydb-internal.h @@ -70,8 +70,9 @@ struct __toku_db_env_internal { DB *persistent_environment; // Stores environment settings, can be used for upgrade OMT open_dbs; // Stores open db handles, sorted first by dname and then by numerical value of pointer to the db (arbitrarily assigned memory location) - char *real_data_dir; // data dir used when the env is opened (relative to cwd) - char *real_log_dir; // log dir used when the env is opened (relative to cwd) + char *real_data_dir; // data dir used when the env is opened (relative to cwd, or absolute with leading /) + char *real_log_dir; // log dir used when the env is opened (relative to cwd, or absolute with leading /) + char *real_tmp_dir; // tmp dir used for temporary files (relative to cwd, or absoulte with leading /) enum { FS_GREEN = 0, // green zone (we have lots of space) diff --git a/src/ydb.c b/src/ydb.c index 17dc83841d0..2323a9c68b5 100644 --- a/src/ydb.c +++ b/src/ydb.c @@ -464,31 +464,34 @@ static int toku_c_count(DBC *cursor, db_recno_t *count, u_int32_t flags); static int toku_c_close(DBC * c); static void -env_setup_real_data_dir(DB_ENV *env) { - toku_free(env->i->real_data_dir); - env->i->real_data_dir = NULL; +env_setup_real_dir(DB_ENV *env, char **real_dir, const char *nominal_dir) { + toku_free(*real_dir); + *real_dir = NULL; assert(env->i->dir); - if (env->i->data_dir) - env->i->real_data_dir = toku_construct_full_name(2, env->i->dir, env->i->data_dir); + if (nominal_dir) + *real_dir = toku_construct_full_name(2, env->i->dir, nominal_dir); else - env->i->real_data_dir = toku_strdup(env->i->dir); + *real_dir = toku_strdup(env->i->dir); +} + + +static void +env_setup_real_data_dir(DB_ENV *env) { + env_setup_real_dir(env, &env->i->real_data_dir, env->i->data_dir); } static void env_setup_real_log_dir(DB_ENV *env) { - toku_free(env->i->real_log_dir); - env->i->real_log_dir = NULL; - - if (env->i->lg_dir) { - assert(env->i->dir); - env->i->real_log_dir = toku_construct_full_name(2, env->i->dir, env->i->lg_dir); - } else { - assert(env->i->dir); - env->i->real_log_dir = toku_strdup(env->i->dir); - } + env_setup_real_dir(env, &env->i->real_log_dir, env->i->lg_dir); } +static void +env_setup_real_tmp_dir(DB_ENV *env) { + env_setup_real_dir(env, &env->i->real_tmp_dir, env->i->tmp_dir); +} + + static int ydb_do_recovery (DB_ENV *env) { assert(env->i->real_log_dir); @@ -783,6 +786,7 @@ toku_env_open(DB_ENV * env, const char *home, u_int32_t flags, int mode) { env_setup_real_data_dir(env); env_setup_real_log_dir(env); + env_setup_real_tmp_dir(env); BOOL need_rollback_cachefile = FALSE; if (flags & (DB_INIT_TXN | DB_INIT_LOG)) {