mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-02-02 12:01:42 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refs Tokutek/ft-index#26 Fixes Tokutek/ft-index#31 Change api for *_multiple to support array indexing. Never call put/del_callback functions for src_db.

Browse source
This commit is contained in:
Yoni Fogel 2013-08-12 14:20:59 -07:00
parent 654e771d60
commit d0642efd6a
90 changed files with 2792 additions and 1212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
buildheader/make_tdb.cc
View file

@ -431,18 +431,18 @@ static void print_db_env_struct (void) {
"int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags)",
"int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,\n"
" const DBT *src_key, const DBT *src_val,\n"
" uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array) /* insert into multiple DBs */",
" uint32_t num_dbs, DB **db_array, DBT_ARRAY *keys, DBT_ARRAY *vals, uint32_t *flags_array) /* insert into multiple DBs */",
"int (*set_generate_row_callback_for_put) (DB_ENV *env, generate_row_for_put_func generate_row_for_put)",
"int (*del_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,\n"
" const DBT *src_key, const DBT *src_val,\n"
" uint32_t num_dbs, DB **db_array, DBT *keys, uint32_t *flags_array) /* delete from multiple DBs */",
" uint32_t num_dbs, DB **db_array, DBT_ARRAY *keys, uint32_t *flags_array) /* delete from multiple DBs */",
"int (*set_generate_row_callback_for_del) (DB_ENV *env, generate_row_for_del_func generate_row_for_del)",
"int (*update_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,\n"
" DBT *old_src_key, DBT *old_src_data,\n"
" DBT *new_src_key, DBT *new_src_data,\n"
" uint32_t num_dbs, DB **db_array, uint32_t *flags_array,\n"
" uint32_t num_keys, DBT *keys,\n"
" uint32_t num_vals, DBT *vals) /* update multiple DBs */",
" uint32_t num_keys, DBT_ARRAY *keys,\n"
" uint32_t num_vals, DBT_ARRAY *vals) /* update multiple DBs */",
"int (*get_redzone) (DB_ENV *env, int *redzone) /* get the redzone limit */",
"int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */",
"int (*set_lk_max_memory) (DB_ENV *env, uint64_t max)",
@ -747,8 +747,17 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
print_dbtype();
print_defines();
printf("typedef int (*generate_row_for_put_func)(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val);\n");
printf("typedef int (*generate_row_for_del_func)(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_val);\n");
printf("typedef struct {\n");
printf(" uint32_t capacity;\n");
printf(" uint32_t size;\n");
printf(" DBT *dbts;\n");
printf("} DBT_ARRAY;\n\n");
printf("typedef int (*generate_row_for_put_func)(DB *dest_db, DB *src_db, DBT_ARRAY * dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val);\n");
printf("typedef int (*generate_row_for_del_func)(DB *dest_db, DB *src_db, DBT_ARRAY * dest_keys, const DBT *src_key, const DBT *src_val);\n");
printf("DBT_ARRAY * toku_dbt_array_init(DBT_ARRAY *dbts, uint32_t size) %s;\n", VISIBLE);
printf("void toku_dbt_array_destroy(DBT_ARRAY *dbts) %s;\n", VISIBLE);
printf("void toku_dbt_array_destroy_shallow(DBT_ARRAY *dbts) %s;\n", VISIBLE);
printf("void toku_dbt_array_resize(DBT_ARRAY *dbts, uint32_t size) %s;\n", VISIBLE);
print_db_env_struct();
print_db_key_range_struct();

101
ft/ftloader.cc
View file

@ -1250,6 +1250,12 @@ static int process_primary_rows_internal (FTLOADER bl, struct rowset *primary_ro
int error_count = 0;
int *XMALLOC_N(bl->N, error_codes);
// If we parallelize the first for loop, dest_keys/dest_vals init&cleanup need to move inside
DBT_ARRAY dest_keys;
DBT_ARRAY dest_vals;
toku_dbt_array_init(&dest_keys, 1);
toku_dbt_array_init(&dest_vals, 1);
for (int i = 0; i < bl->N; i++) {
unsigned int klimit,vlimit; // maximum row sizes.
toku_ft_get_maximum_advised_key_value_lengths(&klimit, &vlimit);
@ -1259,12 +1265,8 @@ static int process_primary_rows_internal (FTLOADER bl, struct rowset *primary_ro
struct merge_fileset *fs = &(bl->fs[i]);
ft_compare_func compare = bl->bt_compare_funs[i];
DBT skey = zero_dbt;
skey.flags = DB_DBT_REALLOC;
DBT sval=skey;
// Don't parallelize this loop, or we have to lock access to add_row() which would be a lot of overehad.
// Also this way we can reuse the DB_DBT_REALLOC'd value inside skey and sval without a race.
// Also this way we can reuse the DB_DBT_REALLOC'd values inside dest_keys/dest_vals without a race.
for (size_t prownum=0; prownum<primary_rowset->n_rows; prownum++) {
if (error_count) break;
@ -1276,23 +1278,32 @@ static int process_primary_rows_internal (FTLOADER bl, struct rowset *primary_ro
pval.data = primary_rowset->data + prow->off + prow->klen;
pval.size = prow->vlen;
DBT *dest_key = &skey;
DBT *dest_val = &sval;
{
int r;
if (bl->dbs[i] != bl->src_db) {
r = bl->generate_row_for_put(bl->dbs[i], bl->src_db, dest_key, dest_val, &pkey, &pval);
if (r != 0) {
error_codes[i] = r;
inc_error_count();
break;
}
} else {
dest_key = &pkey;
dest_val = &pval;
DBT_ARRAY key_array;
DBT_ARRAY val_array;
if (bl->dbs[i] != bl->src_db) {
int r = bl->generate_row_for_put(bl->dbs[i], bl->src_db, &dest_keys, &dest_vals, &pkey, &pval);
if (r != 0) {
error_codes[i] = r;
inc_error_count();
break;
}
paranoid_invariant(dest_keys.size <= dest_keys.capacity);
paranoid_invariant(dest_vals.size <= dest_vals.capacity);
paranoid_invariant(dest_keys.size == dest_vals.size);
key_array = dest_keys;
val_array = dest_vals;
} else {
key_array.size = key_array.capacity = 1;
key_array.dbts = &pkey;
val_array.size = val_array.capacity = 1;
val_array.dbts = &pval;
}
for (uint32_t row = 0; row < key_array.size; row++) {
DBT *dest_key = &key_array.dbts[row];
DBT *dest_val = &val_array.dbts[row];
if (dest_key->size > klimit) {
error_codes[i] = EINVAL;
fprintf(stderr, "Key too big (keysize=%d bytes, limit=%d bytes)\n", dest_key->size, klimit);
@ -1305,51 +1316,31 @@ static int process_primary_rows_internal (FTLOADER bl, struct rowset *primary_ro
inc_error_count();
break;
}
}
bl->extracted_datasizes[i] += ft_loader_leafentry_size(dest_key->size, dest_val->size, leafentry_xid(bl, i));
bl->extracted_datasizes[i] += ft_loader_leafentry_size(dest_key->size, dest_val->size, leafentry_xid(bl, i));
if (row_wont_fit(rows, dest_key->size + dest_val->size)) {
//printf("%s:%d rows.n_rows=%ld rows.n_bytes=%ld\n", __FILE__, __LINE__, rows->n_rows, rows->n_bytes);
int r = sort_and_write_rows(*rows, fs, bl, i, bl->dbs[i], compare); // cannot spawn this because of the race on rows. If we were to create a new rows, and if sort_and_write_rows were to destroy the rows it is passed, we could spawn it, however.
// If we do spawn this, then we must account for the additional storage in the memory_per_rowset() function.
init_rowset(rows, memory_per_rowset_during_extract(bl)); // we passed the contents of rows to sort_and_write_rows.
if (row_wont_fit(rows, dest_key->size + dest_val->size)) {
//printf("%s:%d rows.n_rows=%ld rows.n_bytes=%ld\n", __FILE__, __LINE__, rows->n_rows, rows->n_bytes);
int r = sort_and_write_rows(*rows, fs, bl, i, bl->dbs[i], compare); // cannot spawn this because of the race on rows. If we were to create a new rows, and if sort_and_write_rows were to destroy the rows it is passed, we could spawn it, however.
// If we do spawn this, then we must account for the additional storage in the memory_per_rowset() function.
init_rowset(rows, memory_per_rowset_during_extract(bl)); // we passed the contents of rows to sort_and_write_rows.
if (r != 0) {
error_codes[i] = r;
inc_error_count();
break;
}
}
int r = add_row(rows, dest_key, dest_val);
if (r != 0) {
error_codes[i] = r;
inc_error_count();
break;
}
}
int r = add_row(rows, dest_key, dest_val);
if (r != 0) {
error_codes[i] = r;
inc_error_count();
break;
}
//flags==0 means generate_row_for_put callback changed it
//(and freed any memory necessary to do so) so that values are now stored
//in temporary memory that does not need to be freed. We need to continue
//using DB_DBT_REALLOC however.
if (skey.flags == 0) {
toku_init_dbt(&skey);
skey.flags = DB_DBT_REALLOC;
}
if (sval.flags == 0) {
toku_init_dbt(&sval);
sval.flags = DB_DBT_REALLOC;
}
}
if (bl->dbs[i] != bl->src_db) {
if (skey.flags) {
toku_free(skey.data); skey.data = NULL;
}
if (sval.flags) {
toku_free(sval.data); sval.data = NULL;
}
}
}
toku_dbt_array_destroy(&dest_keys);
toku_dbt_array_destroy(&dest_vals);
destroy_rowset(primary_rowset);
toku_free(primary_rowset);

75
ft/recover.cc
View file

@ -187,6 +187,8 @@ struct recover_env {
ft_update_func update_function;
generate_row_for_put_func generate_row_for_put;
generate_row_for_del_func generate_row_for_del;
DBT_ARRAY dest_keys;
DBT_ARRAY dest_vals;
struct scan_state ss;
struct file_map fmap;
bool goforward;
@ -306,6 +308,8 @@ static int recover_env_init (RECOVER_ENV renv,
file_map_init(&renv->fmap);
renv->goforward = false;
renv->cp = toku_cachetable_get_checkpointer(renv->ct);
toku_dbt_array_init(&renv->dest_keys, 1);
toku_dbt_array_init(&renv->dest_vals, 1);
if (tokudb_recovery_trace)
fprintf(stderr, "%s:%d\n", __FUNCTION__, __LINE__);
return r;
@ -330,6 +334,8 @@ static void recover_env_cleanup (RECOVER_ENV renv) {
} else {
toku_cachetable_close(&renv->ct);
}
toku_dbt_array_destroy(&renv->dest_keys);
toku_dbt_array_destroy(&renv->dest_vals);
if (tokudb_recovery_trace)
fprintf(stderr, "%s:%d\n", __FUNCTION__, __LINE__);
@ -1033,11 +1039,10 @@ static int toku_recover_enq_insert_multiple (struct logtype_enq_insert_multiple
}
if (do_inserts) {
DBT src_key, src_val, dest_key, dest_val;
DBT src_key, src_val;
toku_fill_dbt(&src_key, l->src_key.data, l->src_key.len);
toku_fill_dbt(&src_val, l->src_val.data, l->src_val.len);
toku_init_dbt_flags(&dest_key, DB_DBT_REALLOC);
toku_init_dbt_flags(&dest_val, DB_DBT_REALLOC);
for (uint32_t file = 0; file < l->dest_filenums.num; file++) {
struct file_map_tuple *tuple = NULL;
@ -1045,23 +1050,29 @@ static int toku_recover_enq_insert_multiple (struct logtype_enq_insert_multiple
if (r==0) {
// We found the cachefile. (maybe) Do the insert.
DB *db = &tuple->fake_db;
r = renv->generate_row_for_put(db, src_db, &dest_key, &dest_val, &src_key, &src_val);
assert(r==0);
toku_ft_maybe_insert(tuple->ft_handle, &dest_key, &dest_val, txn, true, l->lsn, false, FT_INSERT);
//flags==0 means generate_row_for_put callback changed it
//(and freed any memory necessary to do so) so that values are now stored
//in temporary memory that does not need to be freed. We need to continue
//using DB_DBT_REALLOC however.
if (dest_key.flags == 0)
toku_init_dbt_flags(&dest_key, DB_DBT_REALLOC);
if (dest_val.flags == 0)
toku_init_dbt_flags(&dest_val, DB_DBT_REALLOC);
DBT_ARRAY key_array;
DBT_ARRAY val_array;
if (db != src_db) {
r = renv->generate_row_for_put(db, src_db, &renv->dest_keys, &renv->dest_vals, &src_key, &src_val);
assert(r==0);
invariant(renv->dest_keys.size <= renv->dest_keys.capacity);
invariant(renv->dest_vals.size <= renv->dest_vals.capacity);
invariant(renv->dest_keys.size == renv->dest_vals.size);
key_array = renv->dest_keys;
val_array = renv->dest_vals;
} else {
key_array.size = key_array.capacity = 1;
key_array.dbts = &src_key;
val_array.size = val_array.capacity = 1;
val_array.dbts = &src_val;
}
for (uint32_t i = 0; i < key_array.size; i++) {
toku_ft_maybe_insert(tuple->ft_handle, &key_array.dbts[i], &val_array.dbts[i], txn, true, l->lsn, false, FT_INSERT);
}
}
}
if (dest_key.data) toku_free(dest_key.data); //TODO: #2321 May need windows hack
if (dest_val.data) toku_free(dest_val.data); //TODO: #2321 May need windows hack
}
return 0;
@ -1085,18 +1096,18 @@ static int toku_recover_enq_delete_multiple (struct logtype_enq_delete_multiple
if (l->src_filenum.fileid == FILENUM_NONE.fileid)
assert(r==DB_NOTFOUND);
else {
if (r == 0)
if (r == 0) {
src_db = &tuple->fake_db;
else
} else {
do_deletes = false; // src file was probably deleted, #3129
}
}
}
if (do_deletes) {
DBT src_key, src_val, dest_key;
DBT src_key, src_val;
toku_fill_dbt(&src_key, l->src_key.data, l->src_key.len);
toku_fill_dbt(&src_val, l->src_val.data, l->src_val.len);
toku_init_dbt_flags(&dest_key, DB_DBT_REALLOC);
for (uint32_t file = 0; file < l->dest_filenums.num; file++) {
struct file_map_tuple *tuple = NULL;
@ -1104,18 +1115,22 @@ static int toku_recover_enq_delete_multiple (struct logtype_enq_delete_multiple
if (r==0) {
// We found the cachefile. (maybe) Do the delete.
DB *db = &tuple->fake_db;
r = renv->generate_row_for_del(db, src_db, &dest_key, &src_key, &src_val);
assert(r==0);
toku_ft_maybe_delete(tuple->ft_handle, &dest_key, txn, true, l->lsn, false);
//flags==0 indicates the return values are stored in temporary memory that does
//not need to be freed. We need to continue using DB_DBT_REALLOC however.
if (dest_key.flags == 0)
toku_init_dbt_flags(&dest_key, DB_DBT_REALLOC);
DBT_ARRAY key_array;
if (db != src_db) {
r = renv->generate_row_for_del(db, src_db, &renv->dest_keys, &src_key, &src_val);
assert(r==0);
invariant(renv->dest_keys.size <= renv->dest_keys.capacity);
key_array = renv->dest_keys;
} else {
key_array.size = key_array.capacity = 1;
key_array.dbts = &src_key;
}
for (uint32_t i = 0; i < key_array.size; i++) {
toku_ft_maybe_delete(tuple->ft_handle, &key_array.dbts[i], txn, true, l->lsn, false);
}
}
}
if (dest_key.flags & DB_DBT_REALLOC && dest_key.data) toku_free(dest_key.data); //TODO: #2321 May need windows hack
}
return 0;

15
ft/tests/ftloader-test-bad-generate.cc
View file

@ -100,15 +100,12 @@ PATENT RIGHTS GRANT:
#include <portability/toku_path.h>
static void copy_dbt(DBT *dest, const DBT *src) {
assert(dest->flags & DB_DBT_REALLOC);
dest->data = toku_realloc(dest->data, src->size);
dest->size = src->size;
memcpy(dest->data, src->data, src->size);
}
static int generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
if (verbose) printf("%s %p %p %p %p %p %p\n", __FUNCTION__, dest_db, src_db, dest_key, dest_val, src_key, src_val);
static int generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
if (verbose) printf("%s %p %p %p %p %p %p\n", __FUNCTION__, dest_db, src_db, dest_keys, dest_vals, src_key, src_val);
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
assert(dest_db == NULL); assert(src_db == NULL);

17
ft/tests/ftloader-test-extractor-errors.cc
View file

@ -101,18 +101,13 @@ PATENT RIGHTS GRANT:
#include <portability/toku_path.h>
static void copy_dbt(DBT *dest, const DBT *src) {
assert(dest->flags & DB_DBT_REALLOC);
dest->data = toku_realloc(dest->data, src->size);
dest->size = src->size;
memcpy(dest->data, src->data, src->size);
}
static int generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
(void) dest_db; (void) src_db; (void) src_key; (void) src_val;
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
static int generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
copy_dbt(dest_key, src_key);
copy_dbt(dest_val, src_val);
copy_dbt(&dest_keys->dbts[0], src_key);
copy_dbt(&dest_vals->dbts[0], src_val);
return 0;
}

13
ft/tests/ftloader-test-extractor.cc
View file

@ -330,14 +330,11 @@ static void verify(int inkey[], int nkeys, const char *testdir) {
free_temp_files(tempfiles);
}
static void copy_dbt(DBT *dest, const DBT *src) {
assert(dest->flags & DB_DBT_REALLOC);
dest->data = toku_realloc(dest->data, src->size);
dest->size = src->size;
memcpy(dest->data, src->data, src->size);
}
static int generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
assert(dest_db == NULL); assert(src_db == NULL);
copy_dbt(dest_key, src_key);

8
ft/tests/test.h
View file

@ -349,6 +349,14 @@ public:
}
};
UU()
static void copy_dbt(DBT *dest, const DBT *src) {
assert(dest->flags & DB_DBT_REALLOC);
dest->data = toku_realloc(dest->data, src->size);
dest->size = src->size;
memcpy(dest->data, src->data, src->size);
}
int verbose=0;
static inline void

28
ft/valgrind.suppressions
View file

@ -253,3 +253,31 @@
obj:/usr/lib/system/libsystem_c.dylib
fun:loadlocale
}
{
<ld_is_not_valgrind_clean_in_ubuntu_13.04_1>
Memcheck:Cond
fun:index
fun:expand_dynamic_string_token
fun:_dl_map_object
fun:map_doit
fun:_dl_catch_error
fun:do_preload
fun:dl_main
fun:_dl_sysdep_start
fun:_dl_start
obj:/lib/x86_64-linux-gnu/ld-2.17.so
}
{
<ld_is_not_valgrind_clean_in_ubuntu_13.04_2>
Memcheck:Cond
fun:index
fun:expand_dynamic_string_token
fun:_dl_map_object
fun:map_doit
fun:_dl_catch_error
fun:do_preload
fun:dl_main
fun:_dl_sysdep_start
fun:_dl_start
obj:/lib/x86_64-linux-gnu/ld-2.17.so
}

59
ft/ybt.cc
View file

@ -108,6 +108,65 @@ toku_init_dbt_flags(DBT *ybt, uint32_t flags) {
return ybt;
}
DBT_ARRAY *
toku_dbt_array_init(DBT_ARRAY *dbts, uint32_t size) {
uint32_t capacity = 1;
while (capacity < size) { capacity *= 2; }
XMALLOC_N(capacity, dbts->dbts);
for (uint32_t i = 0; i < capacity; i++) {
toku_init_dbt_flags(&dbts->dbts[i], DB_DBT_REALLOC);
}
dbts->size = size;
dbts->capacity = capacity;
return dbts;
}
void
toku_dbt_array_resize(DBT_ARRAY *dbts, uint32_t size) {
if (size != dbts->size) {
if (size > dbts->capacity) {
const uint32_t old_capacity = dbts->capacity;
uint32_t new_capacity = dbts->capacity;
while (new_capacity < size) {
new_capacity *= 2;
}
dbts->capacity = new_capacity;
XREALLOC_N(new_capacity, dbts->dbts);
for (uint32_t i = old_capacity; i < new_capacity; i++) {
toku_init_dbt_flags(&dbts->dbts[i], DB_DBT_REALLOC);
}
} else if (size < dbts->size) {
if (dbts->capacity >= 8 && size < dbts->capacity / 4) {
const int old_capacity = dbts->capacity;
const int new_capacity = dbts->capacity / 2;
for (int i = new_capacity; i < old_capacity; i++) {
toku_destroy_dbt(&dbts->dbts[i]);
}
XREALLOC_N(new_capacity, dbts->dbts);
dbts->capacity = new_capacity;
}
}
dbts->size = size;
}
}
void
toku_dbt_array_destroy_shallow(DBT_ARRAY *dbts) {
toku_free(dbts->dbts);
ZERO_STRUCT(*dbts);
}
void
toku_dbt_array_destroy(DBT_ARRAY *dbts) {
for (uint32_t i = 0; i < dbts->capacity; i++) {
toku_destroy_dbt(&dbts->dbts[i]);
}
toku_dbt_array_destroy_shallow(dbts);
}
void
toku_destroy_dbt(DBT *dbt) {
switch (dbt->flags) {

67
portability/toku_path.cc
View file

@ -103,18 +103,75 @@ const char *toku_test_filename(const char *default_filename) {
return filename;
}
// Guarantees NUL termination (unless siz == 0)
// siz is full size of dst (including NUL terminator)
// Appends src to end of dst, (truncating if necessary) to use no more than siz bytes (including NUL terminator)
// Returns strnlen(dst, siz) (size (excluding NUL) of string we tried to create)
size_t toku_strlcat(char *dst, const char *src, size_t siz)
{
if (siz == 0) {
return 0;
}
dst[siz-1] = '\0'; //Guarantee NUL termination.
const size_t old_dst_len = strnlen(dst, siz - 1);
paranoid_invariant(old_dst_len <= siz - 1);
if (old_dst_len == siz - 1) {
// No room for anything more.
return old_dst_len;
}
char *d = &dst[old_dst_len]; //Points to null ptr at end of old string
const size_t remaining_space = siz-old_dst_len-1;
const size_t allowed_src_len = strnlen(src, remaining_space); // Limit to remaining space (leave space for NUL)
paranoid_invariant(allowed_src_len <= remaining_space);
paranoid_invariant(old_dst_len + allowed_src_len < siz);
memcpy(d, src, allowed_src_len);
d[allowed_src_len] = '\0'; // NUL terminate (may be redundant with previous NUL termination)
return old_dst_len + allowed_src_len;
}
// Guarantees NUL termination (unless siz == 0)
// siz is full size of dst (including NUL terminator)
// Appends src to end of dst, (truncating if necessary) to use no more than siz bytes (including NUL terminator)
// Returns strnlen(dst, siz) (size (excluding NUL) of string we tried to create)
//
// Implementation note: implemented for simplicity as oppsed to performance
size_t toku_strlcpy(char *dst, const char *src, size_t siz)
{
if (siz == 0) {
return 0;
}
*dst = '\0';
return toku_strlcat(dst, src, siz);
}
char *toku_path_join(char *dest, int n, const char *base, ...) {
static const char PATHSEP = '/';
char *end = stpncpy(dest, base, TOKU_PATH_MAX);
size_t written;
written = toku_strlcpy(dest, base, TOKU_PATH_MAX);
paranoid_invariant(written < TOKU_PATH_MAX);
paranoid_invariant(dest[written] == '\0');
va_list ap;
va_start(ap, base);
for (int i = 1; end - dest < TOKU_PATH_MAX && i < n; ++i) {
if (*(end - 1) != PATHSEP) {
*(end++) = PATHSEP;
for (int i = 1; written < TOKU_PATH_MAX && i < n; ++i) {
if (dest[written - 1] != PATHSEP) {
if (written+2 >= TOKU_PATH_MAX) {
// No room.
break;
}
dest[written++] = PATHSEP;
dest[written] = '\0';
}
const char *next = va_arg(ap, const char *);
end = stpncpy(end, next, TOKU_PATH_MAX - (end - dest));
written = toku_strlcat(dest, next, TOKU_PATH_MAX);
paranoid_invariant(written < TOKU_PATH_MAX);
paranoid_invariant(dest[written] == '\0');
}
va_end(ap);
// Zero out rest of buffer for security
memset(&dest[written], 0, TOKU_PATH_MAX - written);
return dest;
}

13
portability/toku_path.h
View file

@ -93,6 +93,7 @@ PATENT RIGHTS GRANT:
#include <stdarg.h>
#include <limits.h>
#include <sys/types.h>
__attribute__((nonnull))
const char *toku_test_filename(const char *default_filename);
@ -101,6 +102,18 @@ const char *toku_test_filename(const char *default_filename);
#define TOKU_PATH_MAX PATH_MAX
// Guarantees NUL termination (unless siz == 0)
// siz is full size of dst (including NUL terminator)
// Appends src to end of dst, (truncating if necessary) to use no more than siz bytes (including NUL terminator)
// Returns strnlen(dst, siz)
size_t toku_strlcat(char *dst, const char *src, size_t siz);
// Guarantees NUL termination (unless siz == 0)
// siz is full size of dst (including NUL terminator)
// Appends src to end of dst, (truncating if necessary) to use no more than siz bytes (including NUL terminator)
// Returns strnlen(dst, siz)
size_t toku_strlcpy(char *dst, const char *src, size_t siz);
char *toku_path_join(char *dest, int n, const char *base, ...);
// Effect:
// Concatenate all the parts into a filename, using portable path separators.

5
src/export.map
View file

@ -87,6 +87,11 @@
toku_patent_string;
toku_copyright_string;
toku_dbt_array_init;
toku_dbt_array_destroy;
toku_dbt_array_destroy_shallow;
toku_dbt_array_resize;
local: *;
};

5
src/indexer-internal.h
View file

@ -141,7 +141,8 @@ struct __toku_indexer_internal {
// undo state
struct indexer_commit_keys commit_keys; // set of keys to commit
DBT hotkey, hotval; // current hot key and value
DBT_ARRAY *hot_keys;
DBT_ARRAY *hot_vals;
// test functions
int (*undo_do)(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule);
@ -161,6 +162,6 @@ void indexer_undo_do_init(DB_INDEXER *indexer);
void indexer_undo_do_destroy(DB_INDEXER *indexer);
int indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, struct ule_prov_info *prov_info);
int indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, struct ule_prov_info *prov_info, DBT_ARRAY *hot_keys, DBT_ARRAY *hot_vals);
#endif

171
src/indexer-undo-do.cc
View file

@ -156,7 +156,7 @@ static int indexer_append_xid(DB_INDEXER *indexer, TXNID xid, XIDS *xids_result)
static bool indexer_find_prev_xr(DB_INDEXER *indexer, ULEHANDLE ule, uint64_t xrindex, uint64_t *prev_xrindex);
static int indexer_generate_hot_key_val(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, UXRHANDLE uxr, DBT *hotkey, DBT *hotval);
static int indexer_generate_hot_keys_vals(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, UXRHANDLE uxr, DBT_ARRAY *hotkeys, DBT_ARRAY *hotvals);
static int indexer_ft_delete_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids, TOKUTXN txn);
static int indexer_ft_delete_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
static int indexer_ft_insert_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids, TOKUTXN txn);
@ -169,20 +169,31 @@ static void indexer_lock_key(DB_INDEXER *indexer, DB *hotdb, DBT *key, TXNID out
void
indexer_undo_do_init(DB_INDEXER *indexer) {
indexer_commit_keys_init(&indexer->i->commit_keys);
toku_init_dbt_flags(&indexer->i->hotkey, DB_DBT_REALLOC);
toku_init_dbt_flags(&indexer->i->hotval, DB_DBT_REALLOC);
XMALLOC_N(indexer->i->N, indexer->i->hot_keys);
XMALLOC_N(indexer->i->N, indexer->i->hot_vals);
for (int which = 0; which < indexer->i->N; which++) {
toku_dbt_array_init(&indexer->i->hot_keys[which], 1);
toku_dbt_array_init(&indexer->i->hot_vals[which], 1);
}
}
// destroy the undo globals
void
indexer_undo_do_destroy(DB_INDEXER *indexer) {
indexer_commit_keys_destroy(&indexer->i->commit_keys);
toku_destroy_dbt(&indexer->i->hotkey);
toku_destroy_dbt(&indexer->i->hotval);
if (indexer->i->hot_keys) {
invariant(indexer->i->hot_vals);
for (int which = 0; which < indexer->i->N; which++) {
toku_dbt_array_destroy(&indexer->i->hot_keys[which]);
toku_dbt_array_destroy(&indexer->i->hot_vals[which]);
}
toku_free(indexer->i->hot_keys);
toku_free(indexer->i->hot_vals);
}
}
static int
indexer_undo_do_committed(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
indexer_undo_do_committed(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, DBT_ARRAY *hot_keys, DBT_ARRAY *hot_vals) {
int result = 0;
// init the xids to the root xid
@ -214,30 +225,47 @@ indexer_undo_do_committed(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
; // do nothing
} else if (uxr_is_insert(prevuxr)) {
// generate the hot delete key
result = indexer_generate_hot_key_val(indexer, hotdb, ule, prevuxr, &indexer->i->hotkey, NULL);
result = indexer_generate_hot_keys_vals(indexer, hotdb, ule, prevuxr, hot_keys, NULL);
if (result == 0) {
// send the delete message
result = indexer_ft_delete_committed(indexer, hotdb, &indexer->i->hotkey, xids);
if (result == 0)
indexer_commit_keys_add(&indexer->i->commit_keys, indexer->i->hotkey.size, indexer->i->hotkey.data);
paranoid_invariant(hot_keys->size <= hot_keys->capacity);
for (uint32_t i = 0; i < hot_keys->size; i++) {
DBT *hotkey = &hot_keys->dbts[i];
// send the delete message
result = indexer_ft_delete_committed(indexer, hotdb, hotkey, xids);
if (result == 0) {
indexer_commit_keys_add(&indexer->i->commit_keys, hotkey->size, hotkey->data);
}
}
}
} else
} else {
assert(0);
}
}
if (result != 0)
if (result != 0) {
break;
}
// do
if (uxr_is_delete(uxr)) {
; // do nothing
} else if (uxr_is_insert(uxr)) {
// generate the hot insert key and val
result = indexer_generate_hot_key_val(indexer, hotdb, ule, uxr, &indexer->i->hotkey, &indexer->i->hotval);
result = indexer_generate_hot_keys_vals(indexer, hotdb, ule, uxr, hot_keys, hot_vals);
if (result == 0) {
// send the insert message
result = indexer_ft_insert_committed(indexer, hotdb, &indexer->i->hotkey, &indexer->i->hotval, xids);
if (result == 0)
indexer_commit_keys_add(&indexer->i->commit_keys, indexer->i->hotkey.size, indexer->i->hotkey.data);
paranoid_invariant(hot_keys->size == hot_vals->size);
paranoid_invariant(hot_keys->size <= hot_keys->capacity);
paranoid_invariant(hot_vals->size <= hot_vals->capacity);
for (uint32_t i = 0; i < hot_keys->size; i++) {
DBT *hotkey = &hot_keys->dbts[i];
DBT *hotval = &hot_vals->dbts[i];
// send the insert message
result = indexer_ft_insert_committed(indexer, hotdb, hotkey, hotval, xids);
if (result == 0) {
indexer_commit_keys_add(&indexer->i->commit_keys, hotkey->size, hotkey->data);
}
}
}
} else
assert(0);
@ -276,7 +304,7 @@ exit:
}
static int
indexer_undo_do_provisional(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, struct ule_prov_info *prov_info) {
indexer_undo_do_provisional(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, struct ule_prov_info *prov_info, DBT_ARRAY *hot_keys, DBT_ARRAY *hot_vals) {
int result = 0;
indexer_commit_keys_set_empty(&indexer->i->commit_keys);
@ -352,27 +380,32 @@ indexer_undo_do_provisional(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, struc
; // do nothing
} else if (uxr_is_insert(prevuxr)) {
// generate the hot delete key
result = indexer_generate_hot_key_val(indexer, hotdb, ule, prevuxr, &indexer->i->hotkey, NULL);
result = indexer_generate_hot_keys_vals(indexer, hotdb, ule, prevuxr, hot_keys, NULL);
if (result == 0) {
// send the delete message
switch (outermost_xid_state) {
case TOKUTXN_LIVE:
case TOKUTXN_PREPARING:
invariant(this_xid_state != TOKUTXN_ABORTING);
invariant(!curr_txn || toku_txn_get_state(curr_txn) == TOKUTXN_LIVE || toku_txn_get_state(curr_txn) == TOKUTXN_PREPARING);
result = indexer_ft_delete_provisional(indexer, hotdb, &indexer->i->hotkey, xids, curr_txn);
if (result == 0) {
indexer_lock_key(indexer, hotdb, &indexer->i->hotkey, prov_ids[0], curr_txn);
paranoid_invariant(hot_keys->size <= hot_keys->capacity);
for (uint32_t i = 0; i < hot_keys->size; i++) {
DBT *hotkey = &hot_keys->dbts[i];
// send the delete message
switch (outermost_xid_state) {
case TOKUTXN_LIVE:
case TOKUTXN_PREPARING:
invariant(this_xid_state != TOKUTXN_ABORTING);
invariant(!curr_txn || toku_txn_get_state(curr_txn) == TOKUTXN_LIVE || toku_txn_get_state(curr_txn) == TOKUTXN_PREPARING);
result = indexer_ft_delete_provisional(indexer, hotdb, hotkey, xids, curr_txn);
if (result == 0) {
indexer_lock_key(indexer, hotdb, hotkey, prov_ids[0], curr_txn);
}
break;
case TOKUTXN_COMMITTING:
case TOKUTXN_RETIRED:
result = indexer_ft_delete_committed(indexer, hotdb, hotkey, xids);
if (result == 0)
indexer_commit_keys_add(&indexer->i->commit_keys, hotkey->size, hotkey->data);
break;
case TOKUTXN_ABORTING: // can not happen since we stop processing the leaf entry if the outer most xr is aborting
assert(0);
}
break;
case TOKUTXN_COMMITTING:
case TOKUTXN_RETIRED:
result = indexer_ft_delete_committed(indexer, hotdb, &indexer->i->hotkey, xids);
if (result == 0)
indexer_commit_keys_add(&indexer->i->commit_keys, indexer->i->hotkey.size, indexer->i->hotkey.data);
break;
case TOKUTXN_ABORTING: // can not happen since we stop processing the leaf entry if the outer most xr is aborting
assert(0);
}
}
} else
@ -386,28 +419,36 @@ indexer_undo_do_provisional(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, struc
; // do nothing
} else if (uxr_is_insert(uxr)) {
// generate the hot insert key and val
result = indexer_generate_hot_key_val(indexer, hotdb, ule, uxr, &indexer->i->hotkey, &indexer->i->hotval);
result = indexer_generate_hot_keys_vals(indexer, hotdb, ule, uxr, hot_keys, hot_vals);
if (result == 0) {
// send the insert message
switch (outermost_xid_state) {
case TOKUTXN_LIVE:
case TOKUTXN_PREPARING:
assert(this_xid_state != TOKUTXN_ABORTING);
invariant(!curr_txn || toku_txn_get_state(curr_txn) == TOKUTXN_LIVE || toku_txn_get_state(curr_txn) == TOKUTXN_PREPARING);
result = indexer_ft_insert_provisional(indexer, hotdb, &indexer->i->hotkey, &indexer->i->hotval, xids, curr_txn);
if (result == 0) {
indexer_lock_key(indexer, hotdb, &indexer->i->hotkey, prov_ids[0], prov_txns[0]);
paranoid_invariant(hot_keys->size == hot_vals->size);
paranoid_invariant(hot_keys->size <= hot_keys->capacity);
paranoid_invariant(hot_vals->size <= hot_vals->capacity);
for (uint32_t i = 0; i < hot_keys->size; i++) {
DBT *hotkey = &hot_keys->dbts[i];
DBT *hotval = &hot_vals->dbts[i];
// send the insert message
switch (outermost_xid_state) {
case TOKUTXN_LIVE:
case TOKUTXN_PREPARING:
assert(this_xid_state != TOKUTXN_ABORTING);
invariant(!curr_txn || toku_txn_get_state(curr_txn) == TOKUTXN_LIVE || toku_txn_get_state(curr_txn) == TOKUTXN_PREPARING);
result = indexer_ft_insert_provisional(indexer, hotdb, hotkey, hotval, xids, curr_txn);
if (result == 0) {
indexer_lock_key(indexer, hotdb, hotkey, prov_ids[0], prov_txns[0]);
}
break;
case TOKUTXN_COMMITTING:
case TOKUTXN_RETIRED:
result = indexer_ft_insert_committed(indexer, hotdb, hotkey, hotval, xids);
// no need to do this because we do implicit commits on inserts
if (0 && result == 0)
indexer_commit_keys_add(&indexer->i->commit_keys, hotkey->size, hotkey->data);
break;
case TOKUTXN_ABORTING: // can not happen since we stop processing the leaf entry if the outer most xr is aborting
assert(0);
}
break;
case TOKUTXN_COMMITTING:
case TOKUTXN_RETIRED:
result = indexer_ft_insert_committed(indexer, hotdb, &indexer->i->hotkey, &indexer->i->hotval, xids);
// no need to do this because we do implicit commits on inserts
if (0 && result == 0)
indexer_commit_keys_add(&indexer->i->commit_keys, indexer->i->hotkey.size, indexer->i->hotkey.data);
break;
case TOKUTXN_ABORTING: // can not happen since we stop processing the leaf entry if the outer most xr is aborting
assert(0);
}
}
} else
@ -433,10 +474,10 @@ exit:
}
int
indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, struct ule_prov_info *prov_info) {
int result = indexer_undo_do_committed(indexer, hotdb, ule);
indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, struct ule_prov_info *prov_info, DBT_ARRAY *hot_keys, DBT_ARRAY *hot_vals) {
int result = indexer_undo_do_committed(indexer, hotdb, ule, hot_keys, hot_vals);
if (result == 0) {
result = indexer_undo_do_provisional(indexer, hotdb, ule, prov_info);
result = indexer_undo_do_provisional(indexer, hotdb, ule, prov_info, hot_keys, hot_vals);
}
if (indexer->i->test_only_flags == INDEXER_TEST_ONLY_ERROR_CALLBACK) {
result = EINVAL;
@ -482,7 +523,7 @@ indexer_append_xid(DB_INDEXER *UU(indexer), TXNID xid, XIDS *xids_result) {
}
static int
indexer_generate_hot_key_val(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, UXRHANDLE uxr, DBT *hotkey, DBT *hotval) {
indexer_generate_hot_keys_vals(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, UXRHANDLE uxr, DBT_ARRAY *hotkeys, DBT_ARRAY *hotvals) {
int result = 0;
// setup the source key
@ -495,11 +536,11 @@ indexer_generate_hot_key_val(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, UXRH
// generate the secondary row
DB_ENV *env = indexer->i->env;
if (hotval) {
result = env->i->generate_row_for_put(hotdb, indexer->i->src_db, hotkey, hotval, &srckey, &srcval);
if (hotvals) {
result = env->i->generate_row_for_put(hotdb, indexer->i->src_db, hotkeys, hotvals, &srckey, &srcval);
}
else {
result = env->i->generate_row_for_del(hotdb, indexer->i->src_db, hotkey, &srckey, &srcval);
result = env->i->generate_row_for_del(hotdb, indexer->i->src_db, hotkeys, &srckey, &srcval);
}
toku_destroy_dbt(&srckey);
toku_destroy_dbt(&srcval);

17
src/indexer.cc
View file

@ -561,7 +561,9 @@ build_index(DB_INDEXER *indexer) {
ULEHANDLE ule = prov_info.ule;
for (int which_db = 0; (which_db < indexer->i->N) && (result == 0); which_db++) {
DB *db = indexer->i->dest_dbs[which_db];
result = indexer_undo_do(indexer, db, ule, &prov_info);
DBT_ARRAY *hot_keys = &indexer->i->hot_keys[which_db];
DBT_ARRAY *hot_vals = &indexer->i->hot_vals[which_db];
result = indexer_undo_do(indexer, db, ule, &prov_info, hot_keys, hot_vals);
if ((result != 0) && (indexer->i->error_callback != NULL)) {
// grab the key and call the error callback
DBT key; toku_init_dbt_flags(&key, DB_DBT_REALLOC);
@ -710,12 +712,23 @@ toku_indexer_set_test_only_flags(DB_INDEXER *indexer, int flags) {
// a convenience wrapper that gets and destroys the ule's prov info
static int
test_indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
int which_db;
for (which_db = 0; which_db < indexer->i->N; which_db++) {
if (indexer->i->dest_dbs[which_db] == hotdb) {
break;
}
}
if (which_db == indexer->i->N) {
return EINVAL;
}
struct ule_prov_info prov_info;
memset(&prov_info, 0, sizeof(prov_info));
// pass null for the leafentry - we don't need it, neither does the info
ule_prov_info_init(&prov_info, NULL, ule);
indexer_fill_prov_info(indexer, &prov_info);
int r = indexer_undo_do(indexer, hotdb, ule, &prov_info);
DBT_ARRAY *hot_keys = &indexer->i->hot_keys[which_db];
DBT_ARRAY *hot_vals = &indexer->i->hot_vals[which_db];
int r = indexer_undo_do(indexer, hotdb, ule, &prov_info, hot_keys, hot_vals);
ule_prov_info_destroy(&prov_info);
return r;
}

7
src/tests/blocking-first-empty.cc
View file

@ -98,13 +98,6 @@ struct my_callback_context {
};
#if TOKUDB
static void copy_dbt(DBT *dest, DBT const *src) {
assert(dest->flags == DB_DBT_REALLOC);
dest->size = src->size;
dest->data = toku_xrealloc(dest->data, dest->size);
memcpy(dest->data, src->data, dest->size);
}
static int blocking_first_callback(DBT const *a UU(), DBT const *b UU(), void *e UU()) {
DBT const *found_key = a;
DBT const *found_val = b;

7
src/tests/blocking-first.cc
View file

@ -116,13 +116,6 @@ struct my_callback_context {
};
#if TOKUDB
static void copy_dbt(DBT *dest, DBT const *src) {
assert(dest->flags == DB_DBT_REALLOC);
dest->size = src->size;
dest->data = toku_xrealloc(dest->data, dest->size);
memcpy(dest->data, src->data, dest->size);
}
static int blocking_first_callback(DBT const *a UU(), DBT const *b UU(), void *e UU()) {
DBT const *found_key = a;
DBT const *found_val = b;

7
src/tests/blocking-last.cc
View file

@ -116,13 +116,6 @@ struct my_callback_context {
};
#if TOKUDB
static void copy_dbt(DBT *dest, DBT const *src) {
assert(dest->flags == DB_DBT_REALLOC);
dest->size = src->size;
dest->data = toku_xrealloc(dest->data, dest->size);
memcpy(dest->data, src->data, dest->size);
}
static int blocking_last_callback(DBT const *a UU(), DBT const *b UU(), void *e UU()) {
DBT const *found_key = a;
DBT const *found_val = b;

7
src/tests/blocking-next-prev-deadlock.cc
View file

@ -124,13 +124,6 @@ struct my_callback_context {
};
#if TOKUDB
static void copy_dbt(DBT *dest, DBT const *src) {
assert(dest->flags == DB_DBT_REALLOC);
dest->size = src->size;
dest->data = toku_xrealloc(dest->data, dest->size);
memcpy(dest->data, src->data, dest->size);
}
static int blocking_next_callback(DBT const *a UU(), DBT const *b UU(), void *e UU()) {
DBT const *found_key = a;
DBT const *found_val = b;

7
src/tests/blocking-next-prev.cc
View file

@ -123,13 +123,6 @@ struct my_callback_context {
};
#if TOKUDB
static void copy_dbt(DBT *dest, DBT const *src) {
assert(dest->flags == DB_DBT_REALLOC);
dest->size = src->size;
dest->data = toku_xrealloc(dest->data, dest->size);
memcpy(dest->data, src->data, dest->size);
}
static int blocking_next_callback(DBT const *a UU(), DBT const *b UU(), void *e UU()) {
DBT const *found_key = a;
DBT const *found_val = b;

7
src/tests/blocking-set-range-0.cc
View file

@ -117,13 +117,6 @@ struct my_callback_context {
};
#if TOKUDB
static void copy_dbt(DBT *dest, DBT const *src) {
assert(dest->flags == DB_DBT_REALLOC);
dest->size = src->size;
dest->data = toku_xrealloc(dest->data, dest->size);
memcpy(dest->data, src->data, dest->size);
}
static int blocking_set_range_callback(DBT const *a UU(), DBT const *b UU(), void *e UU()) {
DBT const *found_key = a;
DBT const *found_val = b;

7
src/tests/blocking-set-range-n.cc
View file

@ -117,13 +117,6 @@ struct my_callback_context {
};
#if TOKUDB
static void copy_dbt(DBT *dest, DBT const *src) {
assert(dest->flags == DB_DBT_REALLOC);
dest->size = src->size;
dest->data = toku_xrealloc(dest->data, dest->size);
memcpy(dest->data, src->data, dest->size);
}
static int blocking_set_range_callback(DBT const *a UU(), DBT const *b UU(), void *e UU()) {
DBT const *found_key = a;
DBT const *found_val = b;

7
src/tests/blocking-set-range-reverse-0.cc
View file

@ -111,13 +111,6 @@ static void populate(DB_ENV *db_env, DB *db, uint64_t nrows) {
r = txn->commit(txn, 0); assert(r == 0);
}
static void copy_dbt(DBT *dest, DBT const *src) {
assert(dest->flags == DB_DBT_REALLOC);
dest->size = src->size;
dest->data = toku_xrealloc(dest->data, dest->size);
memcpy(dest->data, src->data, dest->size);
}
struct my_callback_context {
DBT key;
DBT val;

23
src/tests/bug1381.cc
View file

@ -94,20 +94,25 @@ PATENT RIGHTS GRANT:
#include <memory.h>
static int generate_row_for_put(
DB *UU(dest_db),
DB *UU(src_db),
DBT *dest_key,
DBT *dest_val,
const DBT *src_key,
DB *UU(dest_db),
DB *UU(src_db),
DBT_ARRAY *dest_key_arrays,
DBT_ARRAY *dest_val_arrays,
const DBT *src_key,
const DBT *src_val
)
{
)
{
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
dest_key->flags = 0;
dest_val->flags = 0;
dest_key->data = src_key->data;
dest_key->size = src_key->size;
dest_key->flags = 0;
dest_val->data = src_val->data;
dest_val->size = src_val->size;
dest_val->flags = 0;
return 0;
}

8
src/tests/del-multiple-huge-primary-row.cc
View file

@ -105,8 +105,10 @@ get_data(int *v, int i, int ndbs) {
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) src_key; (void) src_data;
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
toku_dbt_array_resize(dest_keys, 1);
DBT *dest_key = &dest_keys->dbts[0];
(void) dest_db; (void) src_db; (void) dest_keys; (void) src_key; (void) src_data;
assert(src_db == NULL);
unsigned int dbnum;
@ -170,7 +172,7 @@ verify_del_multiple(DB_ENV *env, DB *db[], int ndbs, int nrows) {
DBT pri_data; dbt_init(&pri_data, &v[0], sizeof v);
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->del_multiple(env, NULL, deltxn, &pri_key, &pri_data, ndbs, db, keys, flags); assert_zero(r);
r = env_del_multiple_test_no_array(env, NULL, deltxn, &pri_key, &pri_data, ndbs, db, keys, flags); assert_zero(r);
for (int dbnum = 0; dbnum < ndbs; dbnum++)
verify_locked(env, db[dbnum], get_key(i, dbnum));
}

8
src/tests/del-multiple-srcdb.cc
View file

@ -105,8 +105,10 @@ get_data(int *v, int i, int ndbs) {
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) src_key; (void) src_data;
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
toku_dbt_array_resize(dest_keys, 1);
DBT *dest_key = &dest_keys->dbts[0];
(void) dest_db; (void) src_db; (void) dest_keys; (void) src_key; (void) src_data;
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
@ -165,7 +167,7 @@ verify_del_multiple(DB_ENV *env, DB *db[], int ndbs, int nrows) {
DBT pri_data; dbt_init(&pri_data, &v[0], sizeof v);
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->del_multiple(env, ndbs > 0 ? db[0] : NULL, deltxn, &pri_key, &pri_data, ndbs, db, keys, flags); assert_zero(r);
r = env_del_multiple_test_no_array(env, ndbs > 0 ? db[0] : NULL, deltxn, &pri_key, &pri_data, ndbs, db, keys, flags); assert_zero(r);
for (int dbnum = 0; dbnum < ndbs; dbnum++)
verify_locked(env, db[dbnum], get_key(i, dbnum));
}

8
src/tests/del-multiple.cc
View file

@ -105,8 +105,10 @@ get_data(int *v, int i, int ndbs) {
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) src_key; (void) src_data;
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
toku_dbt_array_resize(dest_keys, 1);
DBT *dest_key = &dest_keys->dbts[0];
(void) dest_db; (void) src_db; (void) dest_keys; (void) src_key; (void) src_data;
assert(src_db == NULL);
unsigned int dbnum;
@ -166,7 +168,7 @@ verify_del_multiple(DB_ENV *env, DB *db[], int ndbs, int nrows) {
DBT pri_data; dbt_init(&pri_data, &v[0], sizeof v);
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->del_multiple(env, NULL, deltxn, &pri_key, &pri_data, ndbs, db, keys, flags); assert_zero(r);
r = env_del_multiple_test_no_array(env, NULL, deltxn, &pri_key, &pri_data, ndbs, db, keys, flags); assert_zero(r);
for (int dbnum = 0; dbnum < ndbs; dbnum++)
verify_locked(env, db[dbnum], get_key(i, dbnum));
}

28
src/tests/directory_lock.cc
View file

@ -96,12 +96,12 @@ const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG
static int
put_multiple_callback(DB *dest_db __attribute__((unused)), DB *src_db __attribute__((unused)), DBT *dest_key __attribute__((unused)), DBT *dest_val __attribute__((unused)), const DBT *src_key __attribute__((unused)), const DBT *src_val __attribute__((unused))) {
put_multiple_callback(DB *dest_db UU(), DB *src_db UU(), DBT_ARRAY *dest_keys UU(), DBT_ARRAY *dest_vals UU(), const DBT *src_key UU(), const DBT *src_val UU()) {
return 0;
}
static int
del_multiple_callback(DB *dest_db __attribute__((unused)), DB *src_db __attribute__((unused)), DBT *dest_key __attribute__((unused)), const DBT *src_key __attribute__((unused)), const DBT *src_val __attribute__((unused))) {
del_multiple_callback(DB *dest_db UU(), DB *src_db UU(), DBT_ARRAY *dest_keys UU(), const DBT *src_key UU(), const DBT *src_val UU()) {
return 0;
}
@ -158,7 +158,7 @@ static void verify_shared_ops_fail(DB_ENV* env, DB* db) {
r = txn->commit(txn,0); CKERR(r);
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = env->put_multiple(
r = env_put_multiple_test_no_array(
env, db, txn,
&key, &val,
1, &db, &in_key, &in_val, &flags);
@ -166,7 +166,7 @@ static void verify_shared_ops_fail(DB_ENV* env, DB* db) {
r = txn->commit(txn,0); CKERR(r);
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = env->put_multiple(
r = env_put_multiple_test_no_array(
env, NULL, txn,
&key, &val,
1, &db, &in_key, &in_val, &flags);
@ -176,7 +176,7 @@ static void verify_shared_ops_fail(DB_ENV* env, DB* db) {
flags = DB_DELETE_ANY;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = env->del_multiple(
r = env_del_multiple_test_no_array(
env, db, txn,
&key, &val,
1, &db, &in_key, &flags);
@ -184,7 +184,7 @@ static void verify_shared_ops_fail(DB_ENV* env, DB* db) {
r = txn->commit(txn,0); CKERR(r);
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = env->del_multiple(
r = env_del_multiple_test_no_array(
env, NULL, txn,
&key, &val,
1, &db, &in_key, &flags);
@ -194,7 +194,7 @@ static void verify_shared_ops_fail(DB_ENV* env, DB* db) {
flags = 0;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = env->update_multiple(
r = env_update_multiple_test_no_array(
env, NULL, txn,
&key, &val,
&key, &val,
@ -205,7 +205,7 @@ static void verify_shared_ops_fail(DB_ENV* env, DB* db) {
r = txn->commit(txn,0); CKERR(r);
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = env->update_multiple(
r = env_update_multiple_test_no_array(
env, db, txn,
&key, &val,
&key, &val,
@ -367,14 +367,14 @@ int test_main (int argc, char * const argv[]) {
r = env->txn_begin(env, NULL, &txnb, 0); CKERR(r);
dbt_init(&key, "a", 2);
dbt_init(&val, "a", 2);
env->put_multiple(
env_put_multiple_test_no_array(
env, NULL, txna,
&key, &val,
1, &db, &in_key, &in_val, &flags);
CKERR(r);
dbt_init(&key, "b", 2);
dbt_init(&val, "b", 2);
env->put_multiple(
env_put_multiple_test_no_array(
env, NULL, txnb,
&key, &val,
1, &db, &in_key, &in_val, &flags);
@ -388,14 +388,14 @@ int test_main (int argc, char * const argv[]) {
r = env->txn_begin(env, NULL, &txnb, 0); CKERR(r);
dbt_init(&key, "a", 2);
dbt_init(&val, "a", 2);
env->del_multiple(
env_del_multiple_test_no_array(
env, NULL, txna,
&key, &val,
1, &db, &in_key, &flags);
CKERR(r);
dbt_init(&key, "b", 2);
dbt_init(&val, "b", 2);
env->del_multiple(
env_del_multiple_test_no_array(
env, db, txnb,
&key, &val,
1, &db, &in_key, &flags);
@ -411,7 +411,7 @@ int test_main (int argc, char * const argv[]) {
r = env->txn_begin(env, NULL, &txnb, 0); CKERR(r);
dbt_init(&key, "a", 2);
dbt_init(&val, "a", 2);
env->update_multiple(
env_update_multiple_test_no_array(
env, NULL, txna,
&key, &val,
&key, &val,
@ -421,7 +421,7 @@ int test_main (int argc, char * const argv[]) {
CKERR(r);
dbt_init(&key, "b", 2);
dbt_init(&val, "b", 2);
env->update_multiple(
env_update_multiple_test_no_array(
env, db, txnb,
&key, &val,
&key, &val,

28
src/tests/env-put-multiple.cc
View file

@ -112,7 +112,13 @@ DBT dest_vals[MAX_DBS];
#define CKERR2IFNOT0(r, rexpect) do { if (num_dbs>0) { CKERR2(r, rexpect); } else { CKERR2(r, EINVAL); } } while (0)
static int
put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys_arrays, DBT_ARRAY *dest_datas, const DBT *src_key, const DBT *src_data) {
toku_dbt_array_resize(dest_keys_arrays, 1);
toku_dbt_array_resize(dest_datas, 1);
DBT *dest_key = &dest_keys_arrays->dbts[0];
DBT *dest_data = &dest_datas->dbts[0];
dest_key->flags = 0;
dest_data->flags = 0;
(void) src_db;
@ -120,15 +126,15 @@ put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, con
assert(which < MAX_DBS);
assert(src_key->size == 4);
assert(src_val->size == 4);
assert(src_data->size == 4);
kbuf[which][0] = *(uint32_t*)src_key->data;
kbuf[which][1] = which;
vbuf[which][0] = which;
vbuf[which][1] = *(uint32_t*)src_val->data;
vbuf[which][1] = *(uint32_t*)src_data->data;
dest_key->data = kbuf[which];
dest_key->size = sizeof(kbuf[which]);
dest_val->data = vbuf[which];
dest_val->size = sizeof(vbuf[which]);
dest_data->data = vbuf[which];
dest_data->size = sizeof(vbuf[which]);
return 0;
}
@ -185,7 +191,7 @@ static void run_test (void) {
uint32_t magic2 = ~magic;
DBT keydbt = {.data=&magic, .size=sizeof(magic)};
DBT valdbt = {.data=&magic2, .size=sizeof(magic2)};
r = env->put_multiple(env, NULL, txn, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
CKERRIFNOT0(r);
for (which = 0; which < num_dbs; which++) {
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])};
@ -208,7 +214,7 @@ static void run_test (void) {
uint32_t magic2 = ~magic;
DBT keydbt = {.data=&magic, .size=sizeof(magic)};
DBT valdbt = {.data=&magic2, .size=sizeof(magic2)};
r = env->put_multiple(env, NULL, txn, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
CKERRIFNOT0(r);
for (which = 0; which < num_dbs; which++) {
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])};
@ -231,7 +237,7 @@ static void run_test (void) {
uint32_t magic2 = ~magic;
DBT keydbt = {.data=&magic, .size=sizeof(magic)};
DBT valdbt = {.data=&magic2, .size=sizeof(magic2)};
r = env->put_multiple(env, NULL, txn, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
CKERR2IFNOT0(r, DB_KEYEXIST);
for (which = 0; which < num_dbs; which++) {
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])};
@ -257,7 +263,7 @@ static void run_test (void) {
uint32_t magic2 = ~magic;
DBT keydbt = {.data=&magic, .size=sizeof(magic)};
DBT valdbt = {.data=&magic2, .size=sizeof(magic2)};
r = env->put_multiple(env, NULL, txna, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txna, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
CKERRIFNOT0(r);
for (which = 0; which < num_dbs; which++) {
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])};
@ -272,7 +278,7 @@ static void run_test (void) {
CKERR(r);
//Lock should fail
r = env->put_multiple(env, NULL, txnb, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txnb, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
CKERR2IFNOT0(r, DB_LOCK_NOTGRANTED);
for (which = 0; which < num_dbs; which++) {
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])};
@ -284,7 +290,7 @@ static void run_test (void) {
r = txna->commit(txna, 0);
//Should succeed this time.
r = env->put_multiple(env, NULL, txnb, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txnb, &keydbt, &valdbt, num_dbs, dbs_multiple, dest_keys, dest_vals, flags);
CKERRIFNOT0(r);
for (which = 0; which < num_dbs; which++) {
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])};

2
src/tests/hotindexer-bw.cc
View file

@ -170,7 +170,7 @@ static void * client(void *arg)
while ( retry++ < 10 ) {
toku_mutex_lock(&put_lock);
rr = env->put_multiple(env,
rr = env_put_multiple_test_no_array(env,
cs->dbs[0],
txn,
&key,

27
src/tests/hotindexer-insert-committed-optimized.cc
View file

@ -89,6 +89,7 @@ PATENT RIGHTS GRANT:
#include "test.h"
#include "toku_pthread.h"
#include "key-val.h"
#include <db.h>
#include <sys/stat.h>
@ -103,30 +104,6 @@ struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
(void) src_db;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
(void) progress;
(void) extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
@ -178,7 +155,7 @@ static void run_test(void)
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate_switch); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, TOKU_TEST_FILENAME, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);

27
src/tests/hotindexer-insert-committed.cc
View file

@ -89,6 +89,7 @@ PATENT RIGHTS GRANT:
#include "test.h"
#include "toku_pthread.h"
#include "key-val.h"
#include <db.h>
#include <sys/stat.h>
@ -103,30 +104,6 @@ struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
(void) src_db;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
(void) progress;
(void) extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
@ -178,7 +155,7 @@ static void run_test(void)
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate_switch); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, TOKU_TEST_FILENAME, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);

27
src/tests/hotindexer-insert-provisional.cc
View file

@ -89,6 +89,7 @@ PATENT RIGHTS GRANT:
#include "test.h"
#include "toku_pthread.h"
#include "key-val.h"
#include <db.h>
#include <sys/stat.h>
@ -103,30 +104,6 @@ struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
(void) src_db;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
(void) progress;
(void) extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
@ -178,7 +155,7 @@ static void run_test(void)
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate_switch); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, TOKU_TEST_FILENAME, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);

27
src/tests/hotindexer-lock-test.cc
View file

@ -89,6 +89,7 @@ PATENT RIGHTS GRANT:
#include "test.h"
#include "toku_pthread.h"
#include "key-val.h"
#include <db.h>
#include <sys/stat.h>
@ -103,30 +104,6 @@ struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
(void) src_db;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
(void) progress;
(void) extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void run_indexer(DB *src, DB **dbs)
{
int r;
@ -177,7 +154,7 @@ static void run_test(void)
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate_switch); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, TOKU_TEST_FILENAME, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
db_env_enable_engine_status(0); // disable engine status on crash

2
src/tests/hotindexer-multiclient.cc
View file

@ -161,7 +161,7 @@ static void * client(void *arg)
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
rr = env->put_multiple(env,
rr = env_put_multiple_test_no_array(env,
cs->dbs[0],
txn,
&key,

27
src/tests/hotindexer-nested-insert-committed.cc
View file

@ -89,6 +89,7 @@ PATENT RIGHTS GRANT:
#include "test.h"
#include "toku_pthread.h"
#include "key-val.h"
#include <db.h>
#include <sys/stat.h>
@ -103,30 +104,6 @@ struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
(void) src_db;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
(void) progress;
(void) extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
@ -178,7 +155,7 @@ static void run_test(void)
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate_switch); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, TOKU_TEST_FILENAME, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);

17
src/tests/hotindexer-put-abort.cc
View file

@ -96,16 +96,21 @@ PATENT RIGHTS GRANT:
// to run. the hot indexer erroneously append to the rollback log that is in the process of being aborted.
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) dest_db; (void) src_db; (void) dest_keys; (void) dest_vals; (void) src_key; (void) src_val;
lazy_assert(src_db != NULL && dest_db != NULL);
if (dest_key->flags == DB_DBT_REALLOC) {
dest_key->data = toku_realloc(dest_key->data, src_data->size);
memcpy(dest_key->data, src_data->data, src_data->size);
dest_key->size = src_data->size;
dest_key->data = toku_realloc(dest_key->data, src_val->size);
memcpy(dest_key->data, src_val->data, src_val->size);
dest_key->size = src_val->size;
}
dest_data->size = 0;
dest_val->size = 0;
return 0;
}

16
src/tests/hotindexer-put-commit.cc
View file

@ -96,8 +96,12 @@ PATENT RIGHTS GRANT:
// to run. the hot indexer erroneously append to the rollback log that is in the process of being committed.
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
lazy_assert(src_db != NULL && dest_db != NULL);
@ -106,10 +110,10 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
memcpy(dest_key->data, src_key->data, src_key->size);
dest_key->size = src_key->size;
}
if (dest_data->flags == DB_DBT_REALLOC) {
dest_data->data = toku_realloc(dest_data->data, src_data->size);
memcpy(dest_data->data, src_data->data, src_data->size);
dest_data->size = src_data->size;
if (dest_val->flags == DB_DBT_REALLOC) {
dest_val->data = toku_realloc(dest_val->data, src_val->size);
memcpy(dest_val->data, src_val->data, src_val->size);
dest_val->size = src_val->size;
}
return 0;

12
src/tests/hotindexer-put-multiple.cc
View file

@ -103,7 +103,13 @@ struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
dest_key->flags = 0;
dest_val->flags = 0;
(void) src_db;
@ -208,7 +214,7 @@ static void run_test(void)
// putm (8,9)
pk = 8; pv = 9;
r = env->put_multiple(env, src_db, txn, &prikey, &prival, NUM_DBS+1, putm_dbs, putm_keys, putm_vals, putm_flags);
r = env_put_multiple_test_no_array(env, src_db, txn, &prikey, &prival, NUM_DBS+1, putm_dbs, putm_keys, putm_vals, putm_flags);
CKERR(r);
r = indexer->build(indexer);
@ -216,7 +222,7 @@ static void run_test(void)
// putm (9, 10)
pk = 9; pv = 10;
r = env->put_multiple(env, src_db, txn, &prikey, &prival, NUM_DBS+1, putm_dbs, putm_keys, putm_vals, putm_flags);
r = env_put_multiple_test_no_array(env, src_db, txn, &prikey, &prival, NUM_DBS+1, putm_dbs, putm_keys, putm_vals, putm_flags);
CKERR(r);
r = indexer->close(indexer);

16
src/tests/hotindexer-simple-abort-put.cc
View file

@ -90,13 +90,19 @@ PATENT RIGHTS GRANT:
#include "test.h"
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
lazy_assert(src_db != NULL && dest_db != NULL);
dest_key->data = toku_xmemdup(src_data->data, src_data->size);
dest_key->size = src_data->size;
dest_data->size = 0;
toku_free(dest_key->data);
dest_key->data = toku_xmemdup(src_val->data, src_val->size);
dest_key->ulen = dest_key->size = src_val->size;
dest_val->size = 0;
return 0;
}

16
src/tests/hotindexer-simple-abort.cc
View file

@ -90,14 +90,20 @@ PATENT RIGHTS GRANT:
#include "test.h"
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
dest_key->flags = 0;
dest_val->flags = 0;
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
lazy_assert(src_db != NULL && dest_db != NULL);
dest_key->data = src_data->data;
dest_key->size = src_data->size;
dest_data->size = 0;
dest_key->data = src_val->data;
dest_key->size = src_val->size;
dest_val->size = 0;
return 0;
}

32
src/tests/hotindexer-undo-do-test.cc
View file

@ -242,34 +242,38 @@ print_dbt(DBT *dbt) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
lazy_assert(src_db != NULL && dest_db != NULL);
switch (dest_key->flags) {
case 0:
dest_key->data = src_data->data;
dest_key->size = src_data->size;
dest_key->data = src_val->data;
dest_key->size = src_val->size;
break;
case DB_DBT_REALLOC:
dest_key->data = toku_realloc(dest_key->data, src_data->size);
memcpy(dest_key->data, src_data->data, src_data->size);
dest_key->size = src_data->size;
dest_key->data = toku_realloc(dest_key->data, src_val->size);
memcpy(dest_key->data, src_val->data, src_val->size);
dest_key->size = src_val->size;
break;
default:
lazy_assert(0);
}
if (dest_data)
switch (dest_data->flags) {
if (dest_val)
switch (dest_val->flags) {
case 0:
lazy_assert(0);
break;
case DB_DBT_REALLOC:
dest_data->data = toku_realloc(dest_data->data, src_key->size);
memcpy(dest_data->data, src_key->data, src_key->size);
dest_data->size = src_key->size;
dest_val->data = toku_realloc(dest_val->data, src_key->size);
memcpy(dest_val->data, src_key->data, src_key->size);
dest_val->size = src_key->size;
break;
default:
lazy_assert(0);
@ -279,7 +283,9 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
toku_dbt_array_resize(dest_keys, 1);
DBT *dest_key = &dest_keys->dbts[0];
(void) dest_db; (void) src_db; (void) dest_key; (void) src_key; (void) src_data;
lazy_assert(src_db != NULL && dest_db != NULL);

50
src/tests/key-val.h
View file

@ -174,27 +174,19 @@ static uint32_t UU() pkey_for_val(int key, int i) {
// There is no handlerton in this test, so this function is a local replacement
// for the handlerton's generate_row_for_put().
static int UU() put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int UU() put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) src_db;
(void) src_val;
uint32_t which = *(uint32_t*)dest_db->app_private;
if ( which == 0 ) {
if (dest_key->flags==DB_DBT_REALLOC) {
if (dest_key->data) toku_free(dest_key->data);
dest_key->flags = 0;
dest_key->ulen = 0;
}
if (dest_val->flags==DB_DBT_REALLOC) {
if (dest_val->data) toku_free(dest_val->data);
dest_val->flags = 0;
dest_val->ulen = 0;
}
dbt_init(dest_key, src_key->data, src_key->size);
dbt_init(dest_val, src_val->data, src_val->size);
}
else {
assert(which != 0);
assert(dest_db != src_db);
{
assert(dest_key->flags==DB_DBT_REALLOC);
if (dest_key->ulen < sizeof(uint32_t)) {
dest_key->data = toku_xrealloc(dest_key->data, sizeof(uint32_t));
@ -221,6 +213,30 @@ static int UU() put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DB
return 0;
}
UU()
static int put_multiple_generate_switch(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
dest_key->flags = 0;
dest_val->flags = 0;
(void) src_db;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int UU() uint_cmp(const void *ap, const void *bp) {
unsigned int an = *(unsigned int *)ap;
unsigned int bn = *(unsigned int *)bp;

7
src/tests/keyrange.cc
View file

@ -116,7 +116,12 @@ my_compare(DB *this_db UU(), const DBT *a UU(), const DBT *b UU()) {
}
static int
my_generate_row(DB *dest_db UU(), DB *src_db UU(), DBT *dest_key UU(), DBT *dest_val UU(), const DBT *src_key UU(), const DBT *src_val UU()) {
my_generate_row(DB *dest_db UU(), DB *src_db UU(), DBT_ARRAY *dest_keys UU(), DBT_ARRAY *dest_vals UU(), const DBT *src_key UU(), const DBT *src_val UU()) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
assert(dest_key->flags == DB_DBT_REALLOC);
dest_key->data = toku_realloc(dest_key->data, src_key->size);
memcpy(dest_key->data, src_key->data, src_key->size);

6
src/tests/loader-cleanup-test.cc
View file

@ -545,7 +545,11 @@ static unsigned int pkey_for_val(int key, int i) {
// There is no handlerton in this test, so this function is a local replacement
// for the handlerton's generate_row_for_put().
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) src_db;

2
src/tests/loader-create-abort.cc
View file

@ -96,7 +96,7 @@ PATENT RIGHTS GRANT:
static int loader_flags = 0;
static const char *envdir = TOKU_TEST_FILENAME;
static int put_multiple_generate(DB *UU(dest_db), DB *UU(src_db), DBT *UU(dest_key), DBT *UU(dest_val), const DBT *UU(src_key), const DBT *UU(src_val)) {
static int put_multiple_generate(DB *UU(dest_db), DB *UU(src_db), DBT_ARRAY *UU(dest_keys), DBT_ARRAY *UU(dest_vals), const DBT *UU(src_key), const DBT *UU(src_val)) {
return ENOMEM;
}

2
src/tests/loader-create-close.cc
View file

@ -96,7 +96,7 @@ PATENT RIGHTS GRANT:
static int loader_flags = 0;
static const char *envdir = TOKU_TEST_FILENAME;
static int put_multiple_generate(DB *UU(dest_db), DB *UU(src_db), DBT *UU(dest_key), DBT *UU(dest_val), const DBT *UU(src_key), const DBT *UU(src_val)) {
static int put_multiple_generate(DB *UU(dest_db), DB *UU(src_db), DBT_ARRAY *UU(dest_keys), DBT_ARRAY *UU(dest_vals), const DBT *UU(src_key), const DBT *UU(src_val)) {
return ENOMEM;
}

6
src/tests/loader-dup-test.cc
View file

@ -174,7 +174,11 @@ static unsigned int pkey_for_val(int key, int i) {
// There is no handlerton in this test, so this function is a local replacement
// for the handlerton's generate_row_for_put().
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) src_db;

8
src/tests/loader-no-puts.cc
View file

@ -108,7 +108,13 @@ struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
dest_key->flags = 0;
dest_val->flags = 0;
(void) src_db;

8
src/tests/loader-reference-test.cc
View file

@ -109,7 +109,13 @@ struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{3,6}};
static uint32_t block_size = 0;
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
dest_key->flags = 0;
dest_val->flags = 0;
(void) src_db;

6
src/tests/loader-stress-del.cc
View file

@ -252,7 +252,11 @@ static unsigned int pkey_for_val(int key, int i) {
// There is no handlerton in this test, so this function is a local replacement
// for the handlerton's generate_row_for_put().
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) src_db;

6
src/tests/loader-stress-test.cc
View file

@ -264,7 +264,11 @@ static unsigned int pkey_for_val(int key, int i) {
// There is no handlerton in this test, so this function is a local replacement
// for the handlerton's generate_row_for_put().
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *UU(src_val)) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *UU(src_val)) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
assert(src_db);
assert(dest_db != src_db);

16
src/tests/loader-tpch-load.cc
View file

@ -102,8 +102,8 @@ static int COMPRESS=0;
static int USE_REGION=0;
static const char *envdir = TOKU_TEST_FILENAME;
static int generate_rows_for_region(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) __attribute__((unused));
static int generate_rows_for_lineitem(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) __attribute__((unused));
static int generate_rows_for_region(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) __attribute__((unused));
static int generate_rows_for_lineitem(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) __attribute__((unused));
// linenumber,orderkey form a unique, primary key
// key is a potentially duplicate secondary key
@ -178,8 +178,12 @@ static void tpch_parse_row(char *row, char *fields[], int fields_N)
* region table
*/
static int generate_rows_for_region(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val)
static int generate_rows_for_region(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val)
{
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
// not used
(void) src_db;
@ -230,8 +234,12 @@ static int generate_rows_for_region(DB *dest_db, DB *src_db, DBT *dest_key, DBT
*/
static int generate_rows_for_lineitem(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val)
static int generate_rows_for_lineitem(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val)
{
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
// not used
(void) src_db;
(void) src_key;

8
src/tests/maxsize-for-loader.cc
View file

@ -110,8 +110,12 @@ char random_buf[NUM_DBS][8];
static int put_multiple_generate(DB *dest_db,
DB *src_db __attribute__((__unused__)),
DBT *dest_key, DBT *dest_val,
DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals,
const DBT *src_key, const DBT *src_val __attribute__((__unused__))) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
uint32_t which = *(uint32_t*)dest_db->app_private;
assert(src_key->size==4);
@ -244,7 +248,7 @@ static void test_loader_maxsize(DB **dbs, DB **check_dbs)
v = i;
dbt_init(&key, &k, sizeof(unsigned int));
dbt_init(&val, &v, sizeof(unsigned int));
r = env->put_multiple(env, nullptr, txn, &key, &val, NUM_DBS, check_dbs, keys, vals, flags);
r = env_put_multiple_test_no_array(env, nullptr, txn, &key, &val, NUM_DBS, check_dbs, keys, vals, flags);
CKERR(r);
}
r = txn->commit(txn, 0);

33
src/tests/perf_iibench.cc
View file

@ -216,21 +216,22 @@ static int UU() iibench_put_op(DB_TXN *txn, ARG arg, void *operation_extra, void
const int num_dbs = arg->cli->num_DBs;
DB **dbs = arg->dbp;
DB_ENV *env = arg->env;
DBT mult_key_dbt[num_dbs];
DBT mult_val_dbt[num_dbs];
DBT_ARRAY mult_key_dbt[num_dbs];
DBT_ARRAY mult_val_dbt[num_dbs];
uint32_t mult_put_flags[num_dbs];
memset(mult_key_dbt, 0, sizeof(mult_key_dbt));
memset(mult_val_dbt, 0, sizeof(mult_val_dbt));
// The first index is unique with serial autoincrement keys.
// The rest are have keys generated with this thread's random data.
mult_put_flags[0] = get_put_flags(arg->cli) |
// If the table was already created, don't check for uniqueness.
(arg->cli->num_elements > 0 ? 0 : DB_NOOVERWRITE);
for (int i = 1; i < num_dbs; i++) {
mult_key_dbt[i].flags = DB_DBT_REALLOC;
for (int i = 0; i < num_dbs; i++) {
toku_dbt_array_init(&mult_key_dbt[i], 1);
toku_dbt_array_init(&mult_val_dbt[i], 1);
mult_put_flags[i] = get_put_flags(arg->cli);
}
mult_key_dbt[0].dbts[0].flags = 0;
mult_val_dbt[0].dbts[0].flags = 0;
int r = 0;
@ -247,15 +248,15 @@ static int UU() iibench_put_op(DB_TXN *txn, ARG arg, void *operation_extra, void
int64_t valbuf[3];
iibench_fill_key_buf(pk, keybuf);
iibench_fill_val_buf(pk, valbuf);
dbt_init(&mult_key_dbt[0], keybuf, sizeof keybuf);
dbt_init(&mult_val_dbt[0], valbuf, sizeof valbuf);
dbt_init(&mult_key_dbt[0].dbts[0], keybuf, sizeof keybuf);
dbt_init(&mult_val_dbt[0].dbts[0], valbuf, sizeof valbuf);
r = env->put_multiple(
env,
dbs[0], // source db.
txn,
&mult_key_dbt[0], // source db key
&mult_val_dbt[0], // source db value
&mult_key_dbt[0].dbts[0], // source db key
&mult_val_dbt[0].dbts[0], // source db value
num_dbs, // total number of dbs
dbs, // array of dbs
mult_key_dbt, // array of keys
@ -273,13 +274,19 @@ static int UU() iibench_put_op(DB_TXN *txn, ARG arg, void *operation_extra, void
}
cleanup:
for (int i = 1; i < num_dbs; i++) {
toku_free(mult_key_dbt[i].data);
for (int i = 0; i < num_dbs; i++) {
toku_dbt_array_destroy(&mult_key_dbt[i]);
toku_dbt_array_destroy(&mult_val_dbt[i]);
}
return r;
}
static int iibench_generate_row_for_put(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *UU(src_key), const DBT *src_val) {
static int iibench_generate_row_for_put(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *UU(src_key), const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
invariant(src_db != dest_db);
// 8 byte primary key, REALLOC secondary key
invariant_notnull(src_key->data);

252
src/tests/put-multiple.cc → src/tests/put-del-multiple-array-indexing.cc
View file

@ -92,56 +92,117 @@ PATENT RIGHTS GRANT:
// verify that put_multiple inserts the correct rows into N dictionaries
// verify that pu_multiple locks the correct keys for N dictionaries
static int
get_key(int i, int dbnum) {
return htonl(i + dbnum);
const int max_rows_per_primary = 9;
static uint32_t
get_total_secondary_rows(uint32_t num_primary) {
assert((num_primary % (max_rows_per_primary+1)) == 0);
return num_primary / (max_rows_per_primary+1) *
( (max_rows_per_primary) * (max_rows_per_primary+1) / 2 );
}
static uint8_t
get_num_keys(uint16_t i, uint8_t dbnum) {
return (i+dbnum) % (max_rows_per_primary + 1); // 0..9.. 10 choices
}
static uint16_t
get_total_num_keys(uint16_t i, uint8_t num_dbs) {
uint16_t sum = 0;
for (uint8_t db = 0; db < num_dbs; ++db) {
sum += get_num_keys(i, db);
}
return sum;
}
static uint32_t
get_key(uint16_t i, uint8_t dbnum, uint8_t which) {
uint32_t i32 = i;
uint32_t dbnum32 = dbnum;
uint32_t which32 = which;
uint32_t x = (dbnum32<<24) | (i32) | (which32<<8);
return x;
}
static void
get_data(int *v, int i, int ndbs) {
for (int dbnum = 0; dbnum < ndbs; dbnum++) {
v[dbnum] = get_key(i, dbnum);
get_data(uint32_t *v, uint8_t i, uint8_t ndbs) {
int index = 0;
for (uint8_t dbnum = 0; dbnum < ndbs; dbnum++) {
for (uint8_t which = 0; which < get_num_keys(i, dbnum); ++which) {
v[index++] = get_key(i, dbnum, which);
}
}
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
assert(src_db == NULL);
unsigned int dbnum;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
(void) src_val;
uint8_t dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
int *pri_data = (int *) src_data->data;
assert(dbnum > 0); // Does not get called for primary.
assert(dest_db != src_db);
assert(dest_key->flags == 0);
dest_key->size = sizeof (int);
dest_key->data = &pri_data[dbnum];
assert(src_key->size == 2);
uint16_t i = *(uint16_t*)src_key->data;
uint8_t num_keys = get_num_keys(i, dbnum);
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
} else {
dest_data->size = 0;
toku_dbt_array_resize(dest_keys, num_keys);
if (dest_vals) {
toku_dbt_array_resize(dest_vals, num_keys);
}
for (uint8_t which = 0; which < num_keys; ++which) {
DBT *dest_key = &dest_keys->dbts[which];
assert(dest_key->flags == DB_DBT_REALLOC);
{
// Memory management
if (dest_key->ulen < sizeof(uint32_t)) {
dest_key->data = toku_xrealloc(dest_key->data, sizeof(uint32_t));
dest_key->ulen = sizeof(uint32_t);
}
dest_key->size = sizeof(uint32_t);
}
*(uint32_t*)dest_key->data = get_key(i, dbnum, which);
if (dest_vals) {
DBT *dest_val = &dest_vals->dbts[which];
dest_val->flags = 0;
dest_val->data = nullptr;
dest_val->size = 0;
}
}
return 0;
}
static int
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_keys, NULL, src_key, src_data);
}
static void
verify_locked(DB_ENV *env, DB *db, int k) {
verify_locked(DB_ENV *env, DB *db, uint8_t dbnum, uint16_t i) {
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBT key; dbt_init(&key, &k, sizeof k);
r = db->del(db, txn, &key, DB_DELETE_ANY); assert(r == DB_LOCK_NOTGRANTED);
if (dbnum == 0) {
DBT key; dbt_init(&key, &i, sizeof i);
r = db->del(db, txn, &key, DB_DELETE_ANY); CKERR2(r, DB_LOCK_NOTGRANTED);
} else {
for (uint8_t which = 0; which < get_num_keys(i, dbnum); ++which) {
uint32_t k = get_key(i, dbnum, which);
DBT key; dbt_init(&key, &k, sizeof k);
r = db->del(db, txn, &key, DB_DELETE_ANY); CKERR2(r, DB_LOCK_NOTGRANTED);
}
}
r = txn->abort(txn); assert_zero(r);
}
static void
verify_seq(DB_ENV *env, DB *db, int dbnum, int ndbs, int nrows) {
verify_seq_primary(DB_ENV *env, DB *db, int dbnum, int ndbs, int nrows) {
assert(dbnum==0);
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
@ -155,17 +216,15 @@ verify_seq(DB_ENV *env, DB *db, int dbnum, int ndbs, int nrows) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0)
break;
int k;
uint16_t k;
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
assert(k == get_key(i, dbnum));
assert(k == i);
if (dbnum == 0) {
assert(val.size == ndbs * sizeof (int));
int v[ndbs]; get_data(v, i, ndbs);
assert(memcmp(val.data, v, val.size) == 0);
} else
assert(val.size == 0);
uint32_t total_rows = get_total_num_keys(i, ndbs);
assert(val.size == total_rows * sizeof (uint32_t));
uint32_t v[total_rows]; get_data(v, i, ndbs);
assert(memcmp(val.data, v, val.size) == 0);
}
assert(i == nrows);
r = cursor->c_close(cursor); assert_zero(r);
@ -173,30 +232,106 @@ verify_seq(DB_ENV *env, DB *db, int dbnum, int ndbs, int nrows) {
}
static void
verify(DB_ENV *env, DB *db[], int ndbs, int nrows) {
for (int dbnum = 0; dbnum < ndbs; dbnum++)
verify_seq(env, db[dbnum], dbnum, ndbs, nrows);
}
static void
populate(DB_ENV *env, DB *db[], int ndbs, int nrows) {
verify_seq(DB_ENV *env, DB *db, uint8_t dbnum, uint8_t ndbs, uint16_t nrows_primary) {
assert(dbnum > 0);
assert(dbnum < ndbs);
uint32_t nrows = get_total_secondary_rows(nrows_primary);
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
uint16_t rows_found = 0;
uint16_t source_i = 0;
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
for (source_i = 0; source_i < nrows_primary; ++source_i) {
uint8_t num_keys = get_num_keys(source_i, dbnum);
for (uint8_t which = 0; which < num_keys; ++which) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
CKERR(r);
uint32_t k;
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
assert(k == get_key(source_i, dbnum, which));
assert(val.size == 0);
rows_found++;
}
}
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
CKERR2(r, DB_NOTFOUND);
assert(rows_found == nrows);
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify(DB_ENV *env, DB *db[], int ndbs, int nrows) {
verify_seq_primary(env, db[0], 0, ndbs, nrows);
for (int dbnum = 1; dbnum < ndbs; dbnum++)
verify_seq(env, db[dbnum], dbnum, ndbs, nrows);
}
static void
verify_empty(DB_ENV *env, DB *db) {
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
int i;
for (i = 0; ; i++) {
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0)
break;
}
assert_zero(i);
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify_del(DB_ENV *env, DB *db[], int ndbs) {
for (int dbnum = 0; dbnum < ndbs; dbnum++)
verify_empty(env, db[dbnum]);
}
static void
populate(DB_ENV *env, DB *db[], uint8_t ndbs, uint16_t nrows, bool del) {
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBT_ARRAY key_arrays[ndbs];
DBT_ARRAY val_arrays[ndbs];
for (uint8_t i = 0; i < ndbs; ++i) {
toku_dbt_array_init(&key_arrays[i], 1);
toku_dbt_array_init(&val_arrays[i], 1);
}
// populate
for (int i = 0; i < nrows; i++) {
int k = get_key(i, 0);
int v[ndbs]; get_data(v, i, ndbs);
for (uint16_t i = 0; i < nrows; i++) {
uint32_t total_rows = get_total_num_keys(i, ndbs);
uint16_t k = i;
uint32_t v[total_rows]; get_data(v, i, ndbs);
DBT pri_key; dbt_init(&pri_key, &k, sizeof k);
DBT pri_val; dbt_init(&pri_val, &v[0], sizeof v);
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
DBT vals[ndbs]; memset(vals, 0, sizeof vals);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->put_multiple(env, NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
if (del) {
r = env->del_multiple(env, db[0], txn, &pri_key, &pri_val, ndbs, db, key_arrays, flags);
} else {
r = env->put_multiple(env, db[0], txn, &pri_key, &pri_val, ndbs, db, key_arrays, val_arrays, flags);
}
assert_zero(r);
for (int dbnum = 0; dbnum < ndbs; dbnum++)
verify_locked(env, db[dbnum], get_key(i, dbnum));
verify_locked(env, db[dbnum], dbnum, i);
}
for (uint8_t i = 0; i < ndbs; ++i) {
toku_dbt_array_destroy(&key_arrays[i]);
toku_dbt_array_destroy(&val_arrays[i]);
}
r = txn->commit(txn, 0); assert_zero(r);
@ -209,11 +344,12 @@ run_test(int ndbs, int nrows) {
r = db_env_create(&env, 0); assert_zero(r);
r = env->set_generate_row_callback_for_put(env, put_callback); assert_zero(r);
r = env->set_generate_row_callback_for_del(env, del_callback); assert_zero(r);
r = env->open(env, TOKU_TEST_FILENAME, DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB *db[ndbs];
for (int dbnum = 0; dbnum < ndbs; dbnum++) {
for (uint8_t dbnum = 0; dbnum < ndbs; dbnum++) {
r = db_create(&db[dbnum], env, 0); assert_zero(r);
DBT dbt_dbnum; dbt_init(&dbt_dbnum, &dbnum, sizeof dbnum);
@ -227,10 +363,14 @@ run_test(int ndbs, int nrows) {
});
}
populate(env, db, ndbs, nrows);
populate(env, db, ndbs, nrows, false);
verify(env, db, ndbs, nrows);
populate(env, db, ndbs, nrows, true);
verify_del(env, db, ndbs);
for (int dbnum = 0; dbnum < ndbs; dbnum++)
r = db[dbnum]->close(db[dbnum], 0); assert_zero(r);
@ -240,8 +380,8 @@ run_test(int ndbs, int nrows) {
int
test_main(int argc, char * const argv[]) {
int r;
int ndbs = 2;
int nrows = 2;
int ndbs = 16;
int nrows = 100;
// parse_args(argc, argv);
for (int i = 1; i < argc; i++) {
@ -263,6 +403,14 @@ test_main(int argc, char * const argv[]) {
continue;
}
}
//rows should be divisible by max_rows + 1 (so that we have an equal number of each type and we know the total)
if (nrows % (max_rows_per_primary+1) != 0) {
nrows += (max_rows_per_primary+1) - (nrows % (max_rows_per_primary+1));
}
assert(ndbs >= 0);
assert(ndbs < (1<<8) - 1);
assert(nrows >= 0);
assert(nrows < (1<<15)); // Leave plenty of room
toku_os_recursive_delete(TOKU_TEST_FILENAME);
r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);

30
src/tests/recover-del-multiple-abort.cc
View file

@ -106,16 +106,20 @@ get_data(int *v, int i, int ndbs) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
assert(src_db == NULL);
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum < src_val->size / sizeof (int));
int *pri_data = (int *) src_data->data;
int *pri_data = (int *) src_val->data;
switch (dest_key->flags) {
case 0:
@ -131,14 +135,14 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
assert(0);
}
if (dest_data) {
switch (dest_data->flags) {
if (dest_val) {
switch (dest_val->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
dest_val->size = src_val->size;
dest_val->data = src_val->data;
} else {
dest_data->size = 0;
dest_val->size = 0;
}
break;
case DB_DBT_REALLOC:
@ -152,8 +156,8 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_keys, NULL, src_key, src_data);
}
static void
@ -193,7 +197,7 @@ run_test(int ndbs, int nrows) {
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
DBT vals[ndbs]; memset(vals, 0, sizeof vals);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->put_multiple(env, NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
assert_zero(r);
}
@ -210,7 +214,7 @@ run_test(int ndbs, int nrows) {
DBT pri_data; dbt_init(&pri_data, &v[0], sizeof v);
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->del_multiple(env, NULL, txn, &pri_key, &pri_data, ndbs, db, keys, flags);
r = env_del_multiple_test_no_array(env, NULL, txn, &pri_key, &pri_data, ndbs, db, keys, flags);
assert_zero(r);
}

34
src/tests/recover-del-multiple-srcdb-fdelete-all.cc
View file

@ -106,16 +106,24 @@ get_data(int *v, int i, int ndbs) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = NULL;
if (dest_vals) {
toku_dbt_array_resize(dest_vals, 1);
dest_val = &dest_vals->dbts[0];
}
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
assert(src_db != NULL);
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum < src_val->size / sizeof (int));
int *pri_data = (int *) src_data->data;
int *pri_data = (int *) src_val->data;
switch (dest_key->flags) {
case 0:
@ -131,14 +139,14 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
assert(0);
}
if (dest_data) {
switch (dest_data->flags) {
if (dest_val) {
switch (dest_val->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
dest_val->size = src_val->size;
dest_val->data = src_val->data;
} else {
dest_data->size = 0;
dest_val->size = 0;
}
break;
case DB_DBT_REALLOC:
@ -152,8 +160,8 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_keys, NULL, src_key, src_data);
}
static void
@ -193,7 +201,7 @@ run_test(int ndbs, int nrows) {
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
DBT vals[ndbs]; memset(vals, 0, sizeof vals);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->put_multiple(env, ndbs > 0 ? db[0] : NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
r = env_put_multiple_test_no_array(env, ndbs > 0 ? db[0] : NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
assert_zero(r);
}
@ -210,7 +218,7 @@ run_test(int ndbs, int nrows) {
DBT pri_data; dbt_init(&pri_data, &v[0], sizeof v);
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->del_multiple(env, ndbs > 0 ? db[0] : NULL, txn, &pri_key, &pri_data, ndbs, db, keys, flags);
r = env_del_multiple_test_no_array(env, ndbs > 0 ? db[0] : NULL, txn, &pri_key, &pri_data, ndbs, db, keys, flags);
assert_zero(r);
}

33
src/tests/recover-del-multiple.cc
View file

@ -106,16 +106,23 @@ get_data(int *v, int i, int ndbs) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = NULL;
if (dest_vals) {
toku_dbt_array_resize(dest_vals, 1);
dest_val = &dest_vals->dbts[0];
}
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
assert(src_db == NULL);
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum < src_val->size / sizeof (int));
int *pri_data = (int *) src_data->data;
int *pri_data = (int *) src_val->data;
switch (dest_key->flags) {
case 0:
@ -131,14 +138,14 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
assert(0);
}
if (dest_data) {
switch (dest_data->flags) {
if (dest_val) {
switch (dest_val->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
dest_val->size = src_val->size;
dest_val->data = src_val->data;
} else {
dest_data->size = 0;
dest_val->size = 0;
}
break;
case DB_DBT_REALLOC:
@ -152,8 +159,8 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_keys, NULL, src_key, src_data);
}
static void
@ -193,7 +200,7 @@ run_test(int ndbs, int nrows) {
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
DBT vals[ndbs]; memset(vals, 0, sizeof vals);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->put_multiple(env, NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
assert_zero(r);
}
@ -210,7 +217,7 @@ run_test(int ndbs, int nrows) {
DBT pri_data; dbt_init(&pri_data, &v[0], sizeof v);
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->del_multiple(env, NULL, txn, &pri_key, &pri_data, ndbs, db, keys, flags);
r = env_del_multiple_test_no_array(env, NULL, txn, &pri_key, &pri_data, ndbs, db, keys, flags);
assert_zero(r);
}

18
src/tests/recover-hotindexer-simple-abort-put.cc
View file

@ -90,14 +90,22 @@ PATENT RIGHTS GRANT:
#include "test.h"
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
lazy_assert(src_db != NULL && dest_db != NULL);
dest_key->data = toku_xmemdup(src_data->data, src_data->size);
dest_key->size = src_data->size;
dest_data->size = 0;
if (dest_key->flags == DB_DBT_REALLOC) {
toku_free(dest_key->data);
}
dest_key->flags = DB_DBT_REALLOC;
dest_key->data = toku_xmemdup(src_val->data, src_val->size);
dest_key->size = src_val->size;
dest_val->size = 0;
return 0;
}

6
src/tests/recover-loader-test.cc
View file

@ -307,7 +307,11 @@ static unsigned int generate_val(int key, int i) {
// There is no handlerton in this test, so this function is a local replacement
// for the handlerton's generate_row_for_put().
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) src_db;

10
src/tests/recover-lsn-filter-multiple.cc
View file

@ -104,7 +104,11 @@ static DBT dest_vals[num_dbs];
bool do_test=false, do_recover=false;
static int
put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
if (src_db) {
assert(src_db->descriptor);
assert(src_db->descriptor->dbt.size == 4);
@ -171,7 +175,7 @@ static void run_test (void) {
DBT k,v;
dbt_init(&k, "a", 2);
dbt_init(&v, "b", 2);
r = env->put_multiple(env, dba, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, dba, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
CKERR(r);
r = txn->abort(txn); CKERR(r);
}
@ -187,7 +191,7 @@ static void run_test (void) {
DBT k,v;
dbt_init(&k, "a", 2);
dbt_init(&v, "b", 2);
r = env->put_multiple(env, NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
CKERR(r);
}

28
src/tests/recover-put-multiple-abort.cc
View file

@ -106,16 +106,20 @@ get_data(int *v, int i, int ndbs) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
assert(src_db == NULL);
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum < src_val->size / sizeof (int));
int *pri_data = (int *) src_data->data;
int *pri_data = (int *) src_val->data;
switch (dest_key->flags) {
case 0:
@ -131,14 +135,14 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
assert(0);
}
if (dest_data) {
switch (dest_data->flags) {
if (dest_val) {
switch (dest_val->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
dest_val->size = src_val->size;
dest_val->data = src_val->data;
} else {
dest_data->size = 0;
dest_val->size = 0;
}
break;
case DB_DBT_REALLOC:
@ -152,8 +156,8 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_keys, NULL, src_key, src_data);
}
static void
@ -195,7 +199,7 @@ run_test(int ndbs, int nrows) {
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
DBT vals[ndbs]; memset(vals, 0, sizeof vals);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->put_multiple(env, NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
assert_zero(r);
}

11
src/tests/recover-put-multiple-fdelete-all.cc
View file

@ -104,7 +104,12 @@ static DBT dest_vals[num_dbs];
bool do_test=false, do_recover=false;
static int
put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
assert(src_db == NULL);
assert(dest_db->descriptor->dbt.size == 4);
uint32_t which = *(uint32_t*)dest_db->descriptor->dbt.data;
@ -169,7 +174,7 @@ static void run_test (void) {
dbt_init(&k, "a", 2);
dbt_init(&v, "b", 2);
r = env->put_multiple(env, NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
CKERR(r);
r = txn->abort(txn); CKERR(r);
}
@ -186,7 +191,7 @@ static void run_test (void) {
dbt_init(&k, "a", 2);
dbt_init(&v, "b", 2);
r = env->put_multiple(env, NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
CKERR(r);
r = txn->commit(txn, 0); CKERR(r);
}

10
src/tests/recover-put-multiple-fdelete-some.cc
View file

@ -104,7 +104,11 @@ static DBT dest_vals[num_dbs];
bool do_test=false, do_recover=false;
static int
put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
if (src_db) {
assert(src_db->descriptor);
assert(src_db->descriptor->dbt.size == 4);
@ -173,7 +177,7 @@ static void run_test (void) {
dbt_init(&k, "a", 2);
dbt_init(&v, "b", 2);
r = env->put_multiple(env, dba, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, dba, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
CKERR(r);
r = txn->abort(txn); CKERR(r);
}
@ -190,7 +194,7 @@ static void run_test (void) {
dbt_init(&k, "a", 2);
dbt_init(&v, "b", 2);
r = env->put_multiple(env, NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
CKERR(r);
r = txn->commit(txn, 0); CKERR(r);
}

10
src/tests/recover-put-multiple-srcdb-fdelete-all.cc
View file

@ -104,7 +104,11 @@ static DBT dest_vals[num_dbs];
bool do_test=false, do_recover=false;
static int
put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
assert(src_db != NULL);
assert(dest_db->descriptor->dbt.size == 4);
uint32_t which = *(uint32_t*)dest_db->descriptor->dbt.data;
@ -171,7 +175,7 @@ static void run_test (void) {
dbt_init(&k, "a", 2);
dbt_init(&v, "b", 2);
r = env->put_multiple(env, num_dbs > 0 ? dbs[0] : NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, num_dbs > 0 ? dbs[0] : NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
CKERR(r);
r = txn->abort(txn); CKERR(r);
}
@ -188,7 +192,7 @@ static void run_test (void) {
dbt_init(&k, "a", 2);
dbt_init(&v, "b", 2);
r = env->put_multiple(env, num_dbs > 0 ? dbs[0] : NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
r = env_put_multiple_test_no_array(env, num_dbs > 0 ? dbs[0] : NULL, txn, &k, &v, num_dbs, dbs, dest_keys, dest_vals, flags);
CKERR(r);
r = txn->commit(txn, 0); CKERR(r);
}

36
src/tests/recover-put-multiple.cc
View file

@ -106,16 +106,20 @@ get_data(int *v, int i, int ndbs) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
assert(src_db == NULL);
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum < src_val->size / sizeof (int));
int *pri_data = (int *) src_data->data;
int *pri_data = (int *) src_val->data;
switch (dest_key->flags) {
case 0:
@ -131,22 +135,22 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
assert(0);
}
if (dest_data) {
switch (dest_data->flags) {
if (dest_val) {
switch (dest_val->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
dest_val->size = src_val->size;
dest_val->data = src_val->data;
} else
dest_data->size = 0;
dest_val->size = 0;
break;
case DB_DBT_REALLOC:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = toku_realloc(dest_data->data, dest_data->size);
memcpy(dest_data->data, src_data->data, dest_data->size);
dest_val->size = src_val->size;
dest_val->data = toku_realloc(dest_val->data, dest_val->size);
memcpy(dest_val->data, src_val->data, dest_val->size);
} else
dest_data->size = 0;
dest_val->size = 0;
break;
default:
assert(0);
@ -157,8 +161,8 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key_arrays, NULL, src_key, src_data);
}
static void
@ -200,7 +204,7 @@ run_test(int ndbs, int nrows) {
DBT keys[ndbs]; memset(keys, 0, sizeof keys);
DBT vals[ndbs]; memset(vals, 0, sizeof vals);
uint32_t flags[ndbs]; memset(flags, 0, sizeof flags);
r = env->put_multiple(env, NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
r = env_put_multiple_test_no_array(env, NULL, txn, &pri_key, &pri_val, ndbs, db, keys, vals, flags);
assert_zero(r);
}

6
src/tests/recover-test-logsuppress-put.cc
View file

@ -113,7 +113,11 @@ DB_TXN *child;
// There is no handlerton in this test, so this function is a local replacement
// for the handlerton's generate_row_for_put().
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
(void) src_db;

27
src/tests/recover-test-logsuppress.cc
View file

@ -112,27 +112,20 @@ DB_TXN *child;
// There is no handlerton in this test, so this function is a local replacement
// for the handlerton's generate_row_for_put().
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_keys, 1);
toku_dbt_array_resize(dest_vals, 1);
DBT *dest_key = &dest_keys->dbts[0];
DBT *dest_val = &dest_vals->dbts[0];
(void) src_db;
(void) src_key;
(void) src_val;
uint32_t which = *(uint32_t*)dest_db->app_private;
if ( which == 0 ) {
if (dest_key->flags==DB_DBT_REALLOC) {
if (dest_key->data) toku_free(dest_key->data);
dest_key->flags = 0;
dest_key->ulen = 0;
}
if (dest_val->flags==DB_DBT_REALLOC) {
if (dest_val->data) toku_free(dest_val->data);
dest_val->flags = 0;
dest_val->ulen = 0;
}
dbt_init(dest_key, src_key->data, src_key->size);
dbt_init(dest_val, src_val->data, src_val->size);
}
else {
assert(which != 0);
assert(dest_db != src_db);
{
assert(dest_key->flags==DB_DBT_REALLOC);
if (dest_key->ulen < sizeof(unsigned int)) {
dest_key->data = toku_xrealloc(dest_key->data, sizeof(unsigned int));

355
src/tests/recover-update-multiple-abort.cc
View file

@ -94,115 +94,200 @@ PATENT RIGHTS GRANT:
static const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE;
static int
get_key(int i, int dbnum) {
return htonl(2*(i + dbnum));
get_num_new_keys(int i, int dbnum) {
if (dbnum == 0) return 1;
if (i & (1<<4)) {
dbnum++; // Shift every once in a while.
}
return (i + dbnum) % 3; // 0, 1, or 2
}
static int
get_new_key(int i, int dbnum) {
return htonl(2*(i + dbnum) + 1);
get_num_keys(int i, int dbnum) {
if (dbnum == 0) return 1;
return (i + dbnum) % 3; // 0, 1, or 2
}
static int
get_total_secondary_rows(int num_primary) {
assert(num_primary % 3 == 0);
return num_primary / 3 * (0 + 1 + 2);
}
static int
get_total_num_keys(int i, int num_dbs) {
int sum = 0;
for (int db = 1; db < num_dbs; ++db) {
sum += get_num_keys(i, db);
}
return sum;
}
static int
get_total_num_new_keys(int i, int num_dbs) {
int sum = 0;
for (int db = 1; db < num_dbs; ++db) {
sum += get_num_new_keys(i, db);
}
return sum;
}
static int
get_key(int i, int dbnum, int which) {
assert(i < INT16_MAX / 2);
assert(which >= 0);
assert(which < get_num_keys(i, dbnum));
assert(which < 4);
assert(dbnum < 16);
if (dbnum == 0) {
assert(which == 0);
return htonl((2*i) << 16);
} else {
return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1));
}
}
static int
get_new_key(int i, int dbnum, int which) {
assert(which >= 0);
assert(which < get_num_new_keys(i, dbnum));
assert(which < 4);
assert(dbnum < 16);
if (dbnum == 0) {
assert(which == 0);
return htonl((2*i+1) << 16);
} else if ((i+dbnum+which) & (1<<5)) {
return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1)); // no change from original
} else {
return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1) + 1);
}
}
static void
get_data(int *v, int i, int ndbs) {
for (int dbnum = 0; dbnum < ndbs; dbnum++) {
v[dbnum] = get_key(i, dbnum);
int index = 0;
for (int dbnum = 1; dbnum < ndbs; dbnum++) {
for (int which = 0; which < get_num_keys(i, dbnum); ++which) {
v[index++] = get_key(i, dbnum, which);
}
}
}
static void
get_new_data(int *v, int i, int ndbs) {
for (int dbnum = 0; dbnum < ndbs; dbnum++) {
if ((i % ndbs) == dbnum)
v[dbnum] = get_new_key(i, dbnum);
else
v[dbnum] = get_key(i, dbnum);
int index = 0;
for (int dbnum = 1; dbnum < ndbs; dbnum++) {
for (int which = 0; which < get_num_new_keys(i, dbnum); ++which) {
v[index++] = get_new_key(i, dbnum, which);
if (which > 0) {
assert(index >= 2);
assert(memcmp(&v[index-2], &v[index-1], sizeof(v[0])) < 0);
}
}
}
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
assert(src_db == NULL);
unsigned int dbnum;
static int
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
(void)src_val;
assert(src_db != dest_db);
assert(src_db);
int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum > 0);
int *pri_key = (int *) src_key->data;
int *pri_data = (int *) src_data->data;
int pri_key = *(int *) src_key->data;
int* pri_val = (int*) src_val->data;
switch (dest_key->flags) {
case 0:
dest_key->size = sizeof (int);
dest_key->data = dbnum == 0 ? &pri_key[dbnum] : &pri_data[dbnum];
break;
case DB_DBT_REALLOC:
dest_key->size = sizeof (int);
dest_key->data = toku_realloc(dest_key->data, dest_key->size);
memcpy(dest_key->data, dbnum == 0 ? &pri_key[dbnum] : &pri_data[dbnum], dest_key->size);
break;
default:
assert(0);
bool is_new = (ntohl(pri_key) >> 16) % 2 == 1;
int i = (ntohl(pri_key) >> 16) / 2;
int num_keys = is_new ? get_num_new_keys(i, dbnum) : get_num_keys(i, dbnum);
toku_dbt_array_resize(dest_key_arrays, num_keys);
if (dest_val_arrays) {
toku_dbt_array_resize(dest_val_arrays, num_keys);
}
if (dest_data) {
switch (dest_data->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
} else
dest_data->size = 0;
break;
case DB_DBT_REALLOC:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = toku_realloc(dest_data->data, dest_data->size);
memcpy(dest_data->data, src_data->data, dest_data->size);
} else
dest_data->size = 0;
break;
default:
assert(0);
int index = 0;
for (int idb = 1; idb < dbnum; idb++) {
index += is_new ? get_num_new_keys(i, idb) : get_num_keys(i, idb);
}
assert(src_val->size % sizeof(int) == 0);
assert((int)src_val->size / 4 >= index + num_keys);
for (int which = 0; which < num_keys; which++) {
DBT *dest_key = &dest_key_arrays->dbts[which];
DBT *dest_val = NULL;
assert(dest_key->flags == DB_DBT_REALLOC);
if (dest_key->ulen < sizeof(int)) {
dest_key->data = toku_xrealloc(dest_key->data, sizeof(int));
dest_key->ulen = sizeof(int);
}
dest_key->size = sizeof(int);
if (dest_val_arrays) {
dest_val = &dest_val_arrays->dbts[which];
assert(dest_val->flags == DB_DBT_REALLOC);
dest_val->size = 0;
}
int new_key = is_new ? get_new_key(i, dbnum, which) : get_key(i, dbnum, which);
assert(new_key == pri_val[index + which]);
*(int*)dest_key->data = new_key;
}
return 0;
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_keys, NULL, src_key, src_data);
}
static void
update_diagonal(DB_ENV *env, DB_TXN *txn, DB *db[], int ndbs, int nrows) {
assert(ndbs > 0);
int r;
int narrays = 2 * ndbs;
DBT_ARRAY keys[narrays];
DBT_ARRAY vals[narrays];
for (int i = 0; i < narrays; i++) {
toku_dbt_array_init(&keys[i], 1);
toku_dbt_array_init(&vals[i], 1);
}
for (int i = 0; i < nrows; i++) {
// update the data i % ndbs col from x to x+1
int k = get_key(i, 0);
DBT old_key; dbt_init(&old_key, &k, sizeof k);
DBT new_key = old_key;
int old_k = get_key(i, 0, 0);
DBT old_key; dbt_init(&old_key, &old_k, sizeof old_k);
int new_k = get_new_key(i, 0, 0);
DBT new_key; dbt_init(&new_key, &new_k, sizeof new_k);
int v[ndbs]; get_data(v, i, ndbs);
int num_old_keys = get_total_num_keys(i, ndbs);
int v[num_old_keys]; get_data(v, i, ndbs);
DBT old_data; dbt_init(&old_data, &v[0], sizeof v);
int newv[ndbs]; get_new_data(newv, i, ndbs);
int num_new_keys = get_total_num_new_keys(i, ndbs);
int newv[num_new_keys]; get_new_data(newv, i, ndbs);
DBT new_data; dbt_init(&new_data, &newv[0], sizeof newv);
int ndbts = 2 * ndbs;
DBT keys[ndbts]; memset(keys, 0, sizeof keys);
DBT vals[ndbts]; memset(vals, 0, sizeof vals);
uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array));
r = env->update_multiple(env, NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
r = env->update_multiple(env, db[0], txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, narrays, keys, narrays, vals);
assert_zero(r);
}
for (int i = 0; i < narrays; i++) {
toku_dbt_array_destroy(&keys[i]);
toku_dbt_array_destroy(&vals[i]);
}
}
static void
@ -213,8 +298,9 @@ populate_primary(DB_ENV *env, DB *db, int ndbs, int nrows) {
// populate
for (int i = 0; i < nrows; i++) {
int k = get_key(i, 0);
int v[ndbs]; get_data(v, i, ndbs);
int k = get_key(i, 0, 0);
int secondary_keys = get_total_num_keys(i, ndbs);
int v[secondary_keys]; get_data(v, i, ndbs);
DBT key; dbt_init(&key, &k, sizeof k);
DBT val; dbt_init(&val, &v[0], sizeof v);
r = db->put(db, txn, &key, &val, 0); assert_zero(r);
@ -231,15 +317,100 @@ populate_secondary(DB_ENV *env, DB *db, int dbnum, int nrows) {
// populate
for (int i = 0; i < nrows; i++) {
int k = get_key(i, dbnum);
DBT key; dbt_init(&key, &k, sizeof k);
DBT val; dbt_init(&val, NULL, 0);
r = db->put(db, txn, &key, &val, 0); assert_zero(r);
for (int which = 0; which < get_num_keys(i, dbnum); which++) {
int k = get_key(i, dbnum, which);
DBT key; dbt_init(&key, &k, sizeof k);
DBT val; dbt_init(&val, NULL, 0);
r = db->put(db, txn, &key, &val, 0); assert_zero(r);
}
}
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify_pri_seq(DB_ENV *env, DB *db, int ndbs, int nrows) {
const int dbnum = 0;
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
int i;
for (i = 0; ; i++) {
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0)
break;
int k;
int expectk = get_key(i, dbnum, 0);
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
assert(k == expectk);
int num_keys = get_total_num_keys(i, ndbs);
assert(val.size == num_keys*sizeof(int));
int v[num_keys]; get_data(v, i, ndbs);
assert(memcmp(val.data, v, val.size) == 0);
}
assert(i == nrows); // if (i != nrows) printf("%s:%d %d %d\n", __FUNCTION__, __LINE__, i, nrows); // assert(i == nrows);
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify_sec_seq(DB_ENV *env, DB *db, int dbnum, int nrows) {
assert(dbnum > 0);
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
int i;
int rows_found = 0;
for (i = 0; ; i++) {
int num_keys = get_num_keys(i, dbnum);
for (int which = 0; which < num_keys; ++which) {
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0) {
CKERR2(r, DB_NOTFOUND);
goto done;
}
rows_found++;
int k;
int expectk = get_key(i, dbnum, which);
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
int got_i = (ntohl(k) >> 16) / 2;
if (got_i < i) {
// Will fail. Too many old i's
assert(k == expectk);
} else if (got_i > i) {
// Will fail. Too few in previous i.
assert(k == expectk);
}
if (k != expectk && which < get_num_new_keys(i, dbnum) && k == get_new_key(i, dbnum, which)) {
// Will fail, never got updated.
assert(k == expectk);
}
assert(k == expectk);
assert(val.size == 0);
}
}
done:
assert(rows_found == get_total_secondary_rows(nrows));
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
run_test(int ndbs, int nrows) {
@ -287,50 +458,21 @@ run_test(int ndbs, int nrows) {
toku_hard_crash_on_purpose();
}
static void
verify_seq(DB_ENV *env, DB *db, int dbnum, int ndbs, int nrows) {
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
int i;
for (i = 0; ; i++) {
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0)
break;
int k;
int expectk = get_key(i, dbnum);
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
assert(k == expectk);
if (dbnum == 0) {
assert(val.size == ndbs * sizeof (int));
int v[ndbs]; get_data(v, i, ndbs);
assert(memcmp(val.data, v, val.size) == 0);
} else
assert(val.size == 0);
}
assert(i == nrows);
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify_all(DB_ENV *env, int ndbs, int nrows) {
int r;
for (int dbnum = 0; dbnum < ndbs; dbnum++) {
DB *db = NULL;
r = db_create(&db, env, 0);
r = db_create(&db, env, 0);
assert_zero(r);
char dbname[32]; sprintf(dbname, "%d.tdb", dbnum);
r = db->open(db, NULL, dbname, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666);
r = db->open(db, NULL, dbname, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666);
assert_zero(r);
verify_seq(env, db, dbnum, ndbs, nrows);
if (dbnum == 0) {
verify_pri_seq(env, db, ndbs, nrows);
} else {
verify_sec_seq(env, db, dbnum, nrows);
}
r = db->close(db, 0);
assert_zero(r);
}
@ -359,7 +501,7 @@ test_main (int argc, char * const argv[]) {
bool do_test = false;
bool do_recover = false;
int ndbs = 2;
int nrows = 2;
int nrows = 3*(1<<5)*4;
for (int i = 1; i < argc; i++) {
char * const arg = argv[i];
@ -393,6 +535,9 @@ test_main (int argc, char * const argv[]) {
return usage();
}
}
while (nrows % (3*(1<<5)) != 0) {
nrows++;
}
if (do_test)
run_test(ndbs, nrows);

362
src/tests/recover-update-multiple.cc
View file

@ -94,115 +94,200 @@ PATENT RIGHTS GRANT:
static const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE;
static int
get_key(int i, int dbnum) {
return htonl(2*(i + dbnum));
get_num_new_keys(int i, int dbnum) {
if (dbnum == 0) return 1;
if (i & (1<<4)) {
dbnum++; // Shift every once in a while.
}
return (i + dbnum) % 3; // 0, 1, or 2
}
static int
get_new_key(int i, int dbnum) {
return htonl(2*(i + dbnum) + 1);
get_num_keys(int i, int dbnum) {
if (dbnum == 0) return 1;
return (i + dbnum) % 3; // 0, 1, or 2
}
static int
get_total_secondary_rows(int num_primary) {
assert(num_primary % 3 == 0);
return num_primary / 3 * (0 + 1 + 2);
}
static int
get_total_num_keys(int i, int num_dbs) {
int sum = 0;
for (int db = 1; db < num_dbs; ++db) {
sum += get_num_keys(i, db);
}
return sum;
}
static int
get_total_num_new_keys(int i, int num_dbs) {
int sum = 0;
for (int db = 1; db < num_dbs; ++db) {
sum += get_num_new_keys(i, db);
}
return sum;
}
static int
get_key(int i, int dbnum, int which) {
assert(i < INT16_MAX / 2);
assert(which >= 0);
assert(which < get_num_keys(i, dbnum));
assert(which < 4);
assert(dbnum < 16);
if (dbnum == 0) {
assert(which == 0);
return htonl((2*i) << 16);
} else {
return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1));
}
}
static int
get_new_key(int i, int dbnum, int which) {
assert(which >= 0);
assert(which < get_num_new_keys(i, dbnum));
assert(which < 4);
assert(dbnum < 16);
if (dbnum == 0) {
assert(which == 0);
return htonl((2*i+1) << 16);
} else if ((i+dbnum+which) & (1<<5)) {
return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1)); // no change from original
} else {
return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1) + 1);
}
}
static void
get_data(int *v, int i, int ndbs) {
for (int dbnum = 0; dbnum < ndbs; dbnum++) {
v[dbnum] = get_key(i, dbnum);
int index = 0;
for (int dbnum = 1; dbnum < ndbs; dbnum++) {
for (int which = 0; which < get_num_keys(i, dbnum); ++which) {
v[index++] = get_key(i, dbnum, which);
}
}
}
static void
get_new_data(int *v, int i, int ndbs) {
for (int dbnum = 0; dbnum < ndbs; dbnum++) {
if ((i % ndbs) == dbnum)
v[dbnum] = get_new_key(i, dbnum);
else
v[dbnum] = get_key(i, dbnum);
int index = 0;
for (int dbnum = 1; dbnum < ndbs; dbnum++) {
for (int which = 0; which < get_num_new_keys(i, dbnum); ++which) {
v[index++] = get_new_key(i, dbnum, which);
if (which > 0) {
assert(index >= 2);
assert(memcmp(&v[index-2], &v[index-1], sizeof(v[0])) < 0);
}
}
}
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
assert(src_db == NULL);
unsigned int dbnum;
static int
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
(void)src_val;
assert(src_db != dest_db);
assert(src_db);
int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum > 0);
int *pri_key = (int *) src_key->data;
int *pri_data = (int *) src_data->data;
int pri_key = *(int *) src_key->data;
int* pri_val = (int*) src_val->data;
switch (dest_key->flags) {
case 0:
dest_key->size = sizeof (int);
dest_key->data = dbnum == 0 ? &pri_key[dbnum] : &pri_data[dbnum];
break;
case DB_DBT_REALLOC:
dest_key->size = sizeof (int);
dest_key->data = toku_realloc(dest_key->data, dest_key->size);
memcpy(dest_key->data, dbnum == 0 ? &pri_key[dbnum] : &pri_data[dbnum], dest_key->size);
break;
default:
assert(0);
bool is_new = (ntohl(pri_key) >> 16) % 2 == 1;
int i = (ntohl(pri_key) >> 16) / 2;
int num_keys = is_new ? get_num_new_keys(i, dbnum) : get_num_keys(i, dbnum);
toku_dbt_array_resize(dest_key_arrays, num_keys);
if (dest_val_arrays) {
toku_dbt_array_resize(dest_val_arrays, num_keys);
}
if (dest_data) {
switch (dest_data->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
} else
dest_data->size = 0;
break;
case DB_DBT_REALLOC:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = toku_realloc(dest_data->data, dest_data->size);
memcpy(dest_data->data, src_data->data, dest_data->size);
} else
dest_data->size = 0;
break;
default:
assert(0);
int index = 0;
for (int idb = 1; idb < dbnum; idb++) {
index += is_new ? get_num_new_keys(i, idb) : get_num_keys(i, idb);
}
assert(src_val->size % sizeof(int) == 0);
assert((int)src_val->size / 4 >= index + num_keys);
for (int which = 0; which < num_keys; which++) {
DBT *dest_key = &dest_key_arrays->dbts[which];
DBT *dest_val = NULL;
assert(dest_key->flags == DB_DBT_REALLOC);
if (dest_key->ulen < sizeof(int)) {
dest_key->data = toku_xrealloc(dest_key->data, sizeof(int));
dest_key->ulen = sizeof(int);
}
dest_key->size = sizeof(int);
if (dest_val_arrays) {
dest_val = &dest_val_arrays->dbts[which];
assert(dest_val->flags == DB_DBT_REALLOC);
dest_val->size = 0;
}
int new_key = is_new ? get_new_key(i, dbnum, which) : get_key(i, dbnum, which);
assert(new_key == pri_val[index + which]);
*(int*)dest_key->data = new_key;
}
return 0;
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key_arrays, NULL, src_key, src_data);
}
static void
update_diagonal(DB_ENV *env, DB_TXN *txn, DB *db[], int ndbs, int nrows) {
assert(ndbs > 0);
int r;
int narrays = 2 * ndbs;
DBT_ARRAY keys[narrays];
DBT_ARRAY vals[narrays];
for (int i = 0; i < narrays; i++) {
toku_dbt_array_init(&keys[i], 1);
toku_dbt_array_init(&vals[i], 1);
}
for (int i = 0; i < nrows; i++) {
// update the data i % ndbs col from x to x+1
int k = get_key(i, 0);
DBT old_key; dbt_init(&old_key, &k, sizeof k);
DBT new_key = old_key;
int old_k = get_key(i, 0, 0);
DBT old_key; dbt_init(&old_key, &old_k, sizeof old_k);
int new_k = get_new_key(i, 0, 0);
DBT new_key; dbt_init(&new_key, &new_k, sizeof new_k);
int v[ndbs]; get_data(v, i, ndbs);
int num_old_keys = get_total_num_keys(i, ndbs);
int v[num_old_keys]; get_data(v, i, ndbs);
DBT old_data; dbt_init(&old_data, &v[0], sizeof v);
int newv[ndbs]; get_new_data(newv, i, ndbs);
int num_new_keys = get_total_num_new_keys(i, ndbs);
int newv[num_new_keys]; get_new_data(newv, i, ndbs);
DBT new_data; dbt_init(&new_data, &newv[0], sizeof newv);
int ndbts = 2 * ndbs;
DBT keys[ndbts]; memset(keys, 0, sizeof keys);
DBT vals[ndbts]; memset(vals, 0, sizeof vals);
uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array));
r = env->update_multiple(env, NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
r = env->update_multiple(env, db[0], txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, narrays, keys, narrays, vals);
assert_zero(r);
}
for (int i = 0; i < narrays; i++) {
toku_dbt_array_destroy(&keys[i]);
toku_dbt_array_destroy(&vals[i]);
}
}
static void
@ -213,8 +298,9 @@ populate_primary(DB_ENV *env, DB *db, int ndbs, int nrows) {
// populate
for (int i = 0; i < nrows; i++) {
int k = get_key(i, 0);
int v[ndbs]; get_data(v, i, ndbs);
int k = get_key(i, 0, 0);
int secondary_keys = get_total_num_keys(i, ndbs);
int v[secondary_keys]; get_data(v, i, ndbs);
DBT key; dbt_init(&key, &k, sizeof k);
DBT val; dbt_init(&val, &v[0], sizeof v);
r = db->put(db, txn, &key, &val, 0); assert_zero(r);
@ -231,15 +317,100 @@ populate_secondary(DB_ENV *env, DB *db, int dbnum, int nrows) {
// populate
for (int i = 0; i < nrows; i++) {
int k = get_key(i, dbnum);
DBT key; dbt_init(&key, &k, sizeof k);
DBT val; dbt_init(&val, NULL, 0);
r = db->put(db, txn, &key, &val, 0); assert_zero(r);
for (int which = 0; which < get_num_keys(i, dbnum); which++) {
int k = get_key(i, dbnum, which);
DBT key; dbt_init(&key, &k, sizeof k);
DBT val; dbt_init(&val, NULL, 0);
r = db->put(db, txn, &key, &val, 0); assert_zero(r);
}
}
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify_pri_seq(DB_ENV *env, DB *db, int ndbs, int nrows) {
const int dbnum = 0;
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
int i;
for (i = 0; ; i++) {
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0)
break;
int k;
int expectk = get_new_key(i, dbnum, 0);
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
assert(k == expectk);
int num_keys = get_total_num_new_keys(i, ndbs);
assert(val.size == num_keys*sizeof(int));
int v[num_keys]; get_new_data(v, i, ndbs);
assert(memcmp(val.data, v, val.size) == 0);
}
assert(i == nrows); // if (i != nrows) printf("%s:%d %d %d\n", __FUNCTION__, __LINE__, i, nrows); // assert(i == nrows);
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify_sec_seq(DB_ENV *env, DB *db, int dbnum, int nrows) {
assert(dbnum > 0);
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
int i;
int rows_found = 0;
for (i = 0; ; i++) {
int num_keys = get_num_new_keys(i, dbnum);
for (int which = 0; which < num_keys; ++which) {
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0) {
CKERR2(r, DB_NOTFOUND);
goto done;
}
rows_found++;
int k;
int expectk = get_new_key(i, dbnum, which);
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
int got_i = (ntohl(k) >> 16) / 2;
if (got_i < i) {
// Will fail. Too many old i's
assert(k == expectk);
} else if (got_i > i) {
// Will fail. Too few in previous i.
assert(k == expectk);
}
if (k != expectk && which < get_num_keys(i, dbnum) && k == get_key(i, dbnum, which)) {
// Will fail, never got updated.
assert(k == expectk);
}
assert(k == expectk);
assert(val.size == 0);
}
}
done:
assert(rows_found == get_total_secondary_rows(nrows));
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
run_test(int ndbs, int nrows) {
@ -285,47 +456,17 @@ run_test(int ndbs, int nrows) {
update_diagonal(env, txn, db, ndbs, nrows);
r = txn->commit(txn, 0); assert_zero(r);
for (int dbnum = 0; dbnum < ndbs; dbnum++) {
if (dbnum == 0) {
verify_pri_seq(env, db[0], ndbs, nrows);
} else {
verify_sec_seq(env, db[dbnum], dbnum, nrows);
}
}
toku_hard_crash_on_purpose();
}
static void
verify_seq(DB_ENV *env, DB *db, int dbnum, int ndbs, int nrows) {
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
int i;
for (i = 0; ; i++) {
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0)
break;
int k;
int expectk;
if (dbnum == 0 || (i % ndbs) != dbnum)
expectk = get_key(i, dbnum);
else
expectk = get_new_key(i, dbnum);
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
assert(k == expectk);
if (dbnum == 0) {
assert(val.size == ndbs * sizeof (int));
int v[ndbs]; get_new_data(v, i, ndbs);
assert(memcmp(val.data, v, val.size) == 0);
} else
assert(val.size == 0);
}
assert(i == nrows); // if (i != nrows) printf("%s:%d %d %d\n", __FUNCTION__, __LINE__, i, nrows); // assert(i == nrows);
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify_all(DB_ENV *env, int ndbs, int nrows) {
@ -337,7 +478,11 @@ verify_all(DB_ENV *env, int ndbs, int nrows) {
char dbname[32]; sprintf(dbname, "%d.tdb", dbnum);
r = db->open(db, NULL, dbname, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666);
assert_zero(r);
verify_seq(env, db, dbnum, ndbs, nrows);
if (dbnum == 0) {
verify_pri_seq(env, db, ndbs, nrows);
} else {
verify_sec_seq(env, db, dbnum, nrows);
}
r = db->close(db, 0);
assert_zero(r);
}
@ -366,7 +511,7 @@ test_main (int argc, char * const argv[]) {
bool do_test = false;
bool do_recover = false;
int ndbs = 2;
int nrows = 2;
int nrows = 3*(1<<5)*4;
for (int i = 1; i < argc; i++) {
char * const arg = argv[i];
@ -400,6 +545,9 @@ test_main (int argc, char * const argv[]) {
return usage();
}
}
while (nrows % (3*(1<<5)) != 0) {
nrows++;
}
if (do_test)
run_test(ndbs, nrows);

113
src/tests/test.h
View file

@ -392,6 +392,119 @@ default_parse_args (int argc, char * const argv[]) {
}
}
UU()
static void copy_dbt(DBT *dest, const DBT *src) {
assert(dest->flags & DB_DBT_REALLOC);
dest->data = toku_xrealloc(dest->data, src->size);
dest->size = src->size;
memcpy(dest->data, src->data, src->size);
}
UU()
static int
env_update_multiple_test_no_array(
DB_ENV *env,
DB *src_db,
DB_TXN *txn,
DBT *old_src_key, DBT *old_src_data,
DBT *new_src_key, DBT *new_src_data,
uint32_t num_dbs, DB **db_array, uint32_t* flags_array,
uint32_t num_keys, DBT keys[],
uint32_t num_vals, DBT vals[]) {
int r;
DBT_ARRAY key_arrays[num_keys];
DBT_ARRAY val_arrays[num_vals];
for (uint32_t i = 0; i < num_keys; i++) {
toku_dbt_array_init(&key_arrays[i], 1);
key_arrays[i].dbts[0] = keys[i];
}
for (uint32_t i = 0; i < num_vals; i++) {
toku_dbt_array_init(&val_arrays[i], 1);
val_arrays[i].dbts[0] = vals[i];
}
r = env->update_multiple(env, src_db, txn, old_src_key, old_src_data, new_src_key, new_src_data,
num_dbs, db_array, flags_array,
num_keys, &key_arrays[0],
num_vals, &val_arrays[0]);
for (uint32_t i = 0; i < num_keys; i++) {
invariant(key_arrays[i].size == 1);
invariant(key_arrays[i].capacity == 1);
keys[i] = key_arrays[i].dbts[0];
toku_dbt_array_destroy_shallow(&key_arrays[i]);
}
for (uint32_t i = 0; i < num_vals; i++) {
invariant(val_arrays[i].size == 1);
invariant(val_arrays[i].capacity == 1);
vals[i] = val_arrays[i].dbts[0];
toku_dbt_array_destroy_shallow(&val_arrays[i]);
}
return r;
}
UU()
static int env_put_multiple_test_no_array(
DB_ENV *env,
DB *src_db,
DB_TXN *txn,
const DBT *src_key,
const DBT *src_val,
uint32_t num_dbs,
DB **db_array,
DBT *keys,
DBT *vals,
uint32_t *flags_array)
{
int r;
DBT_ARRAY key_arrays[num_dbs];
DBT_ARRAY val_arrays[num_dbs];
for (uint32_t i = 0; i < num_dbs; i++) {
toku_dbt_array_init(&key_arrays[i], 1);
toku_dbt_array_init(&val_arrays[i], 1);
key_arrays[i].dbts[0] = keys[i];
val_arrays[i].dbts[0] = vals[i];
}
r = env->put_multiple(env, src_db, txn, src_key, src_val, num_dbs, db_array, &key_arrays[0], &val_arrays[0], flags_array);
for (uint32_t i = 0; i < num_dbs; i++) {
invariant(key_arrays[i].size == 1);
invariant(key_arrays[i].capacity == 1);
invariant(val_arrays[i].size == 1);
invariant(val_arrays[i].capacity == 1);
keys[i] = key_arrays[i].dbts[0];
vals[i] = val_arrays[i].dbts[0];
toku_dbt_array_destroy_shallow(&key_arrays[i]);
toku_dbt_array_destroy_shallow(&val_arrays[i]);
}
return r;
}
UU()
static int env_del_multiple_test_no_array(
DB_ENV *env,
DB *src_db,
DB_TXN *txn,
const DBT *src_key,
const DBT *src_val,
uint32_t num_dbs,
DB **db_array,
DBT *keys,
uint32_t *flags_array)
{
int r;
DBT_ARRAY key_arrays[num_dbs];
for (uint32_t i = 0; i < num_dbs; i++) {
toku_dbt_array_init(&key_arrays[i], 1);
key_arrays[i].dbts[0] = keys[i];
}
r = env->del_multiple(env, src_db, txn, src_key, src_val, num_dbs, db_array, &key_arrays[0], flags_array);
for (uint32_t i = 0; i < num_dbs; i++) {
invariant(key_arrays[i].size == 1);
invariant(key_arrays[i].capacity == 1);
keys[i] = key_arrays[i].dbts[0];
toku_dbt_array_destroy_shallow(&key_arrays[i]);
}
return r;
}
/* Some macros for evaluating blocks or functions within the scope of a
* transaction. */
#define IN_TXN_COMMIT(env, parent, txn, flags, expr) ({ \

6
src/tests/test_3529_insert_2.cc
View file

@ -117,7 +117,11 @@ my_compare(DB *this_db UU(), const DBT *a UU(), const DBT *b UU()) {
}
static int
my_generate_row(DB *dest_db UU(), DB *src_db UU(), DBT *dest_key UU(), DBT *dest_val UU(), const DBT *src_key UU(), const DBT *src_val UU()) {
my_generate_row(DB *dest_db UU(), DB *src_db UU(), DBT_ARRAY *dest_key_arrays UU(), DBT_ARRAY *dest_val_arrays UU(), const DBT *src_key UU(), const DBT *src_val UU()) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
assert(dest_key->flags == DB_DBT_REALLOC);
dest_key->data = toku_realloc(dest_key->data, src_key->size);
memcpy(dest_key->data, src_key->data, src_key->size);

6
src/tests/test_3529_table_lock.cc
View file

@ -117,7 +117,11 @@ my_compare(DB *this_db UU(), const DBT *a UU(), const DBT *b UU()) {
}
static int
my_generate_row(DB *dest_db UU(), DB *src_db UU(), DBT *dest_key UU(), DBT *dest_val UU(), const DBT *src_key UU(), const DBT *src_val UU()) {
my_generate_row(DB *dest_db UU(), DB *src_db UU(), DBT_ARRAY *dest_key_arrays UU(), DBT_ARRAY *dest_val_arrays UU(), const DBT *src_key UU(), const DBT *src_val UU()) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
assert(dest_key->flags == DB_DBT_REALLOC);
dest_key->data = toku_realloc(dest_key->data, src_key->size);
memcpy(dest_key->data, src_key->data, src_key->size);

8
src/tests/test_5469.cc
View file

@ -98,7 +98,13 @@ PATENT RIGHTS GRANT:
#include <db.h>
static int put_multiple_generate(DB *UU(dest_db), DB *UU(src_db), DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
static int put_multiple_generate(DB *UU(dest_db), DB *UU(src_db), DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
dest_key->flags = 0;
dest_val->flags = 0;
dbt_init(dest_key, src_key->data, src_key->size);
dbt_init(dest_val, src_val->data, src_val->size);
return 0;

8
src/tests/test_cmp_descriptor.cc
View file

@ -103,12 +103,16 @@ uint64_t eight_byte_desc = 0x12345678ffffffff;
static int generate_row_for_put(
DB *UU(dest_db),
DB *UU(src_db),
DBT *dest_key,
DBT *dest_val,
DBT_ARRAY *dest_key_arrays,
DBT_ARRAY *dest_val_arrays,
const DBT *src_key,
const DBT *src_val
)
{
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
dest_key->data = src_key->data;
dest_key->size = src_key->size;
dest_key->flags = 0;

12
src/tests/test_read_txn_invalid_ops.cc
View file

@ -105,8 +105,8 @@ static int update_fun(DB *UU(db),
static int generate_row_for_put(
DB *UU(dest_db),
DB *UU(src_db),
DBT *UU(dest_key),
DBT *UU(dest_val),
DBT_ARRAY *UU(dest_key_arrays),
DBT_ARRAY *UU(dest_val_arrays),
const DBT *UU(src_key),
const DBT *UU(src_val)
)
@ -118,7 +118,7 @@ static int generate_row_for_put(
static int generate_row_for_del(
DB *UU(dest_db),
DB *UU(src_db),
DBT *UU(dest_key),
DBT_ARRAY *UU(dest_key_arrays),
const DBT *UU(src_key),
const DBT *UU(src_val)
)
@ -202,12 +202,12 @@ static void test_invalid_ops(uint32_t iso_flags) {
r = db->update_broadcast(db, txn, &val, 0);
CKERR2(r, EINVAL);
r = env->put_multiple(env, NULL, txn, &key, &val, 1, &db, &key, &val, 0);
r = env_put_multiple_test_no_array(env, NULL, txn, &key, &val, 1, &db, &key, &val, 0);
CKERR2(r, EINVAL);
r = env->del_multiple(env, NULL, txn, &key, &val, 1, &db, &key, 0);
r = env_del_multiple_test_no_array(env, NULL, txn, &key, &val, 1, &db, &key, 0);
CKERR2(r, EINVAL);
uint32_t flags;
r = env->update_multiple(
r = env_update_multiple_test_no_array(
env, NULL, txn,
&key, &val,
&key, &val,

12
src/tests/test_redirect_func.cc
View file

@ -110,12 +110,12 @@ static int update_fun(DB *UU(db),
static int generate_row_for_del(
DB *UU(dest_db),
DB *UU(src_db),
DBT *dest_key,
DBT_ARRAY *dest_key_arrays,
const DBT *UU(src_key),
const DBT *UU(src_val)
)
{
dest_key->size=0;
toku_dbt_array_resize(dest_key_arrays, 0);
assert(false);
return 0;
}
@ -123,12 +123,16 @@ static int generate_row_for_del(
static int generate_row_for_put(
DB *UU(dest_db),
DB *UU(src_db),
DBT *dest_key,
DBT *dest_val,
DBT_ARRAY *dest_key_arrays,
DBT_ARRAY *dest_val_arrays,
const DBT *src_key,
const DBT *src_val
)
{
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
dest_key->size=src_key->size;
dest_key->data=src_key->data;
dest_key->flags = 0;

4
src/tests/test_stress5.cc
View file

@ -130,6 +130,10 @@ stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) {
myargs[1].operation = scan_op;
// make the guy that updates the db
struct loader_op_extra loe;
loe.soe = soe[0];
loe.num_dbs = 3;
myargs[2].operation_extra = &loe;
myargs[2].operation = loader_op;
myargs[3].operation = keyrange_op;
myargs[4].operation = get_key_after_bytes_op;

44
src/tests/test_stress_hot_indexing.cc
View file

@ -107,27 +107,33 @@ uint8_t hi_gid[DB_GID_SIZE];
static int
hi_put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
hi_put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
lazy_assert(src_db != NULL && dest_db != NULL);
if (dest_key->data) {
toku_free(dest_key->data);
dest_key->data = NULL;
}
if (dest_data->data) {
toku_free(dest_data->data);
dest_data->data = NULL;
if (dest_val->data) {
toku_free(dest_val->data);
dest_val->data = NULL;
}
dest_key->data = toku_xmemdup(src_key->data, src_key->size);
dest_key->size = src_key->size;
dest_data->data = toku_xmemdup(src_data->data, src_data->size);
dest_data->size = src_data->size;
dest_val->data = toku_xmemdup(src_val->data, src_val->size);
dest_val->size = src_val->size;
return 0;
}
static int
hi_del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT* UU(src_data)) {
hi_del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT* UU(src_data)) {
toku_dbt_array_resize(dest_key_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
lazy_assert(src_db != NULL && dest_db != NULL);
if (dest_key->data) {
toku_free(dest_key->data);
@ -149,10 +155,12 @@ static int hi_inserts(DB_TXN* UU(txn), ARG arg, void* UU(operation_extra), void
uint32_t flags[2];
flags[0] = 0;
flags[1] = 0;
DBT dest_keys[2];
DBT dest_vals[2];
memset(dest_keys, 0, sizeof(dest_keys));
memset(dest_vals, 0, sizeof(dest_vals));
DBT_ARRAY dest_keys[2];
DBT_ARRAY dest_vals[2];
for (int j = 0; j < 2; j++) {
toku_dbt_array_init(&dest_keys[j], 1);
toku_dbt_array_init(&dest_vals[j], 1);
}
DBT key, val;
uint8_t keybuf[arg->cli->key_size];
@ -194,17 +202,9 @@ static int hi_inserts(DB_TXN* UU(txn), ARG arg, void* UU(operation_extra), void
}
}
cleanup:
if (dest_keys[0].data) {
toku_free(dest_keys[0].data);
}
if (dest_keys[1].data) {
toku_free(dest_keys[1].data);
}
if (dest_vals[0].data) {
toku_free(dest_vals[0].data);
}
if (dest_vals[1].data) {
toku_free(dest_vals[1].data);
for (int j = 0; j < 2; j++) {
toku_dbt_array_destroy(&dest_keys[j]);
toku_dbt_array_destroy(&dest_vals[j]);
}
increment_counter(stats_extra, PUTS, i);
gid_count++;

15
src/tests/test_update_broadcast_indexer.cc
View file

@ -110,11 +110,13 @@ static int update_fun(DB *UU(db),
static int generate_row_for_del(
DB *UU(dest_db),
DB *UU(src_db),
DBT *dest_key,
DBT_ARRAY *dest_key_arrays,
const DBT *UU(src_key),
const DBT *UU(src_val)
)
{
toku_dbt_array_resize(dest_key_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
dest_key->size=0;
return 0;
}
@ -122,12 +124,19 @@ static int generate_row_for_del(
static int generate_row_for_put(
DB *UU(dest_db),
DB *UU(src_db),
DBT *dest_key,
DBT *dest_val,
DBT_ARRAY *dest_key_arrays,
DBT_ARRAY *dest_val_arrays,
const DBT *UU(src_key),
const DBT *UU(src_val)
)
{
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
dest_key->flags = 0;
dest_val->flags = 0;
uint8_t src_val_data;
assert(src_val->size == 1);
src_val_data = *(uint8_t *)src_val->data;

16
src/tests/test_update_broadcast_loader.cc
View file

@ -110,11 +110,14 @@ static int update_fun(DB *UU(db),
static int generate_row_for_del(
DB *UU(dest_db),
DB *UU(src_db),
DBT *dest_key,
DBT_ARRAY *dest_key_arrays,
const DBT *UU(src_key),
const DBT *UU(src_val)
)
{
toku_dbt_array_resize(dest_key_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
dest_key->flags = 0;
dest_key->size=0;
return 0;
}
@ -122,12 +125,19 @@ static int generate_row_for_del(
static int generate_row_for_put(
DB *UU(dest_db),
DB *UU(src_db),
DBT *dest_key,
DBT *dest_val,
DBT_ARRAY *dest_key_arrays,
DBT_ARRAY *dest_val_arrays,
const DBT *UU(src_key),
const DBT *UU(src_val)
)
{
toku_dbt_array_resize(dest_key_arrays, 1);
toku_dbt_array_resize(dest_val_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = &dest_val_arrays->dbts[0];
dest_key->flags = 0;
dest_val->flags = 0;
uint8_t src_val_data;
assert(src_val->size == 1);
src_val_data = *(uint8_t *)src_val->data;

150
src/tests/threaded_stress_test_helpers.h
View file

@ -137,8 +137,8 @@ typedef int (*operation_t)(DB_TXN *txn, ARG arg, void *operation_extra, void *st
// TODO: Properly define these in db.h so we don't have to copy them here
typedef int (*test_update_callback_f)(DB *, const DBT *key, const DBT *old_val, const DBT *extra, void (*set_val)(const DBT *new_val, void *set_extra), void *set_extra);
typedef int (*test_generate_row_for_put_callback)(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data);
typedef int (*test_generate_row_for_del_callback)(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data);
typedef int (*test_generate_row_for_put_callback)(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_data);
typedef int (*test_generate_row_for_del_callback)(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, const DBT *src_key, const DBT *src_data);
enum stress_lock_type {
STRESS_LOCK_NONE = 0,
@ -697,20 +697,82 @@ static int scan_op_and_maybe_check_sum(
}
static int generate_row_for_put(
DB *UU(dest_db),
DB *UU(src_db),
DBT *dest_key,
DBT *dest_val,
const DBT *src_key,
DB *dest_db,
DB *src_db,
DBT_ARRAY *dest_keys,
DBT_ARRAY *dest_vals,
const DBT *src_key,
const DBT *src_val
)
{
dest_key->data = src_key->data;
dest_key->size = src_key->size;
dest_key->flags = 0;
dest_val->data = src_val->data;
dest_val->size = src_val->size;
dest_val->flags = 0;
)
{
invariant(!src_db || src_db != dest_db);
invariant(src_key->size >= sizeof(unsigned int));
// Consistent pseudo random source. Use checksum of key and val, and which db as seed
/*
struct x1764 l;
x1764_init(&l);
x1764_add(&l, src_key->data, src_key->size);
x1764_add(&l, src_val->data, src_val->size);
x1764_add(&l, &dest_db, sizeof(dest_db)); //make it depend on which db
unsigned int seed = x1764_finish(&l);
*/
unsigned int seed = *(unsigned int*)src_key->data;
struct random_data random_data;
ZERO_STRUCT(random_data);
char random_buf[8];
{
int r = myinitstate_r(seed, random_buf, 8, &random_data);
assert_zero(r);
}
uint8_t num_outputs = 0;
while (myrandom_r(&random_data) % 2) {
num_outputs++;
if (num_outputs > 8) {
break;
}
}
toku_dbt_array_resize(dest_keys, num_outputs);
toku_dbt_array_resize(dest_vals, num_outputs);
int sum = 0;
for (uint8_t i = 0; i < num_outputs; i++) {
DBT *dest_key = &dest_keys->dbts[i];
DBT *dest_val = &dest_vals->dbts[i];
invariant(dest_key->flags == DB_DBT_REALLOC);
invariant(dest_val->flags == DB_DBT_REALLOC);
if (dest_key->ulen < src_key->size) {
dest_key->data = toku_xrealloc(dest_key->data, src_key->size);
dest_key->ulen = src_key->size;
}
dest_key->size = src_key->size;
if (dest_val->ulen < src_val->size) {
dest_val->data = toku_xrealloc(dest_val->data, src_val->size);
dest_val->ulen = src_val->size;
}
dest_val->size = src_val->size;
memcpy(dest_key->data, src_key->data, src_key->size);
((uint8_t*)dest_key->data)[src_key->size-1] = i; //Have different keys for each entry.
memcpy(dest_val->data, src_val->data, src_val->size);
invariant(dest_val->size >= sizeof(int));
int number;
if (i == num_outputs - 1) {
// Make sum add to 0
number = -sum;
} else {
// Keep track of sum
number = myrandom_r(&random_data);
}
sum += number;
*(int *) dest_val->data = number;
}
invariant(sum == 0);
return 0;
}
@ -920,21 +982,34 @@ cleanup:
return r;
}
static int UU() loader_op(DB_TXN* txn, ARG UU(arg), void* UU(operation_extra), void *UU(stats_extra)) {
struct loader_op_extra {
struct scan_op_extra soe;
int num_dbs;
};
static int UU() loader_op(DB_TXN* txn, ARG arg, void* operation_extra, void *UU(stats_extra)) {
struct loader_op_extra* CAST_FROM_VOIDP(extra, operation_extra);
invariant(extra->num_dbs >= 1);
DB_ENV* env = arg->env;
int r;
for (int num = 0; num < 2; num++) {
DB *db_load;
uint32_t db_flags = 0;
uint32_t dbt_flags = 0;
r = db_create(&db_load, env, 0);
assert(r == 0);
// TODO: Need to call before_db_open_hook() and after_db_open_hook()
r = db_load->open(db_load, txn, "loader-db", nullptr, DB_BTREE, DB_CREATE, 0666);
assert(r == 0);
DB *dbs_load[extra->num_dbs];
uint32_t db_flags[extra->num_dbs];
uint32_t dbt_flags[extra->num_dbs];
for (int i = 0; i < extra->num_dbs; ++i) {
db_flags[i] = 0;
dbt_flags[i] = 0;
r = db_create(&dbs_load[i], env, 0);
assert(r == 0);
char fname[100];
sprintf(fname, "loader-db-%d", i);
// TODO: Need to call before_db_open_hook() and after_db_open_hook()
r = dbs_load[i]->open(dbs_load[i], txn, fname, nullptr, DB_BTREE, DB_CREATE, 0666);
assert(r == 0);
}
DB_LOADER *loader;
uint32_t loader_flags = (num == 0) ? 0 : LOADER_COMPRESS_INTERMEDIATES;
r = env->create_loader(env, txn, &loader, db_load, 1, &db_load, &db_flags, &dbt_flags, loader_flags);
r = env->create_loader(env, txn, &loader, dbs_load[0], extra->num_dbs, dbs_load, db_flags, dbt_flags, loader_flags);
CKERR(r);
DBT key, val;
@ -943,15 +1018,32 @@ static int UU() loader_op(DB_TXN* txn, ARG UU(arg), void* UU(operation_extra), v
dbt_init(&key, keybuf, sizeof keybuf);
dbt_init(&val, valbuf, sizeof valbuf);
for (int i = 0; i < 1000; i++) {
int sum = 0;
const int num_elements = 1000;
for (int i = 0; i < num_elements; i++) {
fill_key_buf(i, keybuf, arg->cli);
fill_val_buf_random(arg->random_data, valbuf, arg->cli);
assert(val.size >= sizeof(int));
if (i == num_elements - 1) {
// Make sum add to 0
*(int *) val.data = -sum;
} else {
// Keep track of sum
sum += *(int *) val.data;
}
r = loader->put(loader, &key, &val); CKERR(r);
}
r = loader->close(loader); CKERR(r);
r = db_load->close(db_load, 0); CKERR(r);
r = env->dbremove(env, txn, "loader-db", nullptr, 0); CKERR(r);
for (int i = 0; i < extra->num_dbs; ++i) {
r = scan_op_and_maybe_check_sum(dbs_load[i], txn, &extra->soe, true); CKERR(r);
r = dbs_load[i]->close(dbs_load[i], 0); CKERR(r);
char fname[100];
sprintf(fname, "loader-db-%d", i);
r = env->dbremove(env, txn, fname, nullptr, 0); CKERR(r);
}
}
return 0;
}
@ -1017,7 +1109,7 @@ static int UU() verify_op(DB_TXN* UU(txn), ARG UU(arg), void* UU(operation_extra
return r;
}
static int UU() scan_op(DB_TXN *txn, ARG UU(arg), void* operation_extra, void *UU(stats_extra)) {
static int UU() scan_op(DB_TXN *txn, ARG arg, void* operation_extra, void *UU(stats_extra)) {
struct scan_op_extra* CAST_FROM_VOIDP(extra, operation_extra);
for (int i = 0; run_test && i < arg->cli->num_DBs; i++) {
int r = scan_op_and_maybe_check_sum(arg->dbp[i], txn, extra, true);

39
src/tests/update-multiple-data-diagonal.cc
View file

@ -119,16 +119,23 @@ get_new_data(int *v, int i, int ndbs) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = NULL;
if (dest_val_arrays) {
toku_dbt_array_resize(dest_val_arrays, 1);
dest_val = &dest_val_arrays->dbts[0];
}
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum < src_val->size / sizeof (int));
int *pri_key = (int *) src_key->data;
int *pri_data = (int *) src_data->data;
int *pri_data = (int *) src_val->data;
switch (dest_key->flags) {
case 0:
@ -144,22 +151,22 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
assert(0);
}
if (dest_data) {
switch (dest_data->flags) {
if (dest_val) {
switch (dest_val->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
dest_val->size = src_val->size;
dest_val->data = src_val->data;
} else
dest_data->size = 0;
dest_val->size = 0;
break;
case DB_DBT_REALLOC:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = toku_realloc(dest_data->data, dest_data->size);
memcpy(dest_data->data, src_data->data, dest_data->size);
dest_val->size = src_val->size;
dest_val->data = toku_realloc(dest_val->data, dest_val->size);
memcpy(dest_val->data, src_val->data, dest_val->size);
} else
dest_data->size = 0;
dest_val->size = 0;
break;
default:
assert(0);
@ -170,8 +177,8 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key_arrays, NULL, src_key, src_data);
}
#if 0
@ -270,7 +277,7 @@ update_diagonal(DB_ENV *env, DB *db[], int ndbs, int nrows) {
DBT vals[ndbts]; memset(vals, 0, sizeof vals);
uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array));
r = env->update_multiple(env, ndbs > 0 ? db[0] : NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
r = env_update_multiple_test_no_array(env, ndbs > 0 ? db[0] : NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
assert_zero(r);
}
r = txn->commit(txn, 0); assert_zero(r);

40
src/tests/update-multiple-key0.cc
View file

@ -104,16 +104,24 @@ get_data(int *v, int i, int ndbs) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = NULL;
if (dest_val_arrays) {
toku_dbt_array_resize(dest_val_arrays, 1);
dest_val = &dest_val_arrays->dbts[0];
}
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum < src_val->size / sizeof (int));
int *pri_key = (int *) src_key->data;
int *pri_data = (int *) src_data->data;
int *pri_data = (int *) src_val->data;
switch (dest_key->flags) {
case 0:
@ -129,22 +137,22 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
assert(0);
}
if (dest_data) {
switch (dest_data->flags) {
if (dest_val) {
switch (dest_val->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
dest_val->size = src_val->size;
dest_val->data = src_val->data;
} else
dest_data->size = 0;
dest_val->size = 0;
break;
case DB_DBT_REALLOC:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = toku_realloc(dest_data->data, dest_data->size);
memcpy(dest_data->data, src_data->data, dest_data->size);
dest_val->size = src_val->size;
dest_val->data = toku_realloc(dest_val->data, dest_val->size);
memcpy(dest_val->data, src_val->data, dest_val->size);
} else
dest_data->size = 0;
dest_val->size = 0;
break;
default:
assert(0);
@ -155,8 +163,8 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key_arrays, NULL, src_key, src_data);
}
static void
@ -249,7 +257,7 @@ update_key0(DB_ENV *env, DB *db[], int ndbs, int nrows) {
DBT vals[ndbts]; memset(vals, 0, sizeof vals);
uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array));
r = env->update_multiple(env, ndbs > 0 ? db[0] : NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
r = env_update_multiple_test_no_array(env, ndbs > 0 ? db[0] : NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
assert_zero(r);
verify_locked(env, db[0], k);

40
src/tests/update-multiple-nochange.cc
View file

@ -104,15 +104,23 @@ get_data(int *v, int i, int ndbs) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = NULL;
if (dest_val_arrays) {
toku_dbt_array_resize(dest_val_arrays, 1);
dest_val = &dest_val_arrays->dbts[0];
}
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum < src_val->size / sizeof (int));
int *pri_data = (int *) src_data->data;
int *pri_data = (int *) src_val->data;
switch (dest_key->flags) {
case 0:
@ -128,22 +136,22 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
assert(0);
}
if (dest_data) {
switch (dest_data->flags) {
if (dest_val) {
switch (dest_val->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
dest_val->size = src_val->size;
dest_val->data = src_val->data;
} else
dest_data->size = 0;
dest_val->size = 0;
break;
case DB_DBT_REALLOC:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = toku_realloc(dest_data->data, dest_data->size);
memcpy(dest_data->data, src_data->data, dest_data->size);
dest_val->size = src_val->size;
dest_val->data = toku_realloc(dest_val->data, dest_val->size);
memcpy(dest_val->data, src_val->data, dest_val->size);
} else
dest_data->size = 0;
dest_val->size = 0;
break;
default:
assert(0);
@ -154,8 +162,8 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key_arrays, NULL, src_key, src_data);
}
#if 0
@ -244,7 +252,7 @@ verify(DB_ENV *env, DB *db[], int ndbs, int nrows) {
DBT vals[ndbts]; memset(vals, 0, sizeof vals);
uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array));
r = env->update_multiple(env, ndbs > 0 ? db[0] : NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
r = env_update_multiple_test_no_array(env, ndbs > 0 ? db[0] : NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
assert_zero(r);
}
r = txn->commit(txn, 0); assert_zero(r);

508
src/tests/update-multiple-with-indexer-array.cc Normal file
View file

@ -0,0 +1,508 @@
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include "test.h"
// verify that update_multiple where we change the data in row[i] col[j] from x to x+1
static int
get_num_new_keys(int i, int dbnum) {
if (dbnum == 0) return 1;
if (i & (1<<4)) {
dbnum++; // Shift every once in a while.
}
return (i + dbnum) % 3; // 0, 1, or 2
}
static int
get_num_keys(int i, int dbnum) {
if (dbnum == 0) return 1;
return (i + dbnum) % 3; // 0, 1, or 2
}
static int
get_total_secondary_rows(int num_primary) {
assert(num_primary % 3 == 0);
return num_primary / 3 * (0 + 1 + 2);
}
static int
get_total_num_keys(int i, int num_dbs) {
int sum = 0;
for (int db = 1; db < num_dbs; ++db) {
sum += get_num_keys(i, db);
}
return sum;
}
static int
get_total_num_new_keys(int i, int num_dbs) {
int sum = 0;
for (int db = 1; db < num_dbs; ++db) {
sum += get_num_new_keys(i, db);
}
return sum;
}
static int
get_key(int i, int dbnum, int which) {
assert(i < INT16_MAX / 2);
assert(which >= 0);
assert(which < get_num_keys(i, dbnum));
assert(which < 4);
assert(dbnum < 16);
if (dbnum == 0) {
assert(which == 0);
return htonl((2*i) << 16);
} else {
return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1));
}
}
static int
get_new_key(int i, int dbnum, int which) {
assert(which >= 0);
assert(which < get_num_new_keys(i, dbnum));
assert(which < 4);
assert(dbnum < 16);
if (dbnum == 0) {
assert(which == 0);
return htonl((2*i+1) << 16);
} else if ((i+dbnum+which) & (1<<5)) {
return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1)); // no change from original
} else {
return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1) + 1);
}
}
static void
get_data(int *v, int i, int ndbs) {
int index = 0;
for (int dbnum = 1; dbnum < ndbs; dbnum++) {
for (int which = 0; which < get_num_keys(i, dbnum); ++which) {
v[index++] = get_key(i, dbnum, which);
}
}
}
static void
get_new_data(int *v, int i, int ndbs) {
int index = 0;
for (int dbnum = 1; dbnum < ndbs; dbnum++) {
for (int which = 0; which < get_num_new_keys(i, dbnum); ++which) {
v[index++] = get_new_key(i, dbnum, which);
if (which > 0) {
assert(index >= 2);
assert(memcmp(&v[index-2], &v[index-1], sizeof(v[0])) < 0);
}
}
}
}
static int
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
(void)src_val;
assert(src_db != dest_db);
assert(src_db);
int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum > 0);
int pri_key = *(int *) src_key->data;
int* pri_val = (int*) src_val->data;
bool is_new = (ntohl(pri_key) >> 16) % 2 == 1;
int i = (ntohl(pri_key) >> 16) / 2;
int num_keys = is_new ? get_num_new_keys(i, dbnum) : get_num_keys(i, dbnum);
toku_dbt_array_resize(dest_key_arrays, num_keys);
if (dest_val_arrays) {
toku_dbt_array_resize(dest_val_arrays, num_keys);
}
int index = 0;
for (int idb = 1; idb < dbnum; idb++) {
index += is_new ? get_num_new_keys(i, idb) : get_num_keys(i, idb);
}
assert(src_val->size % sizeof(int) == 0);
assert((int)src_val->size / 4 >= index + num_keys);
for (int which = 0; which < num_keys; which++) {
DBT *dest_key = &dest_key_arrays->dbts[which];
DBT *dest_val = NULL;
assert(dest_key->flags == DB_DBT_REALLOC);
if (dest_key->ulen < sizeof(int)) {
dest_key->data = toku_xrealloc(dest_key->data, sizeof(int));
dest_key->ulen = sizeof(int);
}
dest_key->size = sizeof(int);
if (dest_val_arrays) {
dest_val = &dest_val_arrays->dbts[which];
assert(dest_val->flags == DB_DBT_REALLOC);
dest_val->size = 0;
}
int new_key = is_new ? get_new_key(i, dbnum, which) : get_key(i, dbnum, which);
assert(new_key == pri_val[index + which]);
*(int*)dest_key->data = new_key;
}
return 0;
}
static int
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key_arrays, NULL, src_key, src_data);
}
static void
do_updates(DB_ENV *env, DB *db[], int ndbs, int nrows) {
assert(ndbs > 0);
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
int narrays = 2 * ndbs;
DBT_ARRAY keys[narrays];
DBT_ARRAY vals[narrays];
for (int i = 0; i < narrays; i++) {
toku_dbt_array_init(&keys[i], 1);
toku_dbt_array_init(&vals[i], 1);
}
for (int i = 0; i < nrows; i++) {
// update the data i % ndbs col from x to x+1
int old_k = get_key(i, 0, 0);
DBT old_key; dbt_init(&old_key, &old_k, sizeof old_k);
int new_k = get_new_key(i, 0, 0);
DBT new_key; dbt_init(&new_key, &new_k, sizeof new_k);
int num_old_keys = get_total_num_keys(i, ndbs);
int v[num_old_keys]; get_data(v, i, ndbs);
DBT old_data; dbt_init(&old_data, &v[0], sizeof v);
int num_new_keys = get_total_num_new_keys(i, ndbs);
int newv[num_new_keys]; get_new_data(newv, i, ndbs);
DBT new_data; dbt_init(&new_data, &newv[0], sizeof newv);
uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array));
r = env->update_multiple(env, db[0], txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, narrays, keys, narrays, vals);
assert_zero(r);
}
for (int i = 0; i < narrays; i++) {
toku_dbt_array_destroy(&keys[i]);
toku_dbt_array_destroy(&vals[i]);
}
r = txn->commit(txn, 0); assert_zero(r);
}
static void
populate_primary(DB_ENV *env, DB *db, int ndbs, int nrows) {
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
// populate
for (int i = 0; i < nrows; i++) {
int k = get_key(i, 0, 0);
int secondary_keys = get_total_num_keys(i, ndbs);
int v[secondary_keys]; get_data(v, i, ndbs);
DBT key; dbt_init(&key, &k, sizeof k);
DBT val; dbt_init(&val, &v[0], sizeof v);
r = db->put(db, txn, &key, &val, 0); assert_zero(r);
}
r = txn->commit(txn, 0); assert_zero(r);
}
static void
populate_secondary(DB_ENV *env, DB *db, int dbnum, int nrows) {
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
// populate
for (int i = 0; i < nrows; i++) {
for (int which = 0; which < get_num_keys(i, dbnum); which++) {
int k = get_key(i, dbnum, which);
DBT key; dbt_init(&key, &k, sizeof k);
DBT val; dbt_init(&val, NULL, 0);
r = db->put(db, txn, &key, &val, 0); assert_zero(r);
}
}
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify_pri_seq(DB_ENV *env, DB *db, int ndbs, int nrows) {
const int dbnum = 0;
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
int i;
for (i = 0; ; i++) {
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0)
break;
int k;
int expectk = get_new_key(i, dbnum, 0);
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
assert(k == expectk);
int num_keys = get_total_num_new_keys(i, ndbs);
assert(val.size == num_keys*sizeof(int));
int v[num_keys]; get_new_data(v, i, ndbs);
assert(memcmp(val.data, v, val.size) == 0);
}
assert(i == nrows); // if (i != nrows) printf("%s:%d %d %d\n", __FUNCTION__, __LINE__, i, nrows); // assert(i == nrows);
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
verify_sec_seq(DB_ENV *env, DB *db, int dbnum, int nrows) {
assert(dbnum > 0);
int r;
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DBC *cursor = NULL;
r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
int i;
int rows_found = 0;
for (i = 0; ; i++) {
int num_keys = get_num_new_keys(i, dbnum);
for (int which = 0; which < num_keys; ++which) {
DBT key; memset(&key, 0, sizeof key);
DBT val; memset(&val, 0, sizeof val);
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if (r != 0) {
CKERR2(r, DB_NOTFOUND);
goto done;
}
rows_found++;
int k;
int expectk = get_new_key(i, dbnum, which);
assert(key.size == sizeof k);
memcpy(&k, key.data, key.size);
int got_i = (ntohl(k) >> 16) / 2;
if (got_i < i) {
// Will fail. Too many old i's
assert(k == expectk);
} else if (got_i > i) {
// Will fail. Too few in previous i.
assert(k == expectk);
}
if (k != expectk && which < get_num_keys(i, dbnum) && k == get_key(i, dbnum, which)) {
// Will fail, never got updated.
assert(k == expectk);
}
assert(k == expectk);
assert(val.size == 0);
}
}
done:
assert(rows_found == get_total_secondary_rows(nrows));
r = cursor->c_close(cursor); assert_zero(r);
r = txn->commit(txn, 0); assert_zero(r);
}
static void
run_test(int ndbs, int nrows) {
int r;
DB_ENV *env = NULL;
r = db_env_create(&env, 0); assert_zero(r);
r = env->set_generate_row_callback_for_put(env, put_callback); assert_zero(r);
r = env->set_generate_row_callback_for_del(env, del_callback); assert_zero(r);
r = env->open(env, TOKU_TEST_FILENAME, DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB *db[ndbs];
for (int dbnum = 0; dbnum < ndbs; dbnum++) {
r = db_create(&db[dbnum], env, 0); assert_zero(r);
DBT dbt_dbnum; dbt_init(&dbt_dbnum, &dbnum, sizeof dbnum);
char dbname[32]; sprintf(dbname, "%d.tdb", dbnum);
r = db[dbnum]->open(db[dbnum], NULL, dbname, NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
IN_TXN_COMMIT(env, NULL, txn_desc, 0, {
{ int chk_r = db[dbnum]->change_descriptor(db[dbnum], txn_desc, &dbt_dbnum, 0); CKERR(chk_r); }
});
}
populate_primary(env, db[0], ndbs, nrows);
for (int dbnum = 1; dbnum < ndbs-1; dbnum++) {
populate_secondary(env, db[dbnum], dbnum, nrows);
}
DB_TXN *indexer_txn = NULL;
r = env->txn_begin(env, NULL, &indexer_txn, 0); assert_zero(r);
DB_INDEXER *indexer = NULL;
uint32_t db_flags = 0;
assert(ndbs > 2);
r = env->create_indexer(env, indexer_txn, &indexer, db[0], 1, &db[ndbs-1], &db_flags, 0); assert_zero(r);
do_updates(env, db, ndbs, nrows);
r = indexer->build(indexer); assert_zero(r);
r = indexer->close(indexer); assert_zero(r);
r = indexer_txn->commit(indexer_txn, 0); assert_zero(r);
verify_pri_seq(env, db[0], ndbs, nrows);
for (int dbnum = 1; dbnum < ndbs; dbnum++)
verify_sec_seq(env, db[dbnum], dbnum, nrows);
for (int dbnum = 0; dbnum < ndbs; dbnum++)
r = db[dbnum]->close(db[dbnum], 0); assert_zero(r);
r = env->close(env, 0); assert_zero(r);
}
int
test_main(int argc, char * const argv[]) {
int r;
int ndbs = 10;
int nrows = 3*(1<<5)*4;
// parse_args(argc, argv);
for (int i = 1; i < argc; i++) {
char * const arg = argv[i];
if (strcmp(arg, "-v") == 0) {
verbose++;
continue;
}
if (strcmp(arg, "-q") == 0) {
verbose = 0;
continue;
}
if (strcmp(arg, "--ndbs") == 0 && i+1 < argc) {
ndbs = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--nrows") == 0 && i+1 < argc) {
nrows = atoi(argv[++i]);
continue;
}
}
while (nrows % (3*(1<<5)) != 0) {
nrows++;
}
//Need at least one to update, and one to index
while (ndbs < 3) {
ndbs++;
}
toku_os_recursive_delete(TOKU_TEST_FILENAME);
r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
run_test(ndbs, nrows);
return 0;
}

40
src/tests/update-multiple-with-indexer.cc
View file

@ -119,16 +119,24 @@ get_new_data(int *v, int i, int ndbs) {
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data) {
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_data; (void) src_key; (void) src_data;
put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
toku_dbt_array_resize(dest_key_arrays, 1);
DBT *dest_key = &dest_key_arrays->dbts[0];
DBT *dest_val = NULL;
if (dest_val_arrays) {
toku_dbt_array_resize(dest_val_arrays, 1);
dest_val = &dest_val_arrays->dbts[0];
}
(void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
unsigned int dbnum;
assert(dest_db->descriptor->dbt.size == sizeof dbnum);
memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
assert(dbnum < src_data->size / sizeof (int));
assert(dbnum < src_val->size / sizeof (int));
int *pri_key = (int *) src_key->data;
int *pri_data = (int *) src_data->data;
int *pri_data = (int *) src_val->data;
switch (dest_key->flags) {
case 0:
@ -144,22 +152,22 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
assert(0);
}
if (dest_data) {
switch (dest_data->flags) {
if (dest_val) {
switch (dest_val->flags) {
case 0:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = src_data->data;
dest_val->size = src_val->size;
dest_val->data = src_val->data;
} else
dest_data->size = 0;
dest_val->size = 0;
break;
case DB_DBT_REALLOC:
if (dbnum == 0) {
dest_data->size = src_data->size;
dest_data->data = toku_realloc(dest_data->data, dest_data->size);
memcpy(dest_data->data, src_data->data, dest_data->size);
dest_val->size = src_val->size;
dest_val->data = toku_realloc(dest_val->data, dest_val->size);
memcpy(dest_val->data, src_val->data, dest_val->size);
} else
dest_data->size = 0;
dest_val->size = 0;
break;
default:
assert(0);
@ -170,8 +178,8 @@ put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *
}
static int
del_callback(DB *dest_db, DB *src_db, DBT *dest_key, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key, NULL, src_key, src_data);
del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT *src_data) {
return put_callback(dest_db, src_db, dest_key_arrays, NULL, src_key, src_data);
}
#if 0
@ -270,7 +278,7 @@ update_diagonal(DB_ENV *env, DB *db[], int ndbs, int nrows) {
DBT vals[ndbts]; memset(vals, 0, sizeof vals);
uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array));
r = env->update_multiple(env, ndbs > 0 ? db[0] : NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
r = env_update_multiple_test_no_array(env, ndbs > 0 ? db[0] : NULL, txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, ndbts, keys, ndbts, vals);
assert_zero(r);
}
r = txn->commit(txn, 0); assert_zero(r);

487
src/ydb_write.cc
View file

@ -401,15 +401,18 @@ log_del_single(DB_TXN *txn, FT_HANDLE brt, const DBT *key) {
}
static uint32_t
sum_size(uint32_t num_keys, DBT keys[], uint32_t overhead) {
sum_size(uint32_t num_arrays, DBT_ARRAY keys[], uint32_t overhead) {
uint32_t sum = 0;
for (uint32_t i = 0; i < num_keys; i++)
sum += keys[i].size + overhead;
for (uint32_t i = 0; i < num_arrays; i++) {
for (uint32_t j = 0; j < keys[i].size; j++) {
sum += keys[i].dbts[j].size + overhead;
}
}
return sum;
}
static void
log_del_multiple(DB_TXN *txn, DB *src_db, const DBT *key, const DBT *val, uint32_t num_dbs, FT_HANDLE brts[], DBT keys[]) {
log_del_multiple(DB_TXN *txn, DB *src_db, const DBT *key, const DBT *val, uint32_t num_dbs, FT_HANDLE brts[], DBT_ARRAY keys[]) {
if (num_dbs > 0) {
TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn;
FT_HANDLE src_ft = src_db ? src_db->i->ft_handle : NULL;
@ -417,7 +420,9 @@ log_del_multiple(DB_TXN *txn, DB *src_db, const DBT *key, const DBT *val, uint32
uint32_t del_single_sizes = sum_size(num_dbs, keys, toku_log_enq_delete_any_overhead);
if (del_single_sizes < del_multiple_size) {
for (uint32_t i = 0; i < num_dbs; i++) {
log_del_single(txn, brts[i], &keys[i]);
for (uint32_t j = 0; j < keys[i].size; j++) {
log_del_single(txn, brts[i], &keys[i].dbts[j]);
}
}
} else {
toku_ft_log_del_multiple(ttxn, src_ft, brts, num_dbs, key, val);
@ -435,12 +440,14 @@ lookup_src_db(uint32_t num_dbs, DB *db_array[], DB *src_db) {
}
static int
do_del_multiple(DB_TXN *txn, uint32_t num_dbs, DB *db_array[], DBT keys[], DB *src_db, const DBT *src_key) {
do_del_multiple(DB_TXN *txn, uint32_t num_dbs, DB *db_array[], DBT_ARRAY keys[], DB *src_db, const DBT *src_key) {
int r = 0;
TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn;
for (uint32_t which_db = 0; r == 0 && which_db < num_dbs; which_db++) {
DB *db = db_array[which_db];
paranoid_invariant(keys[which_db].size <= keys[which_db].capacity);
// if db is being indexed by an indexer, then insert a delete message into the db if the src key is to the left or equal to the
// indexers cursor. we have to get the src_db from the indexer and find it in the db_array.
int do_delete = true;
@ -454,12 +461,16 @@ do_del_multiple(DB_TXN *txn, uint32_t num_dbs, DB *db_array[], DBT keys[], DB *s
else {
uint32_t which_src_db = lookup_src_db(num_dbs, db_array, indexer_src_db);
invariant(which_src_db < num_dbs);
indexer_src_key = &keys[which_src_db];
// The indexer src db must have exactly one item or we don't know how to continue.
invariant(keys[which_src_db].size == 1);
indexer_src_key = &keys[which_src_db].dbts[0];
}
do_delete = !toku_indexer_is_key_right_of_le_cursor(indexer, indexer_src_key);
}
if (r == 0 && do_delete) {
toku_ft_maybe_delete(db->i->ft_handle, &keys[which_db], ttxn, false, ZERO_LSN, false);
if (do_delete) {
for (uint32_t i = 0; i < keys[which_db].size; i++) {
toku_ft_maybe_delete(db->i->ft_handle, &keys[which_db].dbts[i], ttxn, false, ZERO_LSN, false);
}
}
}
return r;
@ -505,11 +516,11 @@ env_del_multiple(
const DBT *src_val,
uint32_t num_dbs,
DB **db_array,
DBT *keys,
DBT_ARRAY *keys,
uint32_t *flags_array)
{
int r;
DBT del_keys[num_dbs];
DBT_ARRAY del_keys[num_dbs];
DB_INDEXER* indexer = NULL;
HANDLE_PANICKED_ENV(env);
@ -533,20 +544,22 @@ env_del_multiple(
goto cleanup;
}
for (uint32_t which_db = 0; which_db < num_dbs; which_db++) {
DB *db = db_array[which_db];
lock_flags[which_db] = get_prelocked_flags(flags_array[which_db]);
remaining_flags[which_db] = flags_array[which_db] & ~lock_flags[which_db];
if (db == src_db) {
del_keys[which_db] = *src_key;
del_keys[which_db].size = 1;
del_keys[which_db].capacity = 1;
del_keys[which_db].dbts = const_cast<DBT*>(src_key);
}
else {
//Generate the key
r = env->i->generate_row_for_del(db, src_db, &keys[which_db], src_key, src_val);
if (r != 0) goto cleanup;
del_keys[which_db] = keys[which_db];
paranoid_invariant(del_keys[which_db].size <= del_keys[which_db].capacity);
}
if (remaining_flags[which_db] & ~DB_DELETE_ANY) {
@ -554,17 +567,18 @@ env_del_multiple(
goto cleanup;
}
bool error_if_missing = (bool)(!(remaining_flags[which_db]&DB_DELETE_ANY));
if (error_if_missing) {
//Check if the key exists in the db.
r = db_getf_set(db, txn, lock_flags[which_db]|DB_SERIALIZABLE|DB_RMW, &del_keys[which_db], ydb_getf_do_nothing, NULL);
if (r != 0) goto cleanup;
}
//Do locking if necessary.
if (db->i->lt && !(lock_flags[which_db] & DB_PRELOCKED_WRITE)) {
//Needs locking
r = toku_db_get_point_write_lock(db, txn, &del_keys[which_db]);
if (r != 0) goto cleanup;
for (uint32_t which_key = 0; which_key < del_keys[which_db].size; which_key++) {
DBT *del_key = &del_keys[which_db].dbts[which_key];
if (error_if_missing) {
//Check if the key exists in the db.
//Grabs a write lock
r = db_getf_set(db, txn, lock_flags[which_db]|DB_SERIALIZABLE|DB_RMW, del_key, ydb_getf_do_nothing, NULL);
if (r != 0) goto cleanup;
} else if (db->i->lt && !(lock_flags[which_db] & DB_PRELOCKED_WRITE)) { //Do locking if necessary.
//Needs locking
r = toku_db_get_point_write_lock(db, txn, del_key);
if (r != 0) goto cleanup;
}
}
brts[which_db] = db->i->ft_handle;
}
@ -573,15 +587,8 @@ env_del_multiple(
toku_indexer_lock(indexer);
}
toku_multi_operation_client_lock();
if (num_dbs == 1) {
log_del_single(txn, brts[0], &del_keys[0]);
}
else {
log_del_multiple(txn, src_db, src_key, src_val, num_dbs, brts, del_keys);
}
if (r == 0) {
r = do_del_multiple(txn, num_dbs, db_array, del_keys, src_db, src_key);
}
log_del_multiple(txn, src_db, src_key, src_val, num_dbs, brts, del_keys);
r = do_del_multiple(txn, num_dbs, db_array, del_keys, src_db, src_key);
toku_multi_operation_client_unlock();
if (indexer) {
toku_indexer_unlock(indexer);
@ -595,12 +602,6 @@ cleanup:
return r;
}
static void
log_put_single(DB_TXN *txn, FT_HANDLE brt, const DBT *key, const DBT *val) {
TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn;
toku_ft_log_put(ttxn, brt, key, val);
}
static void
log_put_multiple(DB_TXN *txn, DB *src_db, const DBT *src_key, const DBT *src_val, uint32_t num_dbs, FT_HANDLE brts[]) {
if (num_dbs > 0) {
@ -611,34 +612,45 @@ log_put_multiple(DB_TXN *txn, DB *src_db, const DBT *src_key, const DBT *src_val
}
static int
do_put_multiple(DB_TXN *txn, uint32_t num_dbs, DB *db_array[], DBT keys[], DBT vals[], DB *src_db, const DBT *src_key) {
int r = 0;
do_put_multiple(DB_TXN *txn, uint32_t num_dbs, DB *db_array[], DBT_ARRAY keys[], DBT_ARRAY vals[], DB *src_db, const DBT *src_key) {
TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn;
for (uint32_t which_db = 0; r == 0 && which_db < num_dbs; which_db++) {
for (uint32_t which_db = 0; which_db < num_dbs; which_db++) {
DB *db = db_array[which_db];
// if db is being indexed by an indexer, then put into that db if the src key is to the left or equal to the
// indexers cursor. we have to get the src_db from the indexer and find it in the db_array.
int do_put = true;
DB_INDEXER *indexer = toku_db_get_indexer(db);
if (indexer) { // if this db is the index under construction
DB *indexer_src_db = toku_indexer_get_src_db(indexer);
invariant(indexer_src_db != NULL);
const DBT *indexer_src_key;
if (src_db == indexer_src_db)
indexer_src_key = src_key;
else {
uint32_t which_src_db = lookup_src_db(num_dbs, db_array, indexer_src_db);
invariant(which_src_db < num_dbs);
indexer_src_key = &keys[which_src_db];
invariant(keys[which_db].size == vals[which_db].size);
paranoid_invariant(keys[which_db].size <= keys[which_db].capacity);
paranoid_invariant(vals[which_db].size <= vals[which_db].capacity);
if (keys[which_db].size > 0) {
bool do_put = true;
DB_INDEXER *indexer = toku_db_get_indexer(db);
if (indexer) { // if this db is the index under construction
DB *indexer_src_db = toku_indexer_get_src_db(indexer);
invariant(indexer_src_db != NULL);
const DBT *indexer_src_key;
if (src_db == indexer_src_db)
indexer_src_key = src_key;
else {
uint32_t which_src_db = lookup_src_db(num_dbs, db_array, indexer_src_db);
invariant(which_src_db < num_dbs);
// The indexer src db must have exactly one item or we don't know how to continue.
invariant(keys[which_src_db].size == 1);
indexer_src_key = &keys[which_src_db].dbts[0];
}
do_put = !toku_indexer_is_key_right_of_le_cursor(indexer, indexer_src_key);
}
if (do_put) {
for (uint32_t i = 0; i < keys[which_db].size; i++) {
// if db is being indexed by an indexer, then put into that db if the src key is to the left or equal to the
// indexers cursor. we have to get the src_db from the indexer and find it in the db_array.
toku_ft_maybe_insert(db->i->ft_handle,
&keys[which_db].dbts[i], &vals[which_db].dbts[i],
ttxn, false, ZERO_LSN, false, FT_INSERT);
}
}
do_put = !toku_indexer_is_key_right_of_le_cursor(indexer, indexer_src_key);
}
if (r == 0 && do_put) {
toku_ft_maybe_insert(db->i->ft_handle, &keys[which_db], &vals[which_db], ttxn, false, ZERO_LSN, false, FT_INSERT);
}
}
return r;
return 0;
}
static int
@ -650,13 +662,13 @@ env_put_multiple_internal(
const DBT *src_val,
uint32_t num_dbs,
DB **db_array,
DBT *keys,
DBT *vals,
DBT_ARRAY *keys,
DBT_ARRAY *vals,
uint32_t *flags_array)
{
int r;
DBT put_keys[num_dbs];
DBT put_vals[num_dbs];
DBT_ARRAY put_keys[num_dbs];
DBT_ARRAY put_vals[num_dbs];
DB_INDEXER* indexer = NULL;
HANDLE_PANICKED_ENV(env);
@ -689,37 +701,49 @@ env_put_multiple_internal(
//Generate the row
if (db == src_db) {
put_keys[which_db] = *src_key;
put_vals[which_db] = *src_val;
put_keys[which_db].size = put_keys[which_db].capacity = 1;
put_keys[which_db].dbts = const_cast<DBT*>(src_key);
put_vals[which_db].size = put_vals[which_db].capacity = 1;
put_vals[which_db].dbts = const_cast<DBT*>(src_val);
}
else {
r = env->i->generate_row_for_put(db, src_db, &keys[which_db], &vals[which_db], src_key, src_val);
if (r != 0) goto cleanup;
paranoid_invariant(keys[which_db].size <= keys[which_db].capacity);
paranoid_invariant(vals[which_db].size <= vals[which_db].capacity);
paranoid_invariant(keys[which_db].size == vals[which_db].size);
put_keys[which_db] = keys[which_db];
put_vals[which_db] = vals[which_db];
put_vals[which_db] = vals[which_db];
}
for (uint32_t i = 0; i < put_keys[which_db].size; i++) {
DBT &put_key = put_keys[which_db].dbts[i];
DBT &put_val = put_vals[which_db].dbts[i];
// check size constraints
r = db_put_check_size_constraints(db, &put_keys[which_db], &put_vals[which_db]);
if (r != 0) goto cleanup;
//Check overwrite constraints
r = db_put_check_overwrite_constraint(db, txn,
&put_keys[which_db],
lock_flags[which_db], remaining_flags[which_db]);
if (r != 0) goto cleanup;
if (remaining_flags[which_db] == DB_NOOVERWRITE_NO_ERROR) {
//put_multiple does not support delaying the no error, since we would
//have to log the flag in the put_multiple.
r = EINVAL; goto cleanup;
}
//Do locking if necessary. Do not grab the lock again if this DB had a unique
//check performed because the lock was already grabbed by its cursor callback.
if (db->i->lt && !(lock_flags[which_db] & DB_PRELOCKED_WRITE) && !(remaining_flags[which_db] & DB_NOOVERWRITE)) {
//Needs locking
r = toku_db_get_point_write_lock(db, txn, &put_keys[which_db]);
// check size constraints
r = db_put_check_size_constraints(db, &put_key, &put_val);
if (r != 0) goto cleanup;
//Check overwrite constraints
r = db_put_check_overwrite_constraint(db, txn,
&put_key,
lock_flags[which_db], remaining_flags[which_db]);
if (r != 0) goto cleanup;
if (remaining_flags[which_db] == DB_NOOVERWRITE_NO_ERROR) {
//put_multiple does not support delaying the no error, since we would
//have to log the flag in the put_multiple.
r = EINVAL; goto cleanup;
}
//Do locking if necessary. Do not grab the lock again if this DB had a unique
//check performed because the lock was already grabbed by its cursor callback.
if (db->i->lt && !(lock_flags[which_db] & DB_PRELOCKED_WRITE) && !(remaining_flags[which_db] & DB_NOOVERWRITE)) {
//Needs locking
r = toku_db_get_point_write_lock(db, txn, &put_key);
if (r != 0) goto cleanup;
}
}
brts[which_db] = db->i->ft_handle;
}
@ -728,15 +752,8 @@ env_put_multiple_internal(
toku_indexer_lock(indexer);
}
toku_multi_operation_client_lock();
if (num_dbs == 1) {
log_put_single(txn, brts[0], &put_keys[0], &put_vals[0]);
}
else {
log_put_multiple(txn, src_db, src_key, src_val, num_dbs, brts);
}
if (r == 0) {
r = do_put_multiple(txn, num_dbs, db_array, put_keys, put_vals, src_db, src_key);
}
log_put_multiple(txn, src_db, src_key, src_val, num_dbs, brts);
r = do_put_multiple(txn, num_dbs, db_array, put_keys, put_vals, src_db, src_key);
toku_multi_operation_client_unlock();
if (indexer) {
toku_indexer_unlock(indexer);
@ -750,18 +767,30 @@ cleanup:
return r;
}
static void swap_dbts(DBT *a, DBT *b) {
DBT c;
c = *a;
*a = *b;
*b = c;
}
//TODO: 26 Add comment in API description about.. new val.size being generated as '0' REQUIRES old_val.size == 0
//
int
env_update_multiple(DB_ENV *env, DB *src_db, DB_TXN *txn,
env_update_multiple(DB_ENV *env, DB *src_db, DB_TXN *txn,
DBT *old_src_key, DBT *old_src_data,
DBT *new_src_key, DBT *new_src_data,
uint32_t num_dbs, DB **db_array, uint32_t* flags_array,
uint32_t num_keys, DBT keys[],
uint32_t num_vals, DBT vals[]) {
uint32_t num_dbs, DB **db_array, uint32_t* flags_array,
uint32_t num_keys, DBT_ARRAY keys[],
uint32_t num_vals, DBT_ARRAY vals[]) {
int r = 0;
HANDLE_PANICKED_ENV(env);
DB_INDEXER* indexer = NULL;
HANDLE_READ_ONLY_TXN(txn);
DBT_ARRAY old_key_arrays[num_dbs];
DBT_ARRAY new_key_arrays[num_dbs];
DBT_ARRAY new_val_arrays[num_dbs];
if (!txn) {
r = EINVAL;
@ -772,6 +801,10 @@ env_update_multiple(DB_ENV *env, DB *src_db, DB_TXN *txn,
goto cleanup;
}
if (num_dbs + num_dbs > num_keys || num_dbs > num_vals) {
r = ENOMEM; goto cleanup;
}
HANDLE_ILLEGAL_WORKING_PARENT_TXN(env, txn);
r = get_indexer_if_exists(num_dbs, db_array, &indexer);
if (r) {
@ -782,101 +815,195 @@ env_update_multiple(DB_ENV *env, DB *src_db, DB_TXN *txn,
uint32_t n_del_dbs = 0;
DB *del_dbs[num_dbs];
FT_HANDLE del_fts[num_dbs];
DBT del_keys[num_dbs];
DBT_ARRAY del_key_arrays[num_dbs];
uint32_t n_put_dbs = 0;
DB *put_dbs[num_dbs];
FT_HANDLE put_fts[num_dbs];
DBT put_keys[num_dbs];
DBT put_vals[num_dbs];
DBT_ARRAY put_key_arrays[num_dbs];
DBT_ARRAY put_val_arrays[num_dbs];
uint32_t lock_flags[num_dbs];
uint32_t remaining_flags[num_dbs];
for (uint32_t which_db = 0; which_db < num_dbs; which_db++) {
DB *db = db_array[which_db];
DBT curr_old_key, curr_new_key, curr_new_val;
lock_flags[which_db] = get_prelocked_flags(flags_array[which_db]);
remaining_flags[which_db] = flags_array[which_db] & ~lock_flags[which_db];
// keys[0..num_dbs-1] are the new keys
// keys[num_dbs..2*num_dbs-1] are the old keys
// vals[0..num_dbs-1] are the new vals
// Generate the old key and val
if (which_db + num_dbs >= num_keys) {
r = ENOMEM; goto cleanup;
}
if (db == src_db) {
curr_old_key = *old_src_key;
}
else {
// Copy the old keys
old_key_arrays[which_db].size = old_key_arrays[which_db].capacity = 1;
old_key_arrays[which_db].dbts = old_src_key;
// Copy the new keys and vals
new_key_arrays[which_db].size = new_key_arrays[which_db].capacity = 1;
new_key_arrays[which_db].dbts = new_src_key;
new_val_arrays[which_db].size = new_val_arrays[which_db].capacity = 1;
new_val_arrays[which_db].dbts = new_src_data;
} else {
// keys[0..num_dbs-1] are the new keys
// keys[num_dbs..2*num_dbs-1] are the old keys
// vals[0..num_dbs-1] are the new vals
// Generate the old keys
r = env->i->generate_row_for_put(db, src_db, &keys[which_db + num_dbs], NULL, old_src_key, old_src_data);
if (r != 0) goto cleanup;
curr_old_key = keys[which_db + num_dbs];
}
// Generate the new key and val
if (which_db >= num_keys || which_db >= num_vals) {
r = ENOMEM; goto cleanup;
}
if (db == src_db) {
curr_new_key = *new_src_key;
curr_new_val = *new_src_data;
}
else {
paranoid_invariant(keys[which_db+num_dbs].size <= keys[which_db+num_dbs].capacity);
old_key_arrays[which_db] = keys[which_db+num_dbs];
// Generate the new keys and vals
r = env->i->generate_row_for_put(db, src_db, &keys[which_db], &vals[which_db], new_src_key, new_src_data);
if (r != 0) goto cleanup;
curr_new_key = keys[which_db];
curr_new_val = vals[which_db];
paranoid_invariant(keys[which_db].size <= keys[which_db].capacity);
paranoid_invariant(vals[which_db].size <= vals[which_db].capacity);
paranoid_invariant(keys[which_db].size == vals[which_db].size);
new_key_arrays[which_db] = keys[which_db];
new_val_arrays[which_db] = vals[which_db];
}
ft_compare_func cmpfun = toku_db_get_compare_fun(db);
bool key_eq = cmpfun(db, &curr_old_key, &curr_new_key) == 0;
bool key_bytes_eq = (curr_old_key.size == curr_new_key.size &&
(memcmp(curr_old_key.data, curr_new_key.data, curr_old_key.size) == 0)
);
if (!key_eq) {
//Check overwrite constraints only in the case where
// the keys are not equal.
// If the keys are equal, then we do not care of the flag is DB_NOOVERWRITE or 0
r = db_put_check_overwrite_constraint(db, txn,
&curr_new_key,
lock_flags[which_db], remaining_flags[which_db]);
if (r != 0) goto cleanup;
if (remaining_flags[which_db] == DB_NOOVERWRITE_NO_ERROR) {
//update_multiple does not support delaying the no error, since we would
//have to log the flag in the put_multiple.
r = EINVAL; goto cleanup;
DBT_ARRAY &old_keys = old_key_arrays[which_db];
DBT_ARRAY &new_keys = new_key_arrays[which_db];
DBT_ARRAY &new_vals = new_val_arrays[which_db];
uint32_t num_skip = 0;
uint32_t num_del = 0;
uint32_t num_put = 0;
// Next index in old_keys to look at
uint32_t idx_old = 0;
// Next index in new_keys/new_vals to look at
uint32_t idx_new = 0;
uint32_t idx_old_used = 0;
uint32_t idx_new_used = 0;
while (idx_old < old_keys.size || idx_new < new_keys.size) {
// Check for old key, both, new key
DBT *curr_old_key = &old_keys.dbts[idx_old];
DBT *curr_new_key = &new_keys.dbts[idx_new];
DBT *curr_new_val = &new_vals.dbts[idx_new];
bool locked_new_key = false;
int cmp;
if (idx_new == new_keys.size) {
cmp = -1;
} else if (idx_old == old_keys.size) {
cmp = +1;
} else {
ft_compare_func cmpfun = toku_db_get_compare_fun(db);
cmp = cmpfun(db, curr_old_key, curr_new_key);
}
// lock old key
if (db->i->lt && !(lock_flags[which_db] & DB_PRELOCKED_WRITE)) {
r = toku_db_get_point_write_lock(db, txn, &curr_old_key);
bool do_del = false;
bool do_put = false;
bool do_skip = false;
if (cmp > 0) { // New key does not exist in old array
//Check overwrite constraints only in the case where the keys are not equal
//(new key is alone/not equal to old key)
// If the keys are equal, then we do not care of the flag is DB_NOOVERWRITE or 0
r = db_put_check_overwrite_constraint(db, txn,
curr_new_key,
lock_flags[which_db], remaining_flags[which_db]);
if (r != 0) goto cleanup;
if (remaining_flags[which_db] == DB_NOOVERWRITE) {
locked_new_key = true;
}
if (remaining_flags[which_db] == DB_NOOVERWRITE_NO_ERROR) {
//update_multiple does not support delaying the no error, since we would
//have to log the flag in the put_multiple.
r = EINVAL; goto cleanup;
}
do_put = true;
} else if (cmp < 0) {
// lock old key only when it does not exist in new array
// otherwise locking new key takes care of this
if (db->i->lt && !(lock_flags[which_db] & DB_PRELOCKED_WRITE)) {
r = toku_db_get_point_write_lock(db, txn, curr_old_key);
if (r != 0) goto cleanup;
}
do_del = true;
} else {
do_put = curr_new_val->size > 0 ||
curr_old_key->size != curr_new_key->size ||
memcmp(curr_old_key->data, curr_new_key->data, curr_old_key->size);
do_skip = !do_put;
}
// Check put size constraints and insert new key only if keys are unequal (byte for byte) or there is a val
// We assume any val.size > 0 as unequal (saves on generating old val)
// (allows us to avoid generating the old val)
// we assume that any new vals with size > 0 are different than the old val
// if (!key_eq || !(dbt_cmp(&vals[which_db], &vals[which_db + num_dbs]) == 0)) { /* ... */ }
if (do_put) {
r = db_put_check_size_constraints(db, curr_new_key, curr_new_val);
if (r != 0) goto cleanup;
// lock new key unless already locked
if (db->i->lt && !(lock_flags[which_db] & DB_PRELOCKED_WRITE) && !locked_new_key) {
r = toku_db_get_point_write_lock(db, txn, curr_new_key);
if (r != 0) goto cleanup;
}
}
// TODO: 26 Add comments explaining squish and why not just use another stack array
// Add more comments to explain this if elseif else well
if (do_skip) {
paranoid_invariant(cmp == 0);
paranoid_invariant(!do_put);
paranoid_invariant(!do_del);
num_skip++;
idx_old++;
idx_new++;
} else if (do_put) {
paranoid_invariant(cmp >= 0);
paranoid_invariant(!do_skip);
paranoid_invariant(!do_del);
num_put++;
if (idx_new != idx_new_used) {
swap_dbts(&new_keys.dbts[idx_new_used], &new_keys.dbts[idx_new]);
swap_dbts(&new_vals.dbts[idx_new_used], &new_vals.dbts[idx_new]);
}
idx_new++;
idx_new_used++;
if (cmp == 0) {
idx_old++;
}
} else {
invariant(do_del);
paranoid_invariant(cmp < 0);
paranoid_invariant(!do_skip);
paranoid_invariant(!do_put);
num_del++;
if (idx_old != idx_old_used) {
swap_dbts(&old_keys.dbts[idx_old_used], &old_keys.dbts[idx_old]);
}
idx_old++;
idx_old_used++;
}
}
old_keys.size = idx_old_used;
new_keys.size = idx_new_used;
new_vals.size = idx_new_used;
if (num_del > 0) {
del_dbs[n_del_dbs] = db;
del_fts[n_del_dbs] = db->i->ft_handle;
del_keys[n_del_dbs] = curr_old_key;
del_key_arrays[n_del_dbs] = old_keys;
n_del_dbs++;
}
// we take a shortcut and avoid generating the old val
// we assume that any new vals with size > 0 are different than the old val
// if (!key_eq || !(dbt_cmp(&vals[which_db], &vals[which_db + num_dbs]) == 0)) {
if (!key_bytes_eq || curr_new_val.size > 0) {
r = db_put_check_size_constraints(db, &curr_new_key, &curr_new_val);
if (r != 0) goto cleanup;
// lock new key
if (db->i->lt && !(lock_flags[which_db] & DB_PRELOCKED_WRITE)) {
r = toku_db_get_point_write_lock(db, txn, &curr_new_key);
if (r != 0) goto cleanup;
}
// If we put none, but delete some, but not all, then we need the log_put_multiple to happen.
// Include this db in the put_dbs so we do log_put_multiple.
// do_put_multiple will be a no-op for this db.
if (num_put > 0 || (num_del > 0 && num_skip > 0)) {
put_dbs[n_put_dbs] = db;
put_fts[n_put_dbs] = db->i->ft_handle;
put_keys[n_put_dbs] = curr_new_key;
put_vals[n_put_dbs] = curr_new_val;
put_key_arrays[n_put_dbs] = new_keys;
put_val_arrays[n_put_dbs] = new_vals;
n_put_dbs++;
}
}
@ -885,23 +1012,17 @@ env_update_multiple(DB_ENV *env, DB *src_db, DB_TXN *txn,
}
toku_multi_operation_client_lock();
if (r == 0 && n_del_dbs > 0) {
if (n_del_dbs == 1) {
log_del_single(txn, del_fts[0], &del_keys[0]);
} else {
log_del_multiple(txn, src_db, old_src_key, old_src_data, n_del_dbs, del_fts, del_keys);
}
if (r == 0) {
r = do_del_multiple(txn, n_del_dbs, del_dbs, del_keys, src_db, old_src_key);
}
log_del_multiple(txn, src_db, old_src_key, old_src_data, n_del_dbs, del_fts, del_key_arrays);
r = do_del_multiple(txn, n_del_dbs, del_dbs, del_key_arrays, src_db, old_src_key);
}
if (r == 0 && n_put_dbs > 0) {
if (n_put_dbs == 1)
log_put_single(txn, put_fts[0], &put_keys[0], &put_vals[0]);
else
log_put_multiple(txn, src_db, new_src_key, new_src_data, n_put_dbs, put_fts);
if (r == 0)
r = do_put_multiple(txn, n_put_dbs, put_dbs, put_keys, put_vals, src_db, new_src_key);
// We sometimes skip some keys for del/put during runtime, but during recovery
// we (may) delete ALL the keys for a given DB. Therefore we must put ALL the keys during
// recovery so we don't end up losing data.
// So unlike env->put_multiple, we ONLY log a 'put_multiple' log entry.
log_put_multiple(txn, src_db, new_src_key, new_src_data, n_put_dbs, put_fts);
r = do_put_multiple(txn, n_put_dbs, put_dbs, put_key_arrays, put_val_arrays, src_db, new_src_key);
}
toku_multi_operation_client_unlock();
if (indexer) {
@ -974,7 +1095,7 @@ cleanup:
}
int
env_put_multiple(DB_ENV *env, DB *src_db, DB_TXN *txn, const DBT *src_key, const DBT *src_val, uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array) {
env_put_multiple(DB_ENV *env, DB *src_db, DB_TXN *txn, const DBT *src_key, const DBT *src_val, uint32_t num_dbs, DB **db_array, DBT_ARRAY *keys, DBT_ARRAY *vals, uint32_t *flags_array) {
int r = env_check_avail_fs_space(env);
if (r == 0) {
r = env_put_multiple_internal(env, src_db, txn, src_key, src_val, num_dbs, db_array, keys, vals, flags_array);

48
src/ydb_write.h
View file

@ -126,37 +126,37 @@ int autotxn_db_put(DB* db, DB_TXN* txn, DBT* key, DBT* data, uint32_t flags);
int autotxn_db_update(DB *db, DB_TXN *txn, const DBT *key, const DBT *update_function_extra, uint32_t flags);
int autotxn_db_update_broadcast(DB *db, DB_TXN *txn, const DBT *update_function_extra, uint32_t flags);
int env_put_multiple(
DB_ENV *env,
DB *src_db,
DB_TXN *txn,
const DBT *src_key, const DBT *src_val,
uint32_t num_dbs,
DB **db_array,
DBT *keys, DBT *vals,
DB_ENV *env,
DB *src_db,
DB_TXN *txn,
const DBT *src_key, const DBT *src_val,
uint32_t num_dbs,
DB **db_array,
DBT_ARRAY *keys, DBT_ARRAY *vals,
uint32_t *flags_array
);
int env_del_multiple(
DB_ENV *env,
DB *src_db,
DB_TXN *txn,
const DBT *src_key,
const DBT *src_val,
uint32_t num_dbs,
DB **db_array,
DBT *keys,
DB_ENV *env,
DB *src_db,
DB_TXN *txn,
const DBT *src_key,
const DBT *src_val,
uint32_t num_dbs,
DB **db_array,
DBT_ARRAY *keys,
uint32_t *flags_array
);
);
int env_update_multiple(
DB_ENV *env,
DB *src_db,
DB_TXN *txn,
DB_ENV *env,
DB *src_db,
DB_TXN *txn,
DBT *old_src_key, DBT *old_src_data,
DBT *new_src_key, DBT *new_src_data,
uint32_t num_dbs,
DB **db_array,
uint32_t* flags_array,
uint32_t num_keys, DBT keys[],
uint32_t num_vals, DBT vals[]
uint32_t num_dbs,
DB **db_array,
uint32_t* flags_array,
uint32_t num_keys, DBT_ARRAY keys[],
uint32_t num_vals, DBT_ARRAY vals[]
);