/* -*- mode: C; c-basic-offset: 4 -*- */ #ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved." #include "brt-internal.h" #include "key.h" #include "kv-pair.h" #include "pma-internal.h" #include "test.h" #include "toku_assert.h" #include #include #include #include /* we use pma cursors for testing the pma_search function. otherwise, there are no pma cursors */ #include "pma-cursor.h" static TOKULOGGER const null_logger = 0; static TOKUTXN const null_txn = 0; static DB * const null_db = 0; static const DISKOFF null_diskoff = -1; static const FILENUM null_filenum = {0}; static TXNID const null_txnid = 0; #define NULL_ARGS null_logger, null_txnid, null_filenum, null_diskoff void *skey=0, *sval=0; void local_memory_check_all_free(void) { if (skey) toku_free(skey); if (sval) toku_free(sval); skey = sval = 0; toku_memory_check_all_free(); } static void test_make_space_at (void) { PMA pma; char *key; int r; unsigned int newi; struct kv_pair *key_A, *key_B; key = "A"; key_A = kv_pair_malloc(key, strlen(key)+1, 0, 0); key = "B"; key_B = kv_pair_malloc(key, strlen(key)+1, 0, 0); r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0); assert(toku_pma_n_entries(pma)==0); r=toku_pmainternal_make_space_at(null_logger, null_filenum, null_diskoff, pma, 2, &newi, (LSN*)0); assert(r==0); assert(toku_pma_index_limit(pma)==4); assert((unsigned long)pma->pairs[toku_pma_index_limit(pma)]==0xdeadbeefL); if (verbose) toku_print_pma(pma); pma->pairs[2] = key_A; pma->n_pairs_present++; r=toku_pmainternal_make_space_at(null_logger, null_filenum, null_diskoff, pma, 2, &newi, (LSN*)0); assert(r==0); if (verbose) printf("Requested space at 2, got space at %d\n", newi); if (verbose) toku_print_pma(pma); assert(pma->pairs[newi]==0); assert((unsigned long)pma->pairs[toku_pma_index_limit(pma)]==0xdeadbeefL); assert(toku_pma_index_limit(pma)==4); pma->pairs[0] = key_A; pma->pairs[1] = key_B; pma->pairs[2] = 0; pma->pairs[3] = 0; pma->n_pairs_present=2; if (verbose) toku_print_pma(pma); toku_pmainternal_make_space_at(null_logger, null_filenum, null_diskoff, pma, 0, &newi, (LSN*)0); assert(r==0); if (verbose) printf("Requested space at 0, got space at %d\n", newi); if (verbose) toku_print_pma(pma); assert((unsigned long)pma->pairs[toku_pma_index_limit(pma)]==0xdeadbeefL); // make sure it doesn't go off the end. assert(toku_pma_index_limit(pma)==8); pma->pairs[0] = key_A; pma->pairs[1] = 0; pma->pairs[2] = 0; pma->pairs[3] = 0; pma->pairs[4] = key_B; pma->pairs[5] = 0; pma->pairs[6] = 0; pma->pairs[7] = 0; pma->n_pairs_present=2; if (verbose) toku_print_pma(pma); r=toku_pmainternal_make_space_at(null_logger, null_filenum, null_diskoff, pma, 5, &newi, (LSN*)0); assert(r==0); if (verbose) toku_print_pma(pma); if (verbose) printf("r=%d\n", newi); { unsigned int i; for (i=0; ipairs[i]) { assert(ipairs[i] = 0; } } } pma->n_pairs_present = 0; r=toku_pma_free(&pma); assert(r==0); assert(pma==0); kv_pair_free(key_A); kv_pair_free(key_B); } static void test_pma_find (void) { PMA pma; unsigned int i, fidx; int r; const unsigned int N = 16; DBT k; MALLOC(pma); MALLOC_N(N,pma->pairs); // All that is needed to test pma_find is N and pairs. pma->N = N; for (i=0; ipairs[i]=0; assert(toku_pma_index_limit(pma)==N); pma->compare_fun = toku_default_compare_fun; r=toku_pmainternal_find(pma, toku_fill_dbt(&k, "hello", 5)); assert(r==0); pma->pairs[5] = kv_pair_malloc("hello", 5, 0, 0); assert(toku_pma_index_limit(pma)==N); fidx=toku_pmainternal_find(pma, toku_fill_dbt(&k, "hello", 5)); assert(toku_pma_index_limit(pma)==N); assert(fidx==5); fidx=toku_pmainternal_find(pma, toku_fill_dbt(&k, "there", 5)); assert(fidx==6); fidx=toku_pmainternal_find(pma, toku_fill_dbt(&k, "aaa", 3)); assert(fidx==0); pma->pairs[N-1] = kv_pair_malloc("there", 5, 0, 0); fidx=toku_pmainternal_find(pma, toku_fill_dbt(&k, "hello", 5)); assert(fidx==5); fidx=toku_pmainternal_find(pma, toku_fill_dbt(&k, "there", 5)); assert(fidx+1==N); fidx=toku_pmainternal_find(pma, toku_fill_dbt(&k, "aaa", 3)); assert(fidx==0); fidx=toku_pmainternal_find(pma, toku_fill_dbt(&k, "hellob", 6)); assert(fidx==6); fidx=toku_pmainternal_find(pma, toku_fill_dbt(&k, "zzz", 3)); assert(fidx==N); for (i=0; ipairs[i]) kv_pair_free(pma->pairs[i]); toku_free(pma->pairs); toku_free(pma); } static void test_smooth_region_N (int N) { struct kv_pair *pairs[N]; struct kv_pair *strings[N]; char string[N]; int i; int len; if (N<10) len=1; else if (N<100) len=2; else len=8; for (i=0; ikey); assert(cleari&(1<r); } } assert(cleari==0); } } } for (i=0; ikey : "?"); printf("} %d\n", r); } for (i=0; i<7; i++) if (pairs[i]) kv_pair_free(pairs[i]); } static void test_smooth_region (void) { test_smooth_region_N(4); test_smooth_region_N(5); test_smooth_region6(); } static void test_calculate_parameters (void) { struct pma pma; pma.N=4; toku_pmainternal_calculate_parameters(&pma); assert(pma.uplgN==2); assert(pma.udt_step==0.5); pma.N=8; toku_pmainternal_calculate_parameters(&pma); assert(pma.uplgN==4); assert(pma.udt_step==0.5); } static void test_count_region (void) { const int N = 4; struct kv_pair *pairs[N]; int i; char *key; for (i=0; ipairs[toku_pma_index_limit(pma)]==0xdeadbeefL); do_insert(pma, "00000", 6, "d0", 3, rand4fingerprint, &sum, &expect_fingerprint); assert((unsigned long)pma->pairs[toku_pma_index_limit(pma)]==0xdeadbeefL); r=toku_pma_free(&pma); assert(r==0); assert(pma==0); toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(pma!=0); rand4fingerprint = random(); sum = expect_fingerprint = 0; { int i; for (i=0; i<100; i++) { char string[10]; char dstring[10]; snprintf(string,10,"%05d",i); snprintf(dstring,10,"d%d", i); //printf("Inserting %d: string=%s dstring=%s (before sum=%08x) \n", i, string, dstring, sum); do_insert(pma, string, strlen(string)+1, dstring, strlen(dstring)+1, rand4fingerprint, &sum, &expect_fingerprint); } } r=toku_pma_free(&pma); assert(r==0); assert(pma==0); } static int tpi_k,tpi_v; static void do_sum_em (bytevec key, ITEMLEN keylen, bytevec val, ITEMLEN vallen, void *v) { assert((unsigned long)v==0xdeadbeefL); assert(strlen(key)+1==keylen); assert(strlen(val)+1==vallen); tpi_k += atoi(key); tpi_v += atoi(val); } static void test_pma_iterate_internal (PMA pma, int expected_k, int expected_v) { tpi_k=tpi_v=0; toku_pma_iterate(pma, do_sum_em, (void*)0xdeadbeefL); assert(tpi_k==expected_k); assert(tpi_v==expected_v); } static void test_pma_iterate (void) { PMA pma; int r; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); do_insert(pma, "42", 3, "-19", 4, rand4fingerprint, &sum, &expect_fingerprint); test_pma_iterate_internal(pma, 42, -19); do_insert(pma, "12", 3, "-100", 5, rand4fingerprint, &sum, &expect_fingerprint); test_pma_iterate_internal(pma, 42+12, -19-100); r=toku_pma_free(&pma); assert(r==0); assert(pma==0); } static void test_pma_iterate2 (void) { PMA pma0,pma1; int r; int sum=0; int n_items=0; u_int32_t rand4fingerprint0 = random(); u_int32_t sum0 = 0; u_int32_t expect_fingerprint0 = 0; u_int32_t rand4fingerprint1 = random(); u_int32_t sum1 = 0; u_int32_t expect_fingerprint1 = 0; r=toku_pma_create(&pma0, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0); r=toku_pma_create(&pma1, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0); do_insert(pma0, "a", 2, "aval", 5, rand4fingerprint0, &sum0, &expect_fingerprint0); do_insert(pma0, "b", 2, "bval", 5, rand4fingerprint0, &sum0, &expect_fingerprint0); do_insert(pma1, "x", 2, "xval", 5, rand4fingerprint1, &sum1, &expect_fingerprint1); PMA_ITERATE(pma0,kv __attribute__((__unused__)),kl,dv __attribute__((__unused__)),dl, (n_items++,sum+=kl+dl)); PMA_ITERATE(pma1,kv __attribute__((__unused__)),kl,dv __attribute__((__unused__)), dl, (n_items++,sum+=kl+dl)); assert(sum==21); assert(n_items==3); r=toku_pma_free(&pma0); assert(r==0); assert(pma0==0); r=toku_pma_free(&pma1); assert(r==0); assert(pma1==0); } /* Check to see if we can create and kill a cursor. */ static void test_pma_cursor_0 (void) { PMA pma; PMA_CURSOR c=0; int r; r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0); r=toku_pma_cursor(pma, &c, &skey, &sval); assert(r==0); assert(c!=0); if (verbose) printf("%s:%d\n", __FILE__, __LINE__); #if OLDCURSORS r=toku_pma_free(&pma); assert(r!=0); /* didn't deallocate the cursor. */ #endif if (verbose) printf("%s:%d\n", __FILE__, __LINE__); r=toku_pma_cursor_free(&c); assert(r==0); if (verbose) printf("%s:%d\n", __FILE__, __LINE__); r=toku_pma_free(&pma); assert(r==0); /* did deallocate the cursor. */ } /* Make sure we can free the cursors in any order. There is a doubly linked list of cursors * and if we free them in a different order, then different unlinking code is invoked. */ static void test_pma_cursor_1 (void) { #if OLDCURSORS PMA pma; PMA_CURSOR c0=0,c1=0,c2=0; int r; int order; for (order=0; order<6; order++) { r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); assert(r==0); r=toku_pma_cursor(pma, &c0, &skey, &sval); assert(r==0); assert(c0!=0); r=toku_pma_cursor(pma, &c1, &skey, &sval); assert(r==0); assert(c1!=0); r=toku_pma_cursor(pma, &c2, &skey, &sval); assert(r==0); assert(c2!=0); r=toku_pma_free(&pma); assert(r!=0); if (order<2) { r=toku_pma_cursor_free(&c0); assert(r==0); c0=c1; c1=c2; } else if (order<4) { r=toku_pma_cursor_free(&c1); assert(r==0); c1=c2; } else { r=toku_pma_cursor_free(&c2); assert(r==0); } r=toku_pma_free(&pma); assert(r!=0); if (order%2==0) { r=toku_pma_cursor_free(&c0); assert(r==0); c0=c1; } else { r=toku_pma_cursor_free(&c1); assert(r==0); } r=toku_pma_free(&pma); assert(r!=0); r = toku_pma_cursor_free(&c0); assert(r==0); r=toku_pma_free(&pma); assert(r==0); } #endif } static void test_pma_cursor_2 (void) { PMA pma; PMA_CURSOR c=0; int r; DBT key,val; toku_init_dbt(&key); key.flags=DB_DBT_REALLOC; toku_init_dbt(&val); val.flags=DB_DBT_REALLOC; r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0); r=toku_pma_cursor(pma, &c, &skey, &sval); assert(r==0); assert(c!=0); r=toku_pma_cursor_set_position_first(c); assert(r==DB_NOTFOUND); r=toku_pma_cursor_set_position_last(c); assert(r==DB_NOTFOUND); r=toku_pma_cursor_free(&c); assert(r==0); r=toku_pma_free(&pma); assert(r==0); } static void test_pma_cursor_3 (void) { PMA pma; PMA_CURSOR c=0; int r; DBT key,val; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0); do_insert(pma, "x", 2, "xx", 3, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "m", 2, "mm", 3, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "aa", 3, "a", 2, rand4fingerprint, &sum, &expect_fingerprint); toku_init_dbt(&key); key.flags=DB_DBT_REALLOC; toku_init_dbt(&val); val.flags=DB_DBT_REALLOC; r=toku_pma_cursor(pma, &c, &skey, &sval); assert(r==0); assert(c!=0); r=toku_pma_cursor_set_position_first(c); assert(r==0); r=toku_pma_cursor_get_current(c, &key, &val, 0); assert(r==0); assert(key.size=3); assert(memcmp(key.data,"aa",3)==0); assert(val.size=2); assert(memcmp(val.data,"a",2)==0); r=toku_pma_cursor_set_position_next(c); assert(r==0); r=toku_pma_cursor_get_current(c, &key, &val, 0); assert(r==0); assert(key.size=2); assert(memcmp(key.data,"m",2)==0); assert(val.size=3); assert(memcmp(val.data,"mm",3)==0); r=toku_pma_cursor_set_position_next(c); assert(r==0); r=toku_pma_cursor_get_current(c, &key, &val, 0); assert(r==0); assert(key.size=2); assert(memcmp(key.data,"x",2)==0); assert(val.size=3); assert(memcmp(val.data,"xx",3)==0); r=toku_pma_cursor_set_position_next(c); assert(r==DB_NOTFOUND); /* After an error, the cursor should still point at the same thing. */ r=toku_pma_cursor_get_current(c, &key, &val, 0); assert(r==0); assert(key.size=2); assert(memcmp(key.data,"x",2)==0); assert(val.size=3); assert(memcmp(val.data,"xx",3)==0); r=toku_pma_cursor_set_position_next(c); assert(r==DB_NOTFOUND); r = toku_pma_cursor_set_position_first(c); assert(r == 0); r = toku_pma_cursor_set_position_prev(c); assert(r == DB_NOTFOUND); r = toku_pma_cursor_set_position_last(c); assert(r == 0); r = toku_pma_cursor_set_position_next(c); assert(r == DB_NOTFOUND); toku_free(key.data); toku_free(val.data); r=toku_pma_cursor_free(&c); assert(r==0); r=toku_pma_free(&pma); assert(r==0); } static void assert_cursor_val(PMA_CURSOR cursor, int v) { DBT key, val; int r; toku_init_dbt(&key); key.flags = DB_DBT_MALLOC; toku_init_dbt(&val); val.flags = DB_DBT_MALLOC; r = toku_pma_cursor_get_current(cursor, &key, &val, 0); assert(r == 0); assert(v == *(int *)val.data); toku_free(key.data); toku_free(val.data); } /* make sure cursors are adjusted when the pma grows */ static void test_pma_cursor_4 (void) { int r; PMA pma; PMA_CURSOR cursora = 0, cursorb = 0, cursorc = 0; int i; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; if (verbose) printf("test_pma_cursor_4\n"); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); for (i=1; i<=4; i += 1) { char k[5]; int v; sprintf(k, "%4.4d", i); v = i; do_insert(pma, k, strlen(k)+1, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); } assert(toku_pma_n_entries(pma) == 4); if (verbose) { printf("a:"); toku_print_pma(pma); } r = toku_pma_cursor(pma, &cursora, &skey, &sval); assert(r == 0); r = toku_pma_cursor_set_position_first(cursora); assert(r == 0); assert_cursor_val(cursora, 1); r = toku_pma_cursor(pma, &cursorb, &skey, &sval); assert(r == 0); r = toku_pma_cursor_set_position_first(cursorb); assert(r == 0); assert_cursor_val(cursorb, 1); r = toku_pma_cursor_set_position_next(cursorb); assert(r == 0); assert_cursor_val(cursorb, 2); r = toku_pma_cursor(pma, &cursorc, &skey, &sval); assert(r == 0); r = toku_pma_cursor_set_position_last(cursorc); assert(r == 0); assert_cursor_val(cursorc, 4); for (i=5; i<=8; i += 1) { char k[5]; int v; sprintf(k, "%4.4d", i); v = i; do_insert(pma, k, strlen(k)+1, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); } assert(toku_pma_n_entries(pma) == 8); if (verbose) { printf("a:"); toku_print_pma(pma); } assert_cursor_val(cursora, 1); assert_cursor_val(cursorb, 2); assert_cursor_val(cursorc, 4); r = toku_pma_cursor_free(&cursora); assert(r == 0); r = toku_pma_cursor_free(&cursorb); assert(r == 0); r = toku_pma_cursor_free(&cursorc); assert(r == 0); r = toku_pma_free(&pma); assert(r == 0); } static void test_pma_cursor_delete(int n) { if (verbose) printf("test_pma_cursor_delete:%d\n", n); PMA pma; int r; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); /* insert 1 -> 42 */ int k, v; int i; for (i=0; idata; unsigned char *bd=b->data; int siz = a->size; assert(a->size==b->size); // This function requires that the keys be the same size. for (i=0; isize; i++) { if (ad[siz-1-i]bd[siz-1-i]) return +1; } return 0; } static void test_pma_compare_fun (int wrong_endian_p) { PMA pma; PMA_CURSOR c = 0; DBT key,val; int r; char *wrong_endian_expected_keys[] = {"00", "10", "01", "11"}; /* Sorry for being judgemental. But it's wrong. */ char *right_endian_expected_keys[] = {"00", "01", "10", "11"}; char **expected_keys = wrong_endian_p ? wrong_endian_expected_keys : right_endian_expected_keys; int i; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; r = toku_pma_create(&pma, wrong_endian_p ? wrong_endian_compare_fun : toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0); do_insert(pma, "10", 3, "10v", 4, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "00", 3, "00v", 4, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "01", 3, "01v", 4, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "11", 3, "11v", 4, rand4fingerprint, &sum, &expect_fingerprint); toku_init_dbt(&key); key.flags=DB_DBT_REALLOC; toku_init_dbt(&val); val.flags=DB_DBT_REALLOC; r=toku_pma_cursor(pma, &c, &skey, &sval); assert(r==0); assert(c!=0); for (i=0; i<4; i++) { if (i==0) { r=toku_pma_cursor_set_position_first(c); assert(r==0); } else { r=toku_pma_cursor_set_position_next(c); assert(r==0); } r=toku_pma_cursor_get_current(c, &key, &val, 0); assert(r==0); //printf("Got %s, expect %s\n", (char*)key.data, expected_keys[i]); assert(key.size=3); assert(memcmp(key.data,expected_keys[i],3)==0); assert(val.size=4); assert(memcmp(val.data,expected_keys[i],2)==0); assert(memcmp(2+(char*)val.data,"v",2)==0); } r=toku_pma_cursor_set_position_next(c); assert(r==DB_NOTFOUND); for (i=3; i>=0; i--) { if (i==3) { r=toku_pma_cursor_set_position_last(c); assert(r==0); } else { r=toku_pma_cursor_set_position_prev(c); assert(r==0); } r=toku_pma_cursor_get_current(c, &key, &val, 0); assert(r==0); //printf("Got %s, expect %s\n", (char*)key.data, expected_keys[i]); assert(key.size=3); assert(memcmp(key.data,expected_keys[i],3)==0); assert(val.size=4); assert(memcmp(val.data,expected_keys[i],2)==0); assert(memcmp(2+(char*)val.data,"v",2)==0); } r=toku_pma_cursor_set_position_prev(c); assert(r==DB_NOTFOUND); toku_free(key.data); toku_free(val.data); r=toku_pma_cursor_free(&c); assert(r==0); r=toku_pma_free(&pma); assert(r==0); } static void test_pma_dup_split_n(int n, int dup_mode) { if (verbose) printf("test_pma_dup_split_n:%d %d\n", n, dup_mode); int r; u_int32_t expect_fingerprint = 0; u_int32_t arand = random(); u_int32_t asum = 0; u_int32_t crand = random(); u_int32_t csum = 0; struct tuple { uint32_t k; uint64_t v; } tuples[n]; PMA pmaa; r = toku_pma_create(&pmaa, toku_default_compare_fun, null_db, null_filenum, n*(sizeof (struct tuple) + 2*sizeof (int)), 0); assert(r == 0); toku_pma_set_dup_mode(pmaa, dup_mode); toku_pma_set_dup_compare(pmaa, toku_default_compare_fun); PMA pmac; r = toku_pma_create(&pmac, toku_default_compare_fun, null_db, null_filenum, n*(sizeof (struct tuple) + 2*sizeof (int)), 0); assert(r == 0); toku_pma_set_dup_mode(pmac, dup_mode); toku_pma_set_dup_compare(pmac, toku_default_compare_fun); /* generate some unique kv pairs */ int i; for (i=0; ik); } qsort(tuples, n, sizeof (struct tuple), compare_tuple); int compare_key_val_tuple(PMA pma, DBT *k, DBT *v, struct tuple *t) { DBT d; int cmp = pma->compare_fun(pma->db, k, toku_fill_dbt(&d, &t->k, sizeof t->k)); if (cmp == 0) cmp = pma->dup_compare_fun(pma->db, v, toku_fill_dbt(&d, &t->v, sizeof t->v)); return cmp; } /* walk the pma's and verify the sort order with the input kv pairs */ DBT key; memset(&key, 0, sizeof key); key.flags = DB_DBT_REALLOC; DBT val; memset(&val, 0, sizeof val); val.flags = DB_DBT_REALLOC; PMA_CURSOR cursor = 0; i = 0; r = toku_pma_cursor(pmaa, &cursor, &skey, &sval); assert(r == 0); r = toku_pma_cursor_set_position_first(cursor); while (r == 0) { r = toku_pma_cursor_get_current(cursor, &key, &val, 0); assert(r == 0); assert(i < n); assert(compare_key_val_tuple(pmaa, &key, &val, &tuples[i++]) == 0); r = toku_pma_cursor_set_position_next(cursor); } r = toku_pma_cursor_free(&cursor); assert(r == 0); r = toku_pma_cursor(pmac, &cursor, &skey, &sval); assert(r == 0); r = toku_pma_cursor_set_position_first(cursor); while (r == 0) { r = toku_pma_cursor_get_current(cursor, &key, &val, 0); assert(r == 0); assert(i < n); assert(compare_key_val_tuple(pmaa, &key, &val, &tuples[i++]) == 0); r = toku_pma_cursor_set_position_next(cursor); } r = toku_pma_cursor_free(&cursor); assert(r == 0); assert(i == n); /* verify the split key */ int compare_key_val(PMA pma, DBT *k, DBT *v, struct kv_pair *split) { DBT d; int cmp = pma->compare_fun(pma->db, k, toku_fill_dbt(&d, kv_pair_key(split), kv_pair_keylen(split))); if (cmp == 0 && dup_mode) cmp = pma->dup_compare_fun(pma->db, v, toku_fill_dbt(&d, kv_pair_val(split), kv_pair_vallen(split))); return cmp; } if (na > 0) { r = toku_pma_cursor(pmaa, &cursor, &skey, &sval); assert(r == 0); r = toku_pma_cursor_set_position_last(cursor); assert(r == 0); r = toku_pma_cursor_get_current(cursor, &key, &val, 0); assert(r == 0); assert(compare_key_val(pmaa, &key, &val, splitk.data) <= 0); r = toku_pma_cursor_free(&cursor); assert(r == 0); } if (nc > 0) { r = toku_pma_cursor(pmac, &cursor, &skey, &sval); assert(r == 0); r = toku_pma_cursor_set_position_first(cursor); assert(r == 0); r = toku_pma_cursor_get_current(cursor, &key, &val, 0); assert(r == 0); assert(compare_key_val(pmaa, &key, &val, splitk.data) > 0); r = toku_pma_cursor_free(&cursor); assert(r == 0); } if (key.data) toku_free(key.data); if (val.data) toku_free(val.data); if (splitk.data) toku_free(splitk.data); r = toku_pma_free(&pmaa); assert(r == 0); r = toku_pma_free(&pmac); assert(r == 0); } static void test_pma_split_varkey(void) { char *keys[] = { "this", "is", "a", "key", "this is a really really big key", "zz", 0 }; PMA pmaa, pmac; int r; int i; int n, na, nc; u_int32_t expect_fingerprint = 0; u_int32_t arand = random(); u_int32_t asum = 0; u_int32_t crand = random(); u_int32_t csum = 0; if (verbose) printf("test_pma_split_varkey\n"); r = toku_pma_create(&pmaa, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); r = toku_pma_create(&pmac, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); /* insert some kv pairs */ for (i=0; keys[i]; i++) { char v = i; do_insert(pmaa, keys[i], strlen(keys[i])+1, &v, sizeof v, arand, &asum, &expect_fingerprint); } n = i; if (verbose) { printf("a:"); toku_print_pma(pmaa); } r = toku_pma_split(null_logger, null_filenum, null_diskoff, pmaa, 0, arand, &asum, (LSN*)0, 0, null_diskoff, pmac, 0, crand, &csum, (LSN*)0); assert(r == 0); toku_pma_verify(pmaa); toku_pma_verify(pmac); toku_pma_verify_fingerprint(pmaa, arand, asum); toku_pma_verify_fingerprint(pmac, crand, csum); if (verbose) { printf("a:"); toku_print_pma(pmaa); } na = toku_pma_n_entries(pmaa); if (verbose) { printf("c:"); toku_print_pma(pmac); } nc = toku_pma_n_entries(pmac); assert(na + nc == n); r = toku_pma_free(&pmaa); assert(r == 0); r = toku_pma_free(&pmac); assert(r == 0); } #if 0 static void print_cursor(const char *str, PMA_CURSOR cursor) { DBT key, val; int error; if (verbose) printf("cursor %s: ", str); toku_init_dbt(&key); key.flags = DB_DBT_MALLOC; toku_init_dbt(&val); val.flags = DB_DBT_MALLOC; r = toku_pma_cursor_get_current(cursor, &key, &val, 0); assert(r == 0); if (verbose) printf("%s ", (char*)key.data); toku_free(key.data); toku_free(val.data); if (verbose) printf("\n"); } #endif #if OLDCURSORS static void walk_cursor(const char *str, PMA_CURSOR cursor) { DBT key, val; int error; if (verbose) printf("walk %s: ", str); for (;;) { toku_init_dbt(&key); key.flags = DB_DBT_MALLOC; toku_init_dbt(&val); val.flags = DB_DBT_MALLOC; r = toku_pma_cursor_get_current(cursor, &key, &val, 0); assert(r == 0); if (verbose) printf("%s ", (char*)key.data); toku_free(key.data); toku_free(val.data); r = toku_pma_cursor_set_position_next(cursor); if (r != 0) break; } if (verbose) printf("\n"); } static void walk_cursor_reverse(const char *str, PMA_CURSOR cursor) { DBT key, val; int error; if (verbose) printf("walk %s: ", str); for (;;) { toku_init_dbt(&key); key.flags = DB_DBT_MALLOC; toku_init_dbt(&val); val.flags = DB_DBT_MALLOC; r = toku_pma_cursor_get_current(cursor, &key, &val, 0); assert(r == 0); if (verbose) printf("%s ", (char*)key.data); toku_free(key.data); toku_free(val.data); r = toku_pma_cursor_set_position_prev(cursor); if (r != 0) break; } if (verbose) printf("\n"); } #endif static void test_pma_split(void) { test_pma_dup_split_n(0, 0); local_memory_check_all_free(); test_pma_dup_split_n(1, 0); local_memory_check_all_free(); test_pma_dup_split_n(2, 0); local_memory_check_all_free(); test_pma_dup_split_n(4, 0); local_memory_check_all_free(); test_pma_dup_split_n(8, 0); local_memory_check_all_free(); test_pma_dup_split_n(9, 0); local_memory_check_all_free(); test_pma_dup_split_n(1000, 0); local_memory_check_all_free(); test_pma_dup_split_n(0, TOKU_DB_DUP+TOKU_DB_DUPSORT); local_memory_check_all_free(); test_pma_dup_split_n(1, TOKU_DB_DUP+TOKU_DB_DUPSORT); local_memory_check_all_free(); test_pma_dup_split_n(9, TOKU_DB_DUP+TOKU_DB_DUPSORT); local_memory_check_all_free(); test_pma_dup_split_n(1000, TOKU_DB_DUP+TOKU_DB_DUPSORT); local_memory_check_all_free(); test_pma_split_varkey(); local_memory_check_all_free(); } /* * test the toku_pma_bulk_insert function by creating n kv pairs and bulk * inserting them into an empty pma. verify that the pma contains all * of the kv pairs. */ static void test_pma_bulk_insert_n(int n) { PMA pma; int r; int i; DBT *keys, *vals; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; if (verbose) printf("test_pma_bulk_insert_n: %d\n", n); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); /* init n kv pairs */ keys = toku_malloc(n * sizeof (DBT)); assert(keys); vals = toku_malloc(n * sizeof (DBT)); assert(vals); /* init n kv pairs */ for (i=0; iN == PMA_MIN_ARRAY_SIZE); r = toku_pma_free(&pma); assert(r == 0); } /* * test that the pma shrinks to its minimum size after inserting * random keys and then deleting them. */ static void test_pma_delete_random(int n) { PMA pma; int r; int i; int keys[n]; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; if (verbose) printf("test_pma_delete_random:%d\n", n); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + 11 + sizeof (int)), 0); assert(r == 0); for (i=0; iN == PMA_MIN_ARRAY_SIZE); r = toku_pma_free(&pma); assert(r == 0); } static void assert_cursor_equal(PMA_CURSOR pmacursor, int v) { DBT key, val; toku_init_dbt(&key); key.flags = DB_DBT_MALLOC; toku_init_dbt(&val); val.flags = DB_DBT_MALLOC; int r; r = toku_pma_cursor_get_current(pmacursor, &key, &val, 0); assert(r == 0); if (0) printf("key %s\n", (char*) key.data); int thev; assert(val.size == sizeof thev); memcpy(&thev, val.data, val.size); assert(thev == v); toku_free(key.data); toku_free(val.data); } static void assert_cursor_nokey(PMA_CURSOR pmacursor) { DBT key, val; toku_init_dbt(&key); key.flags = DB_DBT_MALLOC; toku_init_dbt(&val); val.flags = DB_DBT_MALLOC; int r; r = toku_pma_cursor_get_current(pmacursor, &key, &val, 0); assert(r != 0); } /* * test that pma delete ops update pma cursors * - insert n keys * - point the cursor at the last key in the pma * - delete keys sequentially. the cursor should be stuck at the * last key until the last key is deleted. */ static void test_pma_delete_cursor(int n) { if (verbose) printf("test_delete_cursor:%d\n", n); PMA pma; int r; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); int i; for (i=0; iN == PMA_MIN_ARRAY_SIZE); r = toku_pma_cursor_free(&pmacursor); assert(r == 0); r = toku_pma_free(&pma); assert(r == 0); } /* * insert k,1 * place cursor at k * delete k * cursor get current * lookup k * insert k,2 * lookup k * cursor get current */ static void test_pma_delete_insert() { if (verbose) printf("test_pma_delete_insert\n"); PMA pma; int r; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); PMA_CURSOR pmacursor = 0; r = toku_pma_cursor(pma, &pmacursor, &skey, &sval); assert(r == 0); DBT key, val; int k, v; k = 1; v = 1; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); r = toku_pma_cursor_set_position_first(pmacursor); assert(r == 0); assert_cursor_equal(pmacursor, 1); k = 1; v = 1; r = do_delete(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); assert(r == 0); assert_cursor_nokey(pmacursor); k = 1; toku_fill_dbt(&key, &k, sizeof k); toku_init_dbt(&val); val.flags = DB_DBT_MALLOC; r = toku_pma_lookup(pma, &key, &val); assert(r != 0); k = 1; v = 2; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); #if OLDCURSORS assert_cursor_equal(pmacursor, 2); #else assert_cursor_nokey(pmacursor); #endif r = toku_pma_cursor_free(&pmacursor); assert(r == 0); r = toku_pma_free(&pma); assert(r == 0); } static void test_pma_double_delete() { if (verbose) printf("test_pma_double_delete\n"); PMA pma; int r; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); PMA_CURSOR pmacursor = 0; r = toku_pma_cursor(pma, &pmacursor, &skey, &sval); assert(r == 0); DBT key; int k, v; k = 1; v = 1; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); r = toku_pma_cursor_set_position_first(pmacursor); assert(r == 0); assert_cursor_equal(pmacursor, 1); k = 1; v = 1; r = do_delete(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); assert(r == 0); assert_cursor_nokey(pmacursor); k = 1; toku_fill_dbt(&key, &k, sizeof k); r = toku_pma_delete(pma, &key, (DBT*)0, (TOKULOGGER)0, (TXNID)0, (DISKOFF)0, rand4fingerprint, &sum, 0, (LSN*)0); assert(r == DB_NOTFOUND); assert(sum == expect_fingerprint); r = toku_pma_cursor_free(&pmacursor); assert(r == 0); r = toku_pma_free(&pma); assert(r == 0); } static void test_pma_cursor_first_delete_last() { if (verbose) printf("test_pma_cursor_first_delete_last\n"); int r; PMA pma; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); int k, v; int i; for (i=1; i<=2; i++) { k = htonl(i); v = i; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); } assert(toku_pma_n_entries(pma) == 2); PMA_CURSOR pmacursor = 0; r = toku_pma_cursor(pma, &pmacursor, &skey, &sval); assert(r == 0); r = toku_pma_cursor_set_position_first(pmacursor); assert(r == 0); k = htonl(1); v = 1; r = do_delete(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); assert(r == 0); #if OLDCURSORS assert(toku_pma_n_entries(pma) == 2); #endif r = toku_pma_cursor_set_position_last(pmacursor); assert(r == 0); assert(toku_pma_n_entries(pma) == 1); r = toku_pma_cursor_free(&pmacursor); assert(r == 0); r = toku_pma_free(&pma); assert(r == 0); } static void test_pma_cursor_last_delete_first() { if (verbose) printf("test_pma_cursor_last_delete_first\n"); int r; PMA pma; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0); int k, v; int i; for (i=1; i<=2; i++) { k = htonl(i); v = i; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); } assert(toku_pma_n_entries(pma) == 2); PMA_CURSOR pmacursor = 0; r = toku_pma_cursor(pma, &pmacursor, &skey, &sval); assert(r == 0); r = toku_pma_cursor_set_position_last(pmacursor); assert(r == 0); k = htonl(2); v = 2; r = do_delete(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); assert(r == 0); #if OLDCURSORS assert(toku_pma_n_entries(pma) == 2); #endif r = toku_pma_cursor_set_position_first(pmacursor); assert(r == 0); assert(toku_pma_n_entries(pma) == 1); r = toku_pma_cursor_free(&pmacursor); assert(r == 0); r = toku_pma_free(&pma); assert(r == 0); } void test_pma_delete_both(int n) { if (verbose) printf("test_pma_delete_both:%d\n", n); int r; PMA pma; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * 4 * sizeof (int), 0); assert(r == 0); toku_pma_set_dup_mode(pma, TOKU_DB_DUP + TOKU_DB_DUPSORT); toku_pma_set_dup_compare(pma, toku_default_compare_fun); int i; for (i=0; i largest_key); } else { toku_init_dbt(&val); val.flags = DB_DBT_MALLOC; r = toku_pma_cursor_get_current(cursor, &key, &val, 0); assert(r == 0); int vv; assert(val.size == sizeof vv); memcpy(&vv, val.data, val.size); if (i <= smallest_key) assert(vv == smallest_key); else assert(vv == (((i+9)/10)*10)); toku_free(val.data); } } r = toku_pma_cursor_free(&cursor); assert(r == 0); r = toku_pma_free(&pma); assert(r == 0); } static void test_pma_cursor_delete_under() { if (verbose) printf("test_pma_cursor_delete_under\n"); int r; PMA pma; u_int32_t rand4fingerprint = random(); u_int32_t sum = 0; u_int32_t expect_fingerprint = 0; const int n = 1000; r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + sizeof (int) + sizeof (int)), 0); assert(r == 0); PMA_CURSOR cursor = 0; r = toku_pma_cursor(pma, &cursor, &skey, &sval); assert(r == 0); u_int32_t kvsize; /* delete under an uninitialized cursor should fail */ r = toku_pma_cursor_delete_under(cursor, &kvsize, (TOKULOGGER)0, (TXNID)0, (DISKOFF)0, rand4fingerprint, &expect_fingerprint, (LSN*)0); assert(r == DB_NOTFOUND || r != 0); int k, v; /* insert 0 .. n-1 */ int i; for (i=0; i0, 1->1, .. n-1->n-1 */ for (i=0; i0, 0->1, .. 0->n-1 */ DBT key, val; int k, v; int i; for (i=0; i1, 3->3 */ k = htonl(1); v = 1; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); toku_pma_verify(pma); k = htonl(3); v = 3; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); toku_pma_verify(pma); u_int32_t sum_before_all_the_duplicates = sum; int i; /* insert 2->0, 2->1, .. 2->n-1 */ for (i=0; i1, 3->3 */ k = htonl(1); v = 1; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); toku_pma_verify(pma); k = htonl(3); v = 3; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); toku_pma_verify(pma); k = htonl(2); int values[n]; int i; for (i=0; in-i */ for (i=0; i1, 3->3 */ k = htonl(1); v = 1; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); toku_pma_verify(pma); k = htonl(3); v = 3; do_insert(pma, &k, sizeof k, &v, sizeof v, rand4fingerprint, &sum, &expect_fingerprint); toku_pma_verify(pma); int i; /* insert 2->0, 2->1, .. 2->n-1 */ for (i=0; i= argc) return usage(argv[0]); srandom(atoi(argv[++i])); } else return usage(argv[0]); } pma_tests(); toku_malloc_cleanup(); return 0; }