diff --git a/newbrt/Makefile b/newbrt/Makefile
index 510cb0f6f1d..3ec990f82de 100644
--- a/newbrt/Makefile
+++ b/newbrt/Makefile
@@ -26,23 +26,28 @@ endif
default: bins
# Put these one-per-line so that if we insert a new one the svn diff can understand it better.
# Also keep them sorted.
-BINS = \
- benchmark-test \
+REGRESSION_TESTS = \
+ ybt-test \
+ pma-test \
brt-serialize-test \
brt-test \
cachetable-test \
+ cachetable-test2 \
hashtest \
- pma-test \
+# This line intentially kept commented so I can have a \ on the end of the previous line
+
+BINS = $(REGRESSION_TESTS) \
+ benchmark-test \
randbrt \
randdb4 \
- ybt-test \
-# This line intentially kept blank so I can have a \ on the end of the previous line
+# This line intentially kept commented so I can have a \ on the end of the previous line
bins: $(BINS)
check: bins
$(DTOOL) ./ybt-test
$(DTOOL) ./pma-test
$(DTOOL) ./cachetable-test
+ $(DTOOL) ./cachetable-test2
$(DTOOL) ./brt-serialize-test
$(DTOOL) ./brt-test
$(DTOOL) ./hashtest
@@ -85,6 +90,9 @@ brt-serialize-test: brt-serialize-test.o brt-serialize.o memory.o hashtable.o pm
cachetable-test.o: cachetable.h memory.h
cachetable-test: cachetable.o memory.o cachetable-test.o
+cachetable-test2.o: cachetable.h memory.h
+cachetable-test2: cachetable.o memory.o cachetable-test2.o
+
benchmark-test: benchmark-test.o ybt.o memory.o brt.o pma.o cachetable.o key.o hashtable.o brt-serialize.o primes.o
benchmark-test.o: brt.h ../include/db.h
diff --git a/newbrt/brt.c b/newbrt/brt.c
index 4a34436de19..96ba7097450 100644
--- a/newbrt/brt.c
+++ b/newbrt/brt.c
@@ -1314,7 +1314,6 @@ int brt_insert (BRT brt, DBT *key, DBT *val, DB* db) {
}
int brt_lookup_node (BRT brt, diskoff off, DBT *k, DBT *v, DB *db) {
- int result;
void *node_v;
int r = cachetable_get_and_pin(brt->cf, off, &node_v,
brtnode_flush_callback, brtnode_fetch_callback, (void*)(long)brt->h->nodesize);
@@ -1325,7 +1324,7 @@ int brt_lookup_node (BRT brt, diskoff off, DBT *k, DBT *v, DB *db) {
int childnum;
if (node->height==0) {
- result = pma_lookup(node->u.l.buffer, k, v, db);
+ int result = pma_lookup(node->u.l.buffer, k, v, db);
//printf("%s:%d looked up something, got answerlen=%d\n", __FILE__, __LINE__, answerlen);
r = cachetable_unpin(brt->cf, off, 0);
assert(r == 0);
@@ -1338,6 +1337,7 @@ int brt_lookup_node (BRT brt, diskoff off, DBT *k, DBT *v, DB *db) {
ITEMLEN hanswerlen;
int type;
if (toku_hash_find (node->u.n.htables[childnum], k->data, k->size, &hanswer, &hanswerlen, &type)==0) {
+ int result;
if (type == BRT_INSERT) {
//printf("Found %d bytes\n", *vallen);
ybt_set_value(v, hanswer, hanswerlen, &brt->sval);
@@ -1345,18 +1345,22 @@ int brt_lookup_node (BRT brt, diskoff off, DBT *k, DBT *v, DB *db) {
result = 0;
} else if (type == BRT_DELETE) {
result = DB_NOTFOUND;
- } else
+ } else {
assert(0);
+ result = -1; // Some versions of gcc complain
+ }
r = cachetable_unpin(brt->cf, off, 0);
assert(r == 0);
return result;
}
}
- result = brt_lookup_node(brt, node->u.n.children[childnum], k, v, db);
- r = cachetable_unpin(brt->cf, off, 0);
- assert(r == 0);
- return result;
+ {
+ int result = brt_lookup_node(brt, node->u.n.children[childnum], k, v, db);
+ r = cachetable_unpin(brt->cf, off, 0);
+ assert(r == 0);
+ return result;
+ }
}
diff --git a/newbrt/cachetable-test.c b/newbrt/cachetable-test.c
index 812a489c35c..5972aa39c84 100644
--- a/newbrt/cachetable-test.c
+++ b/newbrt/cachetable-test.c
@@ -2,9 +2,9 @@
#include "cachetable.h"
#include <assert.h>
-#include <string.h>
-#include <stdlib.h>
#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
#include <unistd.h>
struct item {
diff --git a/newbrt/cachetable-test2.c b/newbrt/cachetable-test2.c
new file mode 100644
index 00000000000..4fb63f1c9e7
--- /dev/null
+++ b/newbrt/cachetable-test2.c
@@ -0,0 +1,161 @@
+#include "memory.h"
+#include "cachetable.h"
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+CACHETABLE ct;
+
+enum { N_PRESENT_LIMIT = 4, TRIALS=200, N_FILES=2 };
+int n_present=0;
+struct present_items {
+ CACHEKEY key;
+ CACHEFILE cf;
+} present_items[N_PRESENT_LIMIT];
+
+static void print_ints(void) __attribute__((__unused__));
+static void print_ints(void) {
+ int i;
+ for (i=0; i<n_present; i++) {
+ if (i==0) printf("{"); else printf(",");
+ printf("{%lld,%p}", present_items[i].key, present_items[i].cf);
+ }
+ printf("}\n");
+}
+
+static void item_becomes_present(CACHEFILE cf, CACHEKEY key) {
+ assert(n_present<N_PRESENT_LIMIT);
+ present_items[n_present].cf = cf;
+ present_items[n_present].key = key;
+ n_present++;
+}
+
+static void item_becomes_not_present(CACHEFILE cf, CACHEKEY key) {
+ int i;
+ //printf("Removing {%4lld %16p}: Initially: ", key, cf); print_ints();
+ assert(n_present<=N_PRESENT_LIMIT);
+ for (i=0; i<n_present; i++) {
+ if (present_items[i].cf==cf && present_items[i].key==key) {
+ present_items[i]=present_items[n_present-1];
+ n_present--;
+ //printf(" Finally: "); print_ints();
+ return;
+ }
+ }
+ printf("Whoops, %p,%lld was already not present\n", cf ,key);
+ abort();
+}
+
+static void file_is_not_present(CACHEFILE cf) {
+ int i;
+ for (i=0; i<n_present; i++) {
+ assert(present_items[i].cf!=cf);
+ }
+}
+
+
+static void flush_forchain (CACHEFILE f __attribute__((__unused__)), CACHEKEY key, void *value, int write_me __attribute__((__unused__)), int keep_me __attribute__((__unused__))) {
+ int *v = value;
+ //cachetable_print_state(ct);
+ //printf("Flush %lld %d\n", key, (int)value);
+ assert((int)v==(int)key);
+ item_becomes_not_present(f, key);
+ //print_ints();
+}
+
+static int fetch_forchain (CACHEFILE f __attribute__((__unused__)), CACHEKEY key, void**value, void*extraargs) {
+ assert((int)extraargs==(int)key);
+ *value = (void*)(int)key;
+ return 0;
+}
+
+void verify_cachetable_against_present (void) {
+ int i;
+ for (i=0; i<n_present; i++) {
+ void *v;
+ int r;
+ assert(cachetable_maybe_get_and_pin(present_items[i].cf,
+ present_items[i].key,
+ &v)==0);
+ r = cachetable_unpin(present_items[i].cf, present_items[i].key, 0);
+ }
+}
+
+
+void test_chaining (void) {
+ /* Make sure that the hash chain and the LRU list don't get confused. */
+ CACHEFILE f[N_FILES];
+ enum { FILENAME_LEN=100 };
+ char fname[N_FILES][FILENAME_LEN];
+ int r;
+ int i, trial;
+ r = create_cachetable(&ct, N_PRESENT_LIMIT); assert(r==0);
+ for (i=0; i<N_FILES; i++) {
+ int r = snprintf(fname[i], FILENAME_LEN, "cachetabletest2.%d.dat", i);
+ assert(r>0 && r<FILENAME_LEN);
+ unlink(fname[i]);
+ r = cachetable_openf(&f[i], ct, fname[i], O_RDWR|O_CREAT, 0777); assert(r==0);
+ }
+ for (i=0; i<N_PRESENT_LIMIT; i++) {
+ int fnum = i%N_FILES;
+ //printf("%s:%d Add %d\n", __FILE__, __LINE__, i);
+ r = cachetable_put(f[fnum], i, (void*)i, flush_forchain, fetch_forchain, (void*)i); assert(r==0);
+ item_becomes_present(f[fnum], i);
+ r = cachetable_unpin(f[fnum], i, 0); assert(r==0);
+ //print_ints();
+ }
+ for (trial=0; trial<TRIALS; trial++) {
+ if (n_present>0) {
+ // First touch some random ones
+ int whichone = random()%n_present;
+ void *value;
+ //printf("Touching %d (%lld, %p)\n", whichone, present_items[whichone].key, present_items[whichone].cf);
+ r = cachetable_get_and_pin(present_items[whichone].cf,
+ present_items[whichone].key,
+ &value,
+ flush_forchain,
+ fetch_forchain,
+ (void*)(int)present_items[whichone].key
+ );
+ assert(r==0);
+ r = cachetable_unpin(present_items[whichone].cf,
+ present_items[whichone].key,
+ 0);
+ assert(r==0);
+ }
+
+ i += 1+ random()%100;
+ int fnum = i%N_FILES;
+ // i is always incrementing, so we need not worry about inserting a duplicate
+ //printf("%s:%d Add {%d,%p}\n", __FILE__, __LINE__, i, f[fnum]);
+ r = cachetable_put(f[fnum], i, (void*)i, flush_forchain, fetch_forchain, (void*)i); assert(r==0);
+ item_becomes_present(f[fnum], i);
+ //print_ints();
+ //cachetable_print_state(ct);
+ r = cachetable_unpin(f[fnum], i, 0); assert(r==0);
+ verify_cachetable_against_present();
+
+ if (random()%10==0) {
+ i = random()%N_FILES;
+ //printf("Close %d (%p), now n_present=%d\n", i, f[i], n_present);
+ //print_ints();
+ CACHEFILE oldcf=f[i];
+ r = cachefile_close(&f[i]); assert(r==0);
+ file_is_not_present(oldcf);
+ r = cachetable_openf(&f[i], ct, fname[i], O_RDWR, 0777); assert(r==0);
+ }
+ }
+ for (i=0; i<N_FILES; i++) {
+ r = cachefile_close(&f[i]); assert(r==0);
+ }
+ r = cachetable_close(&ct); assert(r==0);
+}
+
+int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
+ test_chaining();
+ malloc_cleanup();
+ printf("ok\n");
+ return 0;
+}
diff --git a/newbrt/cachetable.c b/newbrt/cachetable.c
index 61ebcc427ad..b6542c59dea 100644
--- a/newbrt/cachetable.c
+++ b/newbrt/cachetable.c
@@ -53,6 +53,18 @@ struct cachefile {
struct fileid fileid;
};
+void cachetable_print_state (CACHETABLE ct) {
+ int i;
+ for (i=0; i<ct->table_size; i++) {
+ PAIR p;
+ printf("t[%d]=", i);
+ for (p=ct->table[i]; p; p=p->hash_chain) {
+ printf(" {%lld, %p}", p->key, p->cachefile);
+ }
+ printf("\n");
+ }
+}
+
int create_cachetable (CACHETABLE *result, int n_entries) {
TAGMALLOC(CACHETABLE, t);
int i;
@@ -112,7 +124,7 @@ CACHEFILE remove_cf_from_list (CACHEFILE cf, CACHEFILE list) {
}
}
-int cachefile_flush (CACHEFILE cf);
+static int cachefile_flush_and_remove (CACHEFILE cf);
int cachefile_close (CACHEFILE *cfp) {
CACHEFILE cf = *cfp;
@@ -120,7 +132,7 @@ int cachefile_close (CACHEFILE *cfp) {
cf->refcount--;
if (cf->refcount==0) {
int r;
- if ((r = cachefile_flush(cf))) return r;
+ if ((r = cachefile_flush_and_remove(cf))) return r;
r = close(cf->fd);
cf->cachetable->cachefiles = remove_cf_from_list(cf, cf->cachetable->cachefiles);
toku_free(cf);
@@ -337,7 +349,7 @@ int cachetable_maybe_get_and_pin (CACHEFILE cachefile, CACHEKEY key, void**value
*value = p->value;
p->pinned++;
lru_touch(t,p);
- printf("%s:%d cachetable_maybe_get_and_pin(%lld)--> %p\n", __FILE__, __LINE__, key, *value);
+ //printf("%s:%d cachetable_maybe_get_and_pin(%lld)--> %p\n", __FILE__, __LINE__, key, *value);
return 0;
}
}
@@ -374,7 +386,28 @@ int cachetable_flush (CACHETABLE t) {
return 0;
}
-int cachefile_flush (CACHEFILE cf) {
+static void assert_cachefile_is_flushed_and_removed (CACHEFILE cf) {
+ CACHETABLE t = cf->cachetable;
+ int i;
+ // Check it two ways
+ // First way: Look through all the hash chains
+ for (i=0; i<t->table_size; i++) {
+ PAIR p;
+ for (p=t->table[i]; p; p=p->hash_chain) {
+ assert(p->cachefile!=cf);
+ }
+ }
+ // Second way: Look through the LRU list.
+ {
+ PAIR p;
+ for (p=t->head; p; p=p->next) {
+ assert(p->cachefile!=cf);
+ }
+ }
+}
+
+
+static int cachefile_flush_and_remove (CACHEFILE cf) {
int i;
CACHETABLE t = cf->cachetable;
for (i=0; i<t->table_size; i++) {
@@ -390,10 +423,10 @@ int cachefile_flush (CACHEFILE cf) {
}
}
}
+ assert_cachefile_is_flushed_and_removed(cf);
return 0;
}
-
/* Require that it all be flushed. */
int cachetable_close (CACHETABLE *tp) {
CACHETABLE t=*tp;
diff --git a/newbrt/cachetable.h b/newbrt/cachetable.h
index c6484e277f1..47a53d49986 100644
--- a/newbrt/cachetable.h
+++ b/newbrt/cachetable.h
@@ -18,6 +18,7 @@ typedef struct cachefile *CACHEFILE;
* Note: The cachetable should use a common pool of memory, flushing things across cachetables.
* (The first implementation doesn't)
* If you pin something twice, you must unpin it twice.
+ * n_entries says how many items can fit into the cache table at a time.
*/
int create_cachetable (CACHETABLE */*result*/, int /*n_entries*/);
@@ -56,4 +57,7 @@ int cachefile_close (CACHEFILE*);
int cachefile_fd (CACHEFILE);
+// Useful for debugging
+void cachetable_print_state (CACHETABLE ct);
+
#endif