fix[t:3681] Fix the checkpoint/insert deadlock. Closes #3681.

git-svn-id: file:///svn/toku/tokudb@32706 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt.c

@@ -2930,7 +2930,6 @@ static u_int32_t get_roothash (BRT brt) {
     return rh->fullhash;
 }
 
-// apply a single message, stored in root's buffer(s), to all relevant leaves that are in memory
 static void apply_cmd_to_in_memory_non_root_leaves (
     BRT t, 
     CACHEKEY nodenum, 
@@ -2940,13 +2939,15 @@ static void apply_cmd_to_in_memory_non_root_leaves (
     BRTNODE parent, 
     int parents_childnum,
     uint64_t * workdone_this_childpath_p
-    ) 
-{
-    void *node_v;
-    int r = toku_cachetable_get_and_pin_if_in_memory(t->cf, nodenum, fullhash, &node_v);
-    if (r) { goto exit; }
+    );
 
-    BRTNODE node = node_v;
+static void apply_cmd_to_in_memory_non_root_leaves_starting_at_node (BRT t, 
+								     BRTNODE node,
+								     BRT_MSG cmd, 
+								     BOOL is_root, 
+								     BRTNODE parent, 
+								     int parents_childnum,
+								     uint64_t * workdone_this_childpath_p)  {
     // internal node
     if (node->height>0) {
 	if (brt_msg_applies_once(cmd)) {
@@ -2974,6 +2975,26 @@ static void apply_cmd_to_in_memory_non_root_leaves (
     if (parent) {
 	fixup_child_estimates(parent, parents_childnum, node, FALSE);
     }
+}
+
+// apply a single message, stored in root's buffer(s), to all relevant leaves that are in memory
+static void apply_cmd_to_in_memory_non_root_leaves (
+    BRT t, 
+    CACHEKEY nodenum, 
+    u_int32_t fullhash, 
+    BRT_MSG cmd, 
+    BOOL is_root, 
+    BRTNODE parent, 
+    int parents_childnum,
+    uint64_t * workdone_this_childpath_p
+    )
+{
+    void *node_v;
+    int r = toku_cachetable_get_and_pin_if_in_memory(t->cf, nodenum, fullhash, &node_v);
+    if (r) { goto exit; }
+
+    BRTNODE node = node_v;
+    apply_cmd_to_in_memory_non_root_leaves_starting_at_node(t, node, cmd, is_root, parent, parents_childnum, workdone_this_childpath_p);
     
     toku_unpin_brtnode(t, node);
 exit:
@@ -3021,7 +3042,7 @@ toku_brt_root_put_cmd (BRT brt, BRT_MSG_S * cmd)
     // verify that msn of latest message was captured in root node (push_something_at_root() did not release ydb lock)
     invariant(cmd->msn.msn == node->max_msn_applied_to_node_in_memory.msn);
 
-    apply_cmd_to_in_memory_non_root_leaves(brt, *rootp, fullhash, cmd, TRUE, NULL, -1, NULL);
+    apply_cmd_to_in_memory_non_root_leaves_starting_at_node(brt, node, cmd, TRUE, NULL, -1, NULL);
     if (node->height > 0 && nonleaf_node_is_gorged(node)) {
 	// No need for a loop here.  We only inserted one message, so flushing a single child suffices.
 	flush_some_child(brt, node, TRUE, TRUE,

newbrt/tests/test3681.c

@@ -0,0 +1,76 @@
+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
+#ident "Id:"
+
+// Test for #3681: iibench hangs.  The scenario is
+//  * Thread 1 calls root_put_cmd, get_and_pin_root, 1 holds read lock on the root.
+//  * Thread 2 calls checkpoint, marks the root for checkpoint.
+//  * Thread 2 calls end_checkpoint, tries to write lock the root, sets want_write, and blocks on the rwlock because there is a reader.
+//  * Thread 1 calls apply_cmd_to_in_memory_leaves, calls get_and_pin_if_in_memory, tries to get a read lock on the root node and blocks on the rwlock because there is a write request on the lock.
+
+#include "includes.h"
+#include "checkpoint.h"
+#include "test.h"
+
+CACHETABLE ct;
+BRT t;
+
+static DB * const null_db = 0;
+static TOKUTXN const null_txn = 0;
+
+volatile bool done = false;
+
+static void setup (void) {
+    { int r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);                                  assert(r==0); }
+    char fname[] = __FILE__ "test1.dat";
+    unlink(fname);
+    { int r = toku_open_brt(fname, 1, &t, 1024, ct, null_txn, toku_builtin_compare_fun, null_db);         assert(r==0); }
+}
+
+
+static void finish (void) {
+    { int r = toku_close_brt(t, 0);                                                                       assert(r==0); };
+    { int r = toku_cachetable_close(&ct);                                                    assert(r == 0 && ct == 0); }
+}
+
+static void *starta (void *n) {
+    assert(n==NULL);
+    for (int i=0; i<10000; i++) {
+	DBT k,v;
+	char ks[20], vs[20];
+	snprintf(ks, sizeof(ks), "hello%03d", i);
+	snprintf(vs, sizeof(vs), "there%03d", i);
+	int r = toku_brt_insert(t, toku_fill_dbt(&k, ks, strlen(ks)), toku_fill_dbt(&v, vs, strlen(vs)), null_txn);
+	assert(r==0);
+	usleep(1);
+    }
+    done = true;
+    return NULL;
+}
+static void *startb (void *n) {
+    assert(n==NULL);
+    int count=0;
+    while (!done) {
+	int r = toku_checkpoint(ct, NULL, NULL, NULL, NULL, NULL); assert(r==0);
+	count++;
+    }
+    printf("count=%d\n", count);
+    return NULL;
+}
+
+static void test3681 (void) {
+    setup();
+    toku_pthread_t a,b;
+    { int r; r = toku_pthread_create(&a, NULL, starta, NULL); assert(r==0); }
+    { int r; r = toku_pthread_create(&b, NULL, startb, NULL); assert(r==0); }
+    { int r; void *v; r = toku_pthread_join(a, &v);           assert(r==0); assert(v==NULL); }
+    { int r; void *v; r = toku_pthread_join(b, &v);           assert(r==0); assert(v==NULL);  }
+    finish();
+}
+
+int test_main (int argc, const char *argv[]) {
+    default_parse_args(argc, argv);
+    test3681();
+    return 0;
+}
+