Bug#12552516 LF_HASH REQUIRES MY_THREAD_INIT()

Before this fix, a thread instrumented for the performance schema,
that would perform file io operations, could crash inside the LF_HASH
implementation, in cases when my_thread_init is not called.

The crash itself has not been reported in 5.5 but similar crashes have
been found in 5.6-based development branches, using LF_HASH for
more instrumentation.

The possibility of a crash in 5.5 is confirmed by code analysis.

The problem is that, when my_thread_init() is not called,
which can happen for threads in storage engines or thirs party code,
my_thread_var is NULL.

Using my_thread_var->stacks_ends_here in mysys/lf_alloc-pin.c is unsafe.

Given that my_thread_var is used:
- only for stacks_ends_here
- only on platform with HAVE_ALLOCA
- only when there is enough room on the stack
and given that the LF_HASH implementation has a fallback
algorythm implemented already when using alloca is not possible,
using my_thread_var->stacks_ends_here is in fact not a strict requirement,
and can be relaxed.

The fix is to:
- test explicitly if my_thread_var is NULL, to account for cases
  when my_thread_init() is not used by the calling thread.
- not use alloca in this case, and rely on the fall back code already in place.
  so that the LF_HASH can be supported even without my_thread_init().

The implementation of mysys/lf_alloc-pin.c has been fixed to support this new usage.
The units tests in unittest/mysys/lf-t.c have been adjusted accordingly.

mysys/lf_alloc-pin.c

@@ -146,6 +146,7 @@ void lf_pinbox_destroy(LF_PINBOX *pinbox)
 */
 LF_PINS *_lf_pinbox_get_pins(LF_PINBOX *pinbox)
 {
+  struct st_my_thread_var *var;
   uint32 pins, next, top_ver;
   LF_PINS *el;
   /*
@@ -188,7 +189,12 @@ LF_PINS *_lf_pinbox_get_pins(LF_PINBOX *pinbox)
   el->link= pins;
   el->purgatory_count= 0;
   el->pinbox= pinbox;
-  el->stack_ends_here= & my_thread_var->stack_ends_here;
+  var= my_thread_var;
+  /*
+    Threads that do not call my_thread_init() should still be
+    able to use the LF_HASH.
+  */
+  el->stack_ends_here= (var ? & var->stack_ends_here : NULL);
   return el;
 }
 
@@ -327,34 +333,37 @@ static int match_pins(LF_PINS *el, void *addr)
 */
 static void _lf_pinbox_real_free(LF_PINS *pins)
 {
-  int npins, alloca_size;
-  void *list, **addr;
+  int npins;
+  void *list;
+  void **addr= NULL;
   void *first= NULL, *last= NULL;
   LF_PINBOX *pinbox= pins->pinbox;
 
   npins= pinbox->pins_in_array+1;
 
 #ifdef HAVE_ALLOCA
-  alloca_size= sizeof(void *)*LF_PINBOX_PINS*npins;
-  /* create a sorted list of pinned addresses, to speed up searches */
-  if (available_stack_size(&pinbox, *pins->stack_ends_here) > alloca_size)
+  if (pins->stack_ends_here != NULL)
   {
-    struct st_harvester hv;
-    addr= (void **) alloca(alloca_size);
-    hv.granary= addr;
-    hv.npins= npins;
-    /* scan the dynarray and accumulate all pinned addresses */
-    _lf_dynarray_iterate(&pinbox->pinarray,
-                         (lf_dynarray_func)harvest_pins, &hv);
+    int alloca_size;
+    alloca_size= sizeof(void *)*LF_PINBOX_PINS*npins;
+    /* create a sorted list of pinned addresses, to speed up searches */
+    if (available_stack_size(&pinbox, *pins->stack_ends_here) > alloca_size)
+    {
+      struct st_harvester hv;
+      addr= (void **) alloca(alloca_size);
+      hv.granary= addr;
+      hv.npins= npins;
+      /* scan the dynarray and accumulate all pinned addresses */
+      _lf_dynarray_iterate(&pinbox->pinarray,
+                           (lf_dynarray_func)harvest_pins, &hv);
 
-    npins= hv.granary-addr;
-    /* and sort them */
-    if (npins)
-      qsort(addr, npins, sizeof(void *), (qsort_cmp)ptr_cmp);
+      npins= hv.granary-addr;
+      /* and sort them */
+      if (npins)
+        qsort(addr, npins, sizeof(void *), (qsort_cmp)ptr_cmp);
+    }
   }
-  else
 #endif
-    addr= 0;
 
   list= pins->purgatory;
   pins->purgatory= 0;

unittest/mysys/lf-t.c

@@ -27,6 +27,8 @@ int32 inserts= 0, N;
 LF_ALLOCATOR lf_allocator;
 LF_HASH lf_hash;
 
+int with_my_thread_init=0;
+
 /*
   pin allocator - alloc and release an element in a loop
 */
@@ -36,7 +38,8 @@ pthread_handler_t test_lf_pinbox(void *arg)
   int32 x= 0;
   LF_PINS *pins;
 
-  my_thread_init();
+  if (with_my_thread_init)
+    my_thread_init();
 
   pins= lf_pinbox_get_pins(&lf_allocator.pinbox);
 
@@ -49,7 +52,10 @@ pthread_handler_t test_lf_pinbox(void *arg)
   pthread_mutex_lock(&mutex);
   if (!--running_threads) pthread_cond_signal(&cond);
   pthread_mutex_unlock(&mutex);
-  my_thread_end();
+
+  if (with_my_thread_init)
+    my_thread_end();
+
   return 0;
 }
 
@@ -68,7 +74,8 @@ pthread_handler_t test_lf_alloc(void *arg)
   int32 x,y= 0;
   LF_PINS *pins;
 
-  my_thread_init();
+  if (with_my_thread_init)
+    my_thread_init();
 
   pins= lf_alloc_get_pins(&lf_allocator);
 
@@ -101,7 +108,9 @@ pthread_handler_t test_lf_alloc(void *arg)
   }
   if (!--running_threads) pthread_cond_signal(&cond);
   pthread_mutex_unlock(&mutex);
-  my_thread_end();
+
+  if (with_my_thread_init)
+    my_thread_end();
   return 0;
 }
 
@@ -112,7 +121,8 @@ pthread_handler_t test_lf_hash(void *arg)
   int32 x,y,z,sum= 0, ins= 0;
   LF_PINS *pins;
 
-  my_thread_init();
+  if (with_my_thread_init)
+    my_thread_init();
 
   pins= lf_hash_get_pins(&lf_hash);
 
@@ -152,14 +162,15 @@ pthread_handler_t test_lf_hash(void *arg)
   }
   if (!--running_threads) pthread_cond_signal(&cond);
   pthread_mutex_unlock(&mutex);
-  my_thread_end();
+  if (with_my_thread_init)
+    my_thread_end();
   return 0;
 }
 
 
 void do_tests()
 {
-  plan(4);
+  plan(7);
 
   lf_alloc_init(&lf_allocator, sizeof(TLA), offsetof(TLA, not_used));
   lf_hash_init(&lf_hash, sizeof(int), LF_HASH_UNIQUE, 0, sizeof(int), 0,
@@ -168,9 +179,15 @@ void do_tests()
   bad= my_atomic_initialize();
   ok(!bad, "my_atomic_initialize() returned %d", bad);
 
-  test_concurrently("lf_pinbox", test_lf_pinbox, N= THREADS, CYCLES);
-  test_concurrently("lf_alloc",  test_lf_alloc,  N= THREADS, CYCLES);
-  test_concurrently("lf_hash",   test_lf_hash,   N= THREADS, CYCLES/10);
+  with_my_thread_init= 1;
+  test_concurrently("lf_pinbox (with my_thread_init)", test_lf_pinbox, N= THREADS, CYCLES);
+  test_concurrently("lf_alloc (with my_thread_init)",  test_lf_alloc,  N= THREADS, CYCLES);
+  test_concurrently("lf_hash (with my_thread_init)",   test_lf_hash,   N= THREADS, CYCLES/10);
+
+  with_my_thread_init= 0;
+  test_concurrently("lf_pinbox (without my_thread_init)", test_lf_pinbox, N= THREADS, CYCLES);
+  test_concurrently("lf_alloc (without my_thread_init)",  test_lf_alloc,  N= THREADS, CYCLES);
+  test_concurrently("lf_hash (without my_thread_init)",   test_lf_hash,   N= THREADS, CYCLES/10);
 
   lf_hash_destroy(&lf_hash);
   lf_alloc_destroy(&lf_allocator);