MDEV-31893 Valgrind reports issues in main.join_cache_notasan

This is also related to
MDEV-31348 Assertion `last_key_entry >= end_pos' failed in virtual bool
           JOIN_CACHE_HASHED::put_record()

Valgrind exposed a problem with the join_cache for hash joins:
=25636== Conditional jump or move depends on uninitialised value(s)
==25636== at 0xA8FF4E: JOIN_CACHE_HASHED::init_hash_table()
          (sql_join_cache.cc:2901)

The reason for this was that avg_record_length contained a random value
if one had used SET optimizer_switch='optimize_join_buffer_size=off'.

This causes either 'random size' memory to be allocated (up to
join_buffer_size) which can increase memory usage or, if avg_record_length
is less than the row size, memory overwrites in thd->mem_root, which is
bad.

Fixed by setting avg_record_length in JOIN_CACHE_HASHED::init()
before it's used.

There is no test case for MDEV-31893 as valgrind of join_cache_notasan
checks that.
I added a test case for MDEV-31348.

include/no_valgrind_without_big.inc

@@ -0,0 +1,13 @@
+# include/no_valgrind_without_big.inc
+#
+# If we are running with Valgrind ($VALGRIND_TEST <> 0) than the resource
+# consumption (storage space needed, runtime ...) will be extreme.
+# Therefore we require that the option "--big-test" is also set.
+#
+
+if ($VALGRIND_TEST) {
+  if (!$BIG_TEST)
+  {
+    --skip Need "--big-test" when running with Valgrind
+  }
+}

mysql-test/main/join_cache.result

@@ -6256,3 +6256,19 @@ length(concat(t1.f,t2.f))
 DROP TABLE t;
 set @@optimizer_switch=@org_optimizer_switch;
 set @@join_buffer_size=@org_join_buffer_size;
+#
+# MDEV-31348 Assertion `last_key_entry >= end_pos' failed in
+# virtual bool JOIN_CACHE_HASHED::put_record()
+#
+SET JOIN_buffer_size=1;
+Warnings:
+Warning	1292	Truncated incorrect join_buffer_size value: '1'
+SET SESSION JOIN_cache_level=4;
+SET SESSION optimizer_switch='optimize_JOIN_buffer_size=OFF';
+SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name);
+ERROR HY001: Could not create a join buffer. Please check and adjust the value of the variables 'JOIN_BUFFER_SIZE (128)' and 'JOIN_BUFFER_SPACE_LIMIT (2097152)'
+SET JOIN_buffer_size=16384;
+SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name);
+#
+# End of 10.4 tests
+#

mysql-test/main/join_cache.test

@@ -4231,3 +4231,22 @@ SELECT length(concat(t1.f,t2.f)) FROM t t1, t t2;
 DROP TABLE t;
 set @@optimizer_switch=@org_optimizer_switch;
 set @@join_buffer_size=@org_join_buffer_size;
+
+--echo #
+--echo # MDEV-31348 Assertion `last_key_entry >= end_pos' failed in
+--echo # virtual bool JOIN_CACHE_HASHED::put_record()
+--echo #
+
+SET JOIN_buffer_size=1;
+SET SESSION JOIN_cache_level=4;
+SET SESSION optimizer_switch='optimize_JOIN_buffer_size=OFF';
+--error ER_OUTOFMEMORY
+SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name);
+SET JOIN_buffer_size=16384;
+--disable_result_log
+SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name);
+--enable_result_log
+
+--echo #
+--echo # End of 10.4 tests
+--echo #

mysql-test/main/join_cache_notasan.test

@@ -2,7 +2,10 @@
 # Tests that should be in join_cache but cannot be run with ASAN
 
 --source include/have_64bit.inc
+# Disable asan it asan builds crashes when trying to allocate too much memory
 --source include/not_asan.inc
+# Valgrind is useful here, but very slow as lots of memory is allocated
+--source include/no_valgrind_without_big.inc
 --source include/have_innodb.inc
 
 --echo #

sql/sql_join_cache.cc

@@ -800,6 +800,18 @@ size_t JOIN_CACHE::get_min_join_buffer_size()
 }
 
 
+size_t JOIN_CACHE::calc_avg_record_length()
+{
+  size_t len= 0;
+  for (JOIN_TAB *tab= start_tab; tab != join_tab;
+       tab= next_linear_tab(join, tab, WITHOUT_BUSH_ROOTS))
+  {
+    len+= tab->get_used_fieldlength();
+  }
+  len+= get_record_max_affix_length();
+  return len;
+}
+
 /* 
   Get the maximum possible size of the cache join buffer 
 
@@ -822,9 +834,9 @@ size_t JOIN_CACHE::get_min_join_buffer_size()
     'max_buff_size' in order to use it directly at the next
     invocations of the function.
 
-
   RETURN VALUE
-    The maximum possible size of the join buffer of this cache 
+    The maximum possible size of the join buffer of this cache
+    avg_record_length is also updated if optimize_buff_size != 0
 */
 
 size_t JOIN_CACHE::get_max_join_buffer_size(bool optimize_buff_size,
@@ -839,19 +851,13 @@ size_t JOIN_CACHE::get_max_join_buffer_size(bool optimize_buff_size,
     return max_buff_size= limit_sz;
 
   size_t max_sz;
-  size_t len= 0;
+  size_t len;
   double max_records, partial_join_cardinality=
     (join_tab-1)->get_partial_join_cardinality();
   /* Expected join buffer space used for one record */
   size_t space_per_record;
 
-  for (JOIN_TAB *tab= start_tab; tab != join_tab;
+  len= avg_record_length= calc_avg_record_length();
-       tab= next_linear_tab(join, tab, WITHOUT_BUSH_ROOTS))
-  {
-    len+= tab->get_used_fieldlength();
-  }
-  len+= get_record_max_affix_length();
-  avg_record_length= len;
   len+= get_max_key_addon_space_per_record() + avg_aux_buffer_incr;
   space_per_record= len;
     
@@ -2786,7 +2792,6 @@ bool JOIN_CACHE_BKAH::save_explain_data(EXPLAIN_BKA_TYPE *explain)
 int JOIN_CACHE_HASHED::init(bool for_explain)
 {
   TABLE_REF *ref= &join_tab->ref;
-
   DBUG_ENTER("JOIN_CACHE_HASHED::init");
 
   hash_table= 0;
@@ -2873,6 +2878,8 @@ int JOIN_CACHE_HASHED::init_hash_table()
   hash_table= 0;
   key_entries= 0;
 
+  avg_record_length= calc_avg_record_length();
+
   /* Calculate the minimal possible value of size_of_key_ofs greater than 1 */
   uint max_size_of_key_ofs= MY_MAX(2, get_size_of_rec_offset());  
   for (size_of_key_ofs= 2;

sql/sql_join_cache.h

@@ -130,6 +130,7 @@ protected:
     case 4: int4store(ptr, (uint32) ofs); return;
     }
   }
+  size_t calc_avg_record_length();
   
   /* 
     The maximum total length of the fields stored for a record in the cache.