From 4bc960831d1fe5d3e5db4f595ca517c5d541ec39 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Thu, 10 Aug 2023 08:19:34 +0200 Subject: [PATCH 1/2] Make test plan stabil. --- mysql-test/main/group_min_max_innodb.result | 1 + mysql-test/main/group_min_max_innodb.test | 3 +++ 2 files changed, 4 insertions(+) diff --git a/mysql-test/main/group_min_max_innodb.result b/mysql-test/main/group_min_max_innodb.result index 72116e5d949..5041f6587f9 100644 --- a/mysql-test/main/group_min_max_innodb.result +++ b/mysql-test/main/group_min_max_innodb.result @@ -317,6 +317,7 @@ INSERT INTO t1 VALUES CREATE TABLE t2 (a varchar(4), b varchar(50), PRIMARY KEY (b,a), KEY (a)) ENGINE=InnoDB; INSERT INTO t2 VALUES ('BERM','African Methodist Episcopal'),('AUS','Anglican'),('BERM','Anglican'),('BS','Anglican'),('BS','Baptist'),('BS','Methodist'); +ANALYZE TABLE t1,t2 PERSISTENT FOR ALL; EXPLAIN SELECT t1.a FROM (SELECT a FROM t2 GROUP BY a ORDER BY COUNT(DISTINCT b) LIMIT 1) dt JOIN t1 ON dt.a=t1.b; diff --git a/mysql-test/main/group_min_max_innodb.test b/mysql-test/main/group_min_max_innodb.test index fcecbec41b3..d9639f12031 100644 --- a/mysql-test/main/group_min_max_innodb.test +++ b/mysql-test/main/group_min_max_innodb.test @@ -262,6 +262,9 @@ INSERT INTO t1 VALUES CREATE TABLE t2 (a varchar(4), b varchar(50), PRIMARY KEY (b,a), KEY (a)) ENGINE=InnoDB; INSERT INTO t2 VALUES ('BERM','African Methodist Episcopal'),('AUS','Anglican'),('BERM','Anglican'),('BS','Anglican'),('BS','Baptist'),('BS','Methodist'); +-- disable_result_log +ANALYZE TABLE t1,t2 PERSISTENT FOR ALL; +-- enable_result_log let query= SELECT t1.a From 2aea9387497cecb5668ef605b8f80886f9de812c Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 10 Aug 2023 16:13:32 +0300 Subject: [PATCH 2/2] MDEV-31893 Valgrind reports issues in main.join_cache_notasan This is also related to MDEV-31348 Assertion `last_key_entry >= end_pos' failed in virtual bool JOIN_CACHE_HASHED::put_record() Valgrind exposed a problem with the join_cache for hash joins: =25636== Conditional jump or move depends on uninitialised value(s) ==25636== at 0xA8FF4E: JOIN_CACHE_HASHED::init_hash_table() (sql_join_cache.cc:2901) The reason for this was that avg_record_length contained a random value if one had used SET optimizer_switch='optimize_join_buffer_size=off'. This causes either 'random size' memory to be allocated (up to join_buffer_size) which can increase memory usage or, if avg_record_length is less than the row size, memory overwrites in thd->mem_root, which is bad. Fixed by setting avg_record_length in JOIN_CACHE_HASHED::init() before it's used. There is no test case for MDEV-31893 as valgrind of join_cache_notasan checks that. I added a test case for MDEV-31348. --- include/no_valgrind_without_big.inc | 13 +++++++++++ mysql-test/main/join_cache.result | 16 ++++++++++++++ mysql-test/main/join_cache.test | 19 ++++++++++++++++ mysql-test/main/join_cache_notasan.test | 3 +++ sql/sql_join_cache.cc | 29 +++++++++++++++---------- sql/sql_join_cache.h | 1 + 6 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 include/no_valgrind_without_big.inc diff --git a/include/no_valgrind_without_big.inc b/include/no_valgrind_without_big.inc new file mode 100644 index 00000000000..ea1f2ac91ab --- /dev/null +++ b/include/no_valgrind_without_big.inc @@ -0,0 +1,13 @@ +# include/no_valgrind_without_big.inc +# +# If we are running with Valgrind ($VALGRIND_TEST <> 0) than the resource +# consumption (storage space needed, runtime ...) will be extreme. +# Therefore we require that the option "--big-test" is also set. +# + +if ($VALGRIND_TEST) { + if (!$BIG_TEST) + { + --skip Need "--big-test" when running with Valgrind + } +} diff --git a/mysql-test/main/join_cache.result b/mysql-test/main/join_cache.result index 20980d09001..92c04bb002b 100644 --- a/mysql-test/main/join_cache.result +++ b/mysql-test/main/join_cache.result @@ -6256,3 +6256,19 @@ length(concat(t1.f,t2.f)) DROP TABLE t; set @@optimizer_switch=@org_optimizer_switch; set @@join_buffer_size=@org_join_buffer_size; +# +# MDEV-31348 Assertion `last_key_entry >= end_pos' failed in +# virtual bool JOIN_CACHE_HASHED::put_record() +# +SET JOIN_buffer_size=1; +Warnings: +Warning 1292 Truncated incorrect join_buffer_size value: '1' +SET SESSION JOIN_cache_level=4; +SET SESSION optimizer_switch='optimize_JOIN_buffer_size=OFF'; +SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name); +ERROR HY001: Could not create a join buffer. Please check and adjust the value of the variables 'JOIN_BUFFER_SIZE (128)' and 'JOIN_BUFFER_SPACE_LIMIT (2097152)' +SET JOIN_buffer_size=16384; +SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name); +# +# End of 10.4 tests +# diff --git a/mysql-test/main/join_cache.test b/mysql-test/main/join_cache.test index 43ce3b97ad1..f8723447efe 100644 --- a/mysql-test/main/join_cache.test +++ b/mysql-test/main/join_cache.test @@ -4231,3 +4231,22 @@ SELECT length(concat(t1.f,t2.f)) FROM t t1, t t2; DROP TABLE t; set @@optimizer_switch=@org_optimizer_switch; set @@join_buffer_size=@org_join_buffer_size; + +--echo # +--echo # MDEV-31348 Assertion `last_key_entry >= end_pos' failed in +--echo # virtual bool JOIN_CACHE_HASHED::put_record() +--echo # + +SET JOIN_buffer_size=1; +SET SESSION JOIN_cache_level=4; +SET SESSION optimizer_switch='optimize_JOIN_buffer_size=OFF'; +--error ER_OUTOFMEMORY +SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name); +SET JOIN_buffer_size=16384; +--disable_result_log +SELECT * FROM information_schema.statistics JOIN information_schema.COLUMNS USING (table_name,column_name); +--enable_result_log + +--echo # +--echo # End of 10.4 tests +--echo # diff --git a/mysql-test/main/join_cache_notasan.test b/mysql-test/main/join_cache_notasan.test index cfdfe4eff18..406303ef7b3 100644 --- a/mysql-test/main/join_cache_notasan.test +++ b/mysql-test/main/join_cache_notasan.test @@ -2,7 +2,10 @@ # Tests that should be in join_cache but cannot be run with ASAN --source include/have_64bit.inc +# Disable asan it asan builds crashes when trying to allocate too much memory --source include/not_asan.inc +# Valgrind is useful here, but very slow as lots of memory is allocated +--source include/no_valgrind_without_big.inc --source include/have_innodb.inc --echo # diff --git a/sql/sql_join_cache.cc b/sql/sql_join_cache.cc index 1319fd59a99..f1dd23d9618 100644 --- a/sql/sql_join_cache.cc +++ b/sql/sql_join_cache.cc @@ -800,6 +800,18 @@ size_t JOIN_CACHE::get_min_join_buffer_size() } +size_t JOIN_CACHE::calc_avg_record_length() +{ + size_t len= 0; + for (JOIN_TAB *tab= start_tab; tab != join_tab; + tab= next_linear_tab(join, tab, WITHOUT_BUSH_ROOTS)) + { + len+= tab->get_used_fieldlength(); + } + len+= get_record_max_affix_length(); + return len; +} + /* Get the maximum possible size of the cache join buffer @@ -822,9 +834,9 @@ size_t JOIN_CACHE::get_min_join_buffer_size() 'max_buff_size' in order to use it directly at the next invocations of the function. - RETURN VALUE - The maximum possible size of the join buffer of this cache + The maximum possible size of the join buffer of this cache + avg_record_length is also updated if optimize_buff_size != 0 */ size_t JOIN_CACHE::get_max_join_buffer_size(bool optimize_buff_size, @@ -839,19 +851,13 @@ size_t JOIN_CACHE::get_max_join_buffer_size(bool optimize_buff_size, return max_buff_size= limit_sz; size_t max_sz; - size_t len= 0; + size_t len; double max_records, partial_join_cardinality= (join_tab-1)->get_partial_join_cardinality(); /* Expected join buffer space used for one record */ size_t space_per_record; - for (JOIN_TAB *tab= start_tab; tab != join_tab; - tab= next_linear_tab(join, tab, WITHOUT_BUSH_ROOTS)) - { - len+= tab->get_used_fieldlength(); - } - len+= get_record_max_affix_length(); - avg_record_length= len; + len= avg_record_length= calc_avg_record_length(); len+= get_max_key_addon_space_per_record() + avg_aux_buffer_incr; space_per_record= len; @@ -2786,7 +2792,6 @@ bool JOIN_CACHE_BKAH::save_explain_data(EXPLAIN_BKA_TYPE *explain) int JOIN_CACHE_HASHED::init(bool for_explain) { TABLE_REF *ref= &join_tab->ref; - DBUG_ENTER("JOIN_CACHE_HASHED::init"); hash_table= 0; @@ -2873,6 +2878,8 @@ int JOIN_CACHE_HASHED::init_hash_table() hash_table= 0; key_entries= 0; + avg_record_length= calc_avg_record_length(); + /* Calculate the minimal possible value of size_of_key_ofs greater than 1 */ uint max_size_of_key_ofs= MY_MAX(2, get_size_of_rec_offset()); for (size_of_key_ofs= 2; diff --git a/sql/sql_join_cache.h b/sql/sql_join_cache.h index 8bdce1bd592..b0cfb674ef9 100644 --- a/sql/sql_join_cache.h +++ b/sql/sql_join_cache.h @@ -130,6 +130,7 @@ protected: case 4: int4store(ptr, (uint32) ofs); return; } } + size_t calc_avg_record_length(); /* The maximum total length of the fields stored for a record in the cache.