diff --git a/mysql-test/main/debug_sync.test b/mysql-test/main/debug_sync.test index 1dc3dc9c71e..6e75ba9624c 100644 --- a/mysql-test/main/debug_sync.test +++ b/mysql-test/main/debug_sync.test @@ -231,15 +231,12 @@ SHOW VARIABLES LIKE 'DEBUG_SYNC'; # immediately after setting of the DEBUG_SYNC variable. # So it is executed before the SET statement ends. # -# NOTE: There is only one global signal (say "signal post" or "flag mast"). -# A SIGNAL action writes its signal into it ("sets a flag"). -# The signal persists until explicitly overwritten. +# NOTE: There can be multiple active signals at the same time. +# A SIGNAL action appends its signal into signals set. +# The signal persists until waited on. # To avoid confusion for later tests, it is recommended to clear -# the signal by signalling "empty" ("setting the 'empty' flag"): -# SET DEBUG_SYNC= 'now SIGNAL empty'; -# Preferably you can reset the whole facility with: +# the signal set by running # SET DEBUG_SYNC= 'RESET'; -# The signal is then '' (really empty) which connot be done otherwise. # # diff --git a/mysql-test/main/empty_table.result b/mysql-test/main/empty_table.result index 2bca3e792fa..90aec2eda3b 100644 --- a/mysql-test/main/empty_table.result +++ b/mysql-test/main/empty_table.result @@ -16,3 +16,61 @@ ERROR 42S02: Table 'test.t2' doesn't exist show status like "Empty_queries"; Variable_name Value Empty_queries 2 +# End of 4.1 tests +# +# MDEV-30333 Wrong result with not_null_range_scan and LEFT JOIN with empty table +# +set @save_optimizer_switch=@@optimizer_switch; +CREATE TABLE t1 (a INT, b INT) ENGINE=MyISAM; +INSERT INTO t1 (b) VALUES (1),(2); +CREATE TABLE t2 (c INT) ENGINE=MyISAM; +SET optimizer_switch= 'not_null_range_scan=off'; +explain extended SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t2 system NULL NULL NULL NULL 0 0.00 Const row not found +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 100.00 Using where; Using filesort +Warnings: +Note 1003 select `test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` is null order by `test`.`t1`.`b` +SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +b +1 +2 +SET optimizer_switch = 'not_null_range_scan=on'; +explain extended SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t2 system NULL NULL NULL NULL 0 0.00 Const row not found +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 100.00 Using where; Using filesort +Warnings: +Note 1003 select `test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` is null order by `test`.`t1`.`b` +SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +b +1 +2 +flush tables; +SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +b +1 +2 +drop table t1,t2; +# Second test in MDEV-30333 +CREATE TABLE t1 (a int, b varchar(10)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (69,'foo'),(71,'bar'); +CREATE TABLE t2 (c int) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(2); +CREATE TABLE t3 (d int, e int, KEY(e)) ENGINE=MyISAM; +SELECT * FROM t1 LEFT JOIN t2 LEFT JOIN t3 ON t3.e = t3.d ON 1; +a b c d e +69 foo 1 NULL NULL +71 bar 1 NULL NULL +69 foo 2 NULL NULL +71 bar 2 NULL NULL +SET optimizer_switch = 'not_null_range_scan=on'; +SELECT * FROM t1 LEFT JOIN t2 LEFT JOIN t3 ON t3.e = t3.d ON 1; +a b c d e +69 foo 1 NULL NULL +71 bar 1 NULL NULL +69 foo 2 NULL NULL +71 bar 2 NULL NULL +DROP TABLE t1, t2, t3; +set @@optimizer_switch=@save_optimizer_switch; +End of 10.5 tests diff --git a/mysql-test/main/empty_table.test b/mysql-test/main/empty_table.test index 754671868ba..a17b0c897d5 100644 --- a/mysql-test/main/empty_table.test +++ b/mysql-test/main/empty_table.test @@ -21,4 +21,38 @@ drop table t1; select * from t2; show status like "Empty_queries"; -# End of 4.1 tests +--echo # End of 4.1 tests + +--echo # +--echo # MDEV-30333 Wrong result with not_null_range_scan and LEFT JOIN with empty table +--echo # + +set @save_optimizer_switch=@@optimizer_switch; +CREATE TABLE t1 (a INT, b INT) ENGINE=MyISAM; +INSERT INTO t1 (b) VALUES (1),(2); +CREATE TABLE t2 (c INT) ENGINE=MyISAM; +SET optimizer_switch= 'not_null_range_scan=off'; # Default +explain extended SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +SET optimizer_switch = 'not_null_range_scan=on'; +explain extended SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +flush tables; +SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b; +drop table t1,t2; + +--echo # Second test in MDEV-30333 + +CREATE TABLE t1 (a int, b varchar(10)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (69,'foo'),(71,'bar'); +CREATE TABLE t2 (c int) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(2); +CREATE TABLE t3 (d int, e int, KEY(e)) ENGINE=MyISAM; +SELECT * FROM t1 LEFT JOIN t2 LEFT JOIN t3 ON t3.e = t3.d ON 1; +SET optimizer_switch = 'not_null_range_scan=on'; +SELECT * FROM t1 LEFT JOIN t2 LEFT JOIN t3 ON t3.e = t3.d ON 1; +DROP TABLE t1, t2, t3; +set @@optimizer_switch=@save_optimizer_switch; + +--echo End of 10.5 tests + diff --git a/mysql-test/main/fetch_first.result b/mysql-test/main/fetch_first.result index 69a0336d722..df182381d1c 100644 --- a/mysql-test/main/fetch_first.result +++ b/mysql-test/main/fetch_first.result @@ -1378,3 +1378,31 @@ a bar foo DROP TABLE t; +# +# MDEV-30324: Wrong result upon SELECT DISTINCT .. WITH TIES using index +# +CREATE TABLE t1 (a int, b char(3), KEY (a)); +INSERT INTO t1 VALUES (2,'foo'),(3,'bar'),(3,'bar'),(3,'zzz'); +EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 1 ROWS WITH TIES; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL a 5 NULL 1 Using temporary +SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 1 ROWS WITH TIES; +a b +2 foo +EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 2 ROWS WITH TIES; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL a 5 NULL 2 Using temporary +SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 2 ROWS WITH TIES; +a b +2 foo +3 bar +3 zzz +EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 3 ROWS WITH TIES; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 Using temporary; Using filesort +SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 3 ROWS WITH TIES; +a b +2 foo +3 bar +3 zzz +DROP TABLE t1; diff --git a/mysql-test/main/fetch_first.test b/mysql-test/main/fetch_first.test index 62cdd717f81..98bbf1ca06b 100644 --- a/mysql-test/main/fetch_first.test +++ b/mysql-test/main/fetch_first.test @@ -1059,3 +1059,22 @@ SELECT a FROM t ORDER BY a FETCH FIRST 2 ROWS WITH TIES; # Cleanup DROP TABLE t; + +--echo # +--echo # MDEV-30324: Wrong result upon SELECT DISTINCT .. WITH TIES using index +--echo # +CREATE TABLE t1 (a int, b char(3), KEY (a)); +INSERT INTO t1 VALUES (2,'foo'),(3,'bar'),(3,'bar'),(3,'zzz'); + +EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 1 ROWS WITH TIES; +--sorted_result +SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 1 ROWS WITH TIES; +EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 2 ROWS WITH TIES; +--sorted_result +SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 2 ROWS WITH TIES; +EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 3 ROWS WITH TIES; +--sorted_result +SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 3 ROWS WITH TIES; + +# Cleanup +DROP TABLE t1; diff --git a/mysql-test/main/func_json.result b/mysql-test/main/func_json.result index 45a75ab082c..c963fdc966d 100644 --- a/mysql-test/main/func_json.result +++ b/mysql-test/main/func_json.result @@ -1623,6 +1623,21 @@ id doc {"$oid":"611c0a463b150154132f6636"} { "_id" : { "$oid" : "611c0a463b150154132f6636" }, "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : 1.0 } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } DROP TABLE arrNestTest; # +# MDEV-30412 JSON_OBJECTAGG doesn't escape double quote in key +# +SELECT JSON_OBJECTAGG('"', 1); +JSON_OBJECTAGG('"', 1) +{"\"":1} +SELECT JSON_OBJECTAGG('\"', 1); +JSON_OBJECTAGG('\"', 1) +{"\"":1} +SELECT JSON_OBJECTAGG('\\', 1); +JSON_OBJECTAGG('\\', 1) +{"\\":1} +# +# End of 10.5 tests +# +# # MDEV-26054 Server crashes in Item_func_json_arrayagg::get_str_from_field # CREATE TABLE t (a VARCHAR(8)); @@ -1649,5 +1664,5 @@ JSON_INSERT(JSON_OBJECT(l1, l2, l3, l4), '$.k3', 'v3') JSON_SET(JSON_OBJECT(l1, {"k1": "v1", "k2": "v2", "k3": "v3"} {"k1": "v1", "k2": "new v2"} {"k1": "v1", "k2": "new v2"} DROP TABLE t; # -# End of 10.5 tests +# End of 10.6 tests # diff --git a/mysql-test/main/func_json.test b/mysql-test/main/func_json.test index cbc6156a8c7..464e7d762d8 100644 --- a/mysql-test/main/func_json.test +++ b/mysql-test/main/func_json.test @@ -1067,6 +1067,18 @@ INSERT INTO test.arrNestTest (doc) VALUES ('{ "_id" : { "$oid" : "611c0a463b1501 SELECT * FROM arrNestTest; DROP TABLE arrNestTest; +--echo # +--echo # MDEV-30412 JSON_OBJECTAGG doesn't escape double quote in key +--echo # + +SELECT JSON_OBJECTAGG('"', 1); +SELECT JSON_OBJECTAGG('\"', 1); +SELECT JSON_OBJECTAGG('\\', 1); + +--echo # +--echo # End of 10.5 tests +--echo # + --echo # --echo # MDEV-26054 Server crashes in Item_func_json_arrayagg::get_str_from_field --echo # @@ -1078,7 +1090,6 @@ SELECT JSON_ARRAYAGG(a) AS f FROM v; DROP VIEW v; DROP TABLE t; - --echo # --echo # MDEV-29264 JSON functions overflow error based ON LONGTEXT field --echo # @@ -1090,6 +1101,5 @@ SELECT JSON_INSERT(JSON_OBJECT(l1, l2, l3, l4), '$.k3', 'v3'),JSON_SET(JSON_OBJE DROP TABLE t; --echo # ---echo # End of 10.5 tests +--echo # End of 10.6 tests --echo # - diff --git a/mysql-test/suite/innodb/r/alter_copy.result b/mysql-test/suite/innodb/r/alter_copy.result index 72ae28e9652..8c9e5966b2e 100644 --- a/mysql-test/suite/innodb/r/alter_copy.result +++ b/mysql-test/suite/innodb/r/alter_copy.result @@ -51,7 +51,7 @@ ADD INDEX(a,b,d), ADD INDEX(a,d,b), ADD INDEX(b,c,d), ADD INDEX(b,d,c), ALGORITHM=COPY; connection default; SET DEBUG_SYNC='now WAIT_FOR hung'; -# restart: --innodb-force-recovery=3 +# restart: --innodb-force-recovery=3 --debug_dbug=+d,recv_ran_out_of_buffer disconnect hang; FTS_INDEX_1.ibd FTS_INDEX_2.ibd diff --git a/mysql-test/suite/innodb/r/recovery_memory.result b/mysql-test/suite/innodb/r/recovery_memory.result index 4fa31009130..6faea097616 100644 --- a/mysql-test/suite/innodb/r/recovery_memory.result +++ b/mysql-test/suite/innodb/r/recovery_memory.result @@ -1,3 +1,7 @@ +call mtr.add_suppression("InnoDB: The change buffer is corrupted"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted at srv0start.cc"); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); CREATE TABLE t1(c TEXT, KEY(c(3072)))ENGINE=InnoDB; CREATE PROCEDURE dorepeat() LOOP @@ -10,3 +14,19 @@ CALL dorepeat(); connection default; # restart: --innodb_buffer_pool_size=5242880 DROP TABLE t1; +DROP PROCEDURE dorepeat; +# +# MDEV-30552 InnoDB recovery crashes when error +# handling scenario +# +SET DEBUG_DBUG="+d,ib_log_checkpoint_avoid_hard"; +CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; +INSERT INTO t1 SELECT * FROM seq_1_to_65536; +# restart: --innodb_buffer_pool_size=5242880 --debug_dbug=+d,ibuf_init_corrupt +# restart +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `f1` int(11) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/alter_copy.test b/mysql-test/suite/innodb/t/alter_copy.test index 90f2171d10b..b62f812f4b7 100644 --- a/mysql-test/suite/innodb/t/alter_copy.test +++ b/mysql-test/suite/innodb/t/alter_copy.test @@ -57,7 +57,7 @@ ALTER TABLE t ADD INDEX(b,c,d,a),ADD INDEX(b,c,a,d),ADD INDEX(b,a,c,d),ADD INDEX connection default; SET DEBUG_SYNC='now WAIT_FOR hung'; let $shutdown_timeout=0; ---let $restart_parameters= --innodb-force-recovery=3 +--let $restart_parameters= --innodb-force-recovery=3 --debug_dbug="+d,recv_ran_out_of_buffer" --source include/restart_mysqld.inc disconnect hang; let $shutdown_timeout=; diff --git a/mysql-test/suite/innodb/t/recovery_memory.test b/mysql-test/suite/innodb/t/recovery_memory.test index d9afd52c499..e723ba25d36 100644 --- a/mysql-test/suite/innodb/t/recovery_memory.test +++ b/mysql-test/suite/innodb/t/recovery_memory.test @@ -1,5 +1,10 @@ --source include/have_innodb.inc --source include/big_test.inc +--source include/have_sequence.inc +call mtr.add_suppression("InnoDB: The change buffer is corrupted"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted at srv0start.cc"); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); CREATE TABLE t1(c TEXT, KEY(c(3072)))ENGINE=InnoDB; DELIMITER |; @@ -19,3 +24,19 @@ let $shutdown_timeout=0; let $restart_parameters=--innodb_buffer_pool_size=5242880; --source include/restart_mysqld.inc DROP TABLE t1; +DROP PROCEDURE dorepeat; + +--echo # +--echo # MDEV-30552 InnoDB recovery crashes when error +--echo # handling scenario +--echo # +SET DEBUG_DBUG="+d,ib_log_checkpoint_avoid_hard"; +CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; +INSERT INTO t1 SELECT * FROM seq_1_to_65536; +let $shutdown_timeout=0; +let $restart_parameters=--innodb_buffer_pool_size=5242880 --debug_dbug="+d,ibuf_init_corrupt"; +--source include/restart_mysqld.inc +let $restart_parameters=; +--source include/restart_mysqld.inc +SHOW CREATE TABLE t1; +DROP TABLE t1; diff --git a/sql/debug_sync.cc b/sql/debug_sync.cc index aa6956060b4..eac111d32d7 100644 --- a/sql/debug_sync.cc +++ b/sql/debug_sync.cc @@ -1344,7 +1344,8 @@ static bool debug_sync_eval_action(THD *thd, char *action_str, char *action_end) /* Try NO_CLEAR_EVENT. */ - if (!my_strcasecmp(system_charset_info, token, "NO_CLEAR_EVENT")) { + if (!my_strcasecmp(system_charset_info, token, "NO_CLEAR_EVENT")) + { action->clear_event= false; /* Get next token. If none follows, set action. */ if (!(ptr = debug_sync_token(&token, &token_length, ptr, action_end))) goto set_action; @@ -1634,8 +1635,8 @@ static void debug_sync_execute(THD *thd, st_debug_sync_action *action) /* - Wait until global signal string matches the wait_for string. - Interrupt when thread or query is killed or facility disabled. + Wait until the signal set contains the wait_for string. + Interrupt when thread or query is killed or facility is disabled. The facility can become disabled when some thread cannot get the required dynamic memory allocated. */ diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index ba6922a65ee..d32d2d30d4a 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -4143,7 +4143,7 @@ bool Item_func_json_objectagg::add() result.append(STRING_WITH_LEN(", ")); result.append('"'); - result.append(*key); + st_append_escaped(&result,key); result.append(STRING_WITH_LEN("\":")); buf.length(0); diff --git a/sql/sql_hset.h b/sql/sql_hset.h index c9ffc8a560b..41573fb5f03 100644 --- a/sql/sql_hset.h +++ b/sql/sql_hset.h @@ -40,7 +40,7 @@ public: Hash_set(PSI_memory_key psi_key, CHARSET_INFO *charset, ulong default_array_elements, size_t key_offset, size_t key_length, my_hash_get_key get_key, - void (*free_element)(void*),uint flags) + void (*free_element)(void*), uint flags) { my_hash_init(psi_key, &m_hash, charset, default_array_elements, key_offset, key_length, get_key, free_element, flags); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 9ab87f8134e..214167af3ec 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -4166,7 +4166,7 @@ JOIN::optimize_distinct() } /* Optimize "select distinct b from t1 order by key_part_1 limit #" */ - if (order && skip_sort_order) + if (order && skip_sort_order && !unit->lim.is_with_ties()) { /* Should already have been optimized away */ DBUG_ASSERT(ordered_index_usage == ordered_index_order_by); @@ -30088,7 +30088,6 @@ void JOIN::make_notnull_conds_for_range_scans() { DBUG_ENTER("JOIN::make_notnull_conds_for_range_scans"); - if (impossible_where || !optimizer_flag(thd, OPTIMIZER_SWITCH_NOT_NULL_RANGE_SCAN)) { @@ -30168,7 +30167,6 @@ bool build_notnull_conds_for_range_scans(JOIN *join, Item *cond, table_map allowed) { THD *thd= join->thd; - DBUG_ENTER("build_notnull_conds_for_range_scans"); for (JOIN_TAB *s= join->join_tab; @@ -30176,13 +30174,13 @@ bool build_notnull_conds_for_range_scans(JOIN *join, Item *cond, { /* Clear all needed bitmaps to mark found fields */ if ((allowed & s->table->map) && - !(s->table->map && join->const_table_map)) + !(s->table->map & join->const_table_map)) bitmap_clear_all(&s->table->tmp_set); } /* Find all null-rejected fields assuming that cond is null-rejected and - only formulas over tables from 'allowed' are to be taken into account + only formulas over tables from 'allowed' are to be taken into account */ if (cond->find_not_null_fields(allowed)) DBUG_RETURN(true); diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index f87888d90da..287a2681bd5 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1940,28 +1940,22 @@ static void buf_relocate(buf_page_t *bpage, buf_page_t *dpage) buf_pool.page_hash.replace(chain, bpage, dpage); } -/** Register a watch for a page identifier. The caller must hold an -exclusive page hash latch. The *hash_lock may be released, -relocated, and reacquired. -@param id page identifier -@param chain hash table chain with exclusively held page_hash -@return a buffer pool block corresponding to id -@retval nullptr if the block was not present, and a watch was installed */ -inline buf_page_t *buf_pool_t::watch_set(const page_id_t id, - buf_pool_t::hash_chain &chain) +buf_page_t *buf_pool_t::watch_set(const page_id_t id, + buf_pool_t::hash_chain &chain) { ut_ad(&chain == &page_hash.cell_get(id.fold())); - ut_ad(page_hash.lock_get(chain).is_write_locked()); + page_hash.lock_get(chain).lock(); -retry: - if (buf_page_t *bpage= page_hash.get(id, chain)) + buf_page_t *bpage= page_hash.get(id, chain); + + if (bpage) { - if (!watch_is_sentinel(*bpage)) - /* The page was loaded meanwhile. */ - return bpage; - /* Add to an existing watch. */ +got_block: bpage->fix(); - return nullptr; + if (watch_is_sentinel(*bpage)) + bpage= nullptr; + page_hash.lock_get(chain).unlock(); + return bpage; } page_hash.lock_get(chain).unlock(); @@ -1990,25 +1984,23 @@ retry: w->set_state(buf_page_t::UNFIXED + 1); w->id_= id; - buf_page_t *bpage= page_hash.get(id, chain); + page_hash.lock_get(chain).lock(); + bpage= page_hash.get(id, chain); if (UNIV_LIKELY_NULL(bpage)) { w->set_state(buf_page_t::NOT_USED); - page_hash.lock_get(chain).lock(); mysql_mutex_unlock(&mutex); - goto retry; + goto got_block; } - page_hash.lock_get(chain).lock(); ut_ad(w->state() == buf_page_t::UNFIXED + 1); buf_pool.page_hash.append(chain, w); mysql_mutex_unlock(&mutex); + page_hash.lock_get(chain).unlock(); return nullptr; } ut_error; - mysql_mutex_unlock(&mutex); - return nullptr; } /** Stop watching whether a page has been read in. @@ -2456,16 +2448,13 @@ loop: case BUF_PEEK_IF_IN_POOL: return nullptr; case BUF_GET_IF_IN_POOL_OR_WATCH: - /* We cannot easily use a memory transaction here. */ - hash_lock.lock(); + /* Buffer-fixing inside watch_set() will prevent eviction */ block = reinterpret_cast (buf_pool.watch_set(page_id, chain)); - /* buffer-fixing will prevent eviction */ - state = block ? block->page.fix() : 0; - hash_lock.unlock(); if (block) { - goto got_block; + state = block->page.state(); + goto got_block_fixed; } return nullptr; @@ -2504,6 +2493,7 @@ loop: got_block: ut_ad(!block->page.in_zip_hash); state++; +got_block_fixed: ut_ad(state > buf_page_t::FREED); if (state > buf_page_t::READ_FIX && state < buf_page_t::WRITE_FIX) { @@ -2713,24 +2703,19 @@ re_evict: const bool evicted = buf_LRU_free_page(&block->page, true); space->release(); + if (!evicted) { + block->fix(); + } + + mysql_mutex_unlock(&buf_pool.mutex); + if (evicted) { - page_hash_latch& hash_lock - = buf_pool.page_hash.lock_get(chain); - hash_lock.lock(); - mysql_mutex_unlock(&buf_pool.mutex); - /* We may set the watch, as it would have - been set if the page were not in the - buffer pool in the first place. */ - block= reinterpret_cast( - mode == BUF_GET_IF_IN_POOL_OR_WATCH - ? buf_pool.watch_set(page_id, chain) - : buf_pool.page_hash.get(page_id, chain)); - hash_lock.unlock(); + if (mode == BUF_GET_IF_IN_POOL_OR_WATCH) { + buf_pool.watch_set(page_id, chain); + } return(NULL); } - block->fix(); - mysql_mutex_unlock(&buf_pool.mutex); buf_flush_sync(); state = block->page.state(); @@ -3438,8 +3423,7 @@ or decrypt/decompress just failed. @retval DB_SUCCESS if page has been read and is not corrupted @retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted @retval DB_DECRYPTION_FAILED if page post encryption checksum matches but -after decryption normal page checksum does not match. -@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ +after decryption normal page checksum does not match. */ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, const fil_node_t &node) { @@ -3496,7 +3480,8 @@ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, @param node data file @return whether the operation succeeded @retval DB_PAGE_CORRUPTED if the checksum fails -@retval DB_DECRYPTION_FAILED if the page cannot be decrypted */ +@retval DB_DECRYPTION_FAILED if the page cannot be decrypted +@retval DB_FAIL if the page contains the wrong ID */ dberr_t buf_page_t::read_complete(const fil_node_t &node) { const page_id_t expected_id{id()}; diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 20f7544466f..6d652e72ad8 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -397,6 +397,11 @@ buf_block_t *buf_LRU_get_free_block(bool have_mutex) mysql_mutex_assert_owner(&buf_pool.mutex); goto got_mutex; } + DBUG_EXECUTE_IF("recv_ran_out_of_buffer", + if (recv_recovery_is_on() + && recv_sys.apply_log_recs) { + goto flush_lru; + }); mysql_mutex_lock(&buf_pool.mutex); got_mutex: buf_LRU_check_size_of_non_data_objects(); @@ -480,7 +485,9 @@ not_found: removing the block from buf_pool.page_hash and buf_pool.LRU is fairly involved (particularly in case of ROW_FORMAT=COMPRESSED pages). We can do that in a separate patch sometime in future. */ - +#ifndef DBUG_OFF +flush_lru: +#endif if (!buf_flush_LRU(innodb_lru_flush_size)) { MONITOR_INC(MONITOR_LRU_SINGLE_FLUSH_FAILURE_COUNT); ++flush_failures; diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 4a2ac88d97c..2a1375f27db 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -331,6 +331,9 @@ nothing_read: /* The i/o was already completed in space->io() */ *err = bpage->read_complete(*fio.node); space->release(); + if (*err == DB_FAIL) { + *err = DB_PAGE_CORRUPTED; + } } return true; diff --git a/storage/innobase/dict/dict0defrag_bg.cc b/storage/innobase/dict/dict0defrag_bg.cc index ea2914e52dc..bec6da8e6af 100644 --- a/storage/innobase/dict/dict0defrag_bg.cc +++ b/storage/innobase/dict/dict0defrag_bg.cc @@ -314,7 +314,7 @@ btr_get_size_and_reserved( return ULINT_UNDEFINED; } - mtr->s_lock_space(index->table->space); + mtr->x_lock_space(index->table->space); ulint n = fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_LEAF + root->page.frame, used, mtr); diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 3c7de3cd040..39f5943d5a4 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1490,7 +1490,7 @@ invalid: goto invalid; } - mtr.s_lock_space(index->table->space); + mtr.x_lock_space(index->table->space); ulint dummy, size; index->stat_index_size @@ -2696,7 +2696,7 @@ empty_index: } uint16_t root_level = btr_page_get_level(root->page.frame); - mtr.s_lock_space(index->table->space); + mtr.x_lock_space(index->table->space); ulint dummy, size; result.index_size = fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_LEAF diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 067aca49b4c..4918cefb947 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -1678,8 +1678,9 @@ fil_crypt_get_page_throttle( return NULL; } - if (DB_SUCCESS_LOCKED_REC - != fseg_page_is_allocated(space, state->offset)) { + if (offset % (zip_size ? zip_size : srv_page_size) + && DB_SUCCESS_LOCKED_REC + != fseg_page_is_allocated(space, offset)) { /* page is already freed */ return NULL; } diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index d63febf01f1..c6044b201fe 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -2650,7 +2650,7 @@ dberr_t fseg_page_is_allocated(fil_space_t *space, unsigned page) mtr.start(); if (!space->is_owner()) - mtr.s_lock_space(space); + mtr.x_lock_space(space); if (page >= space->free_limit || page >= space->size_in_header); else if (const buf_block_t *b= diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index e55835425be..fec18439c84 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -426,6 +426,10 @@ err_exit: goto err_exit; } + DBUG_EXECUTE_IF("ibuf_init_corrupt", + err = DB_CORRUPTION; + goto err_exit;); + if (page_is_comp(root) || fil_page_get_type(root) != FIL_PAGE_INDEX || btr_page_get_index_id(root) != DICT_IBUF_ID_MIN) { err = DB_CORRUPTION; diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 7e822f2648b..c02180ad9ee 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -769,7 +769,8 @@ public: @param node data file @return whether the operation succeeded @retval DB_PAGE_CORRUPTED if the checksum fails - @retval DB_DECRYPTION_FAILED if the page cannot be decrypted */ + @retval DB_DECRYPTION_FAILED if the page cannot be decrypted + @retval DB_FAIL if the page contains the wrong ID */ dberr_t read_complete(const fil_node_t &node); /** Note that a block is no longer dirty, while not removing @@ -1465,14 +1466,12 @@ public: return !watch_is_sentinel(*page_hash.get(id, chain)); } - /** Register a watch for a page identifier. The caller must hold an - exclusive page hash latch. The *hash_lock may be released, - relocated, and reacquired. + /** Register a watch for a page identifier. @param id page identifier - @param chain hash table chain with exclusively held page_hash - @return a buffer pool block corresponding to id - @retval nullptr if the block was not present, and a watch was installed */ - inline buf_page_t *watch_set(const page_id_t id, hash_chain &chain); + @param chain page_hash.cell_get(id.fold()) + @return a buffer page corresponding to id + @retval nullptr if the block was not present in page_hash */ + buf_page_t *watch_set(const page_id_t id, hash_chain &chain); /** Stop watching whether a page has been read in. watch_set(id) must have returned nullptr before. diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index c64e663509a..e787d81e8c2 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -243,9 +243,15 @@ private: @param lsn log sequence number of the shrink operation */ inline void trim(const page_id_t page_id, lsn_t lsn); - /** Truncated undo tablespace size for which truncate has been logged - (indexed by page_id_t::space() - srv_undo_space_id_start), or 0 */ - unsigned truncated_undo_spaces[127]; + /** Undo tablespaces for which truncate has been logged + (indexed by page_id_t::space() - srv_undo_space_id_start) */ + struct trunc + { + /** log sequence number of FILE_CREATE, or 0 if none */ + lsn_t lsn; + /** truncated size of the tablespace, or 0 if not truncated */ + unsigned pages; + } truncated_undo_spaces[127]; public: /** The contents of the doublewrite buffer */ diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 299f658e98a..ccb174da596 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -283,17 +283,6 @@ struct mtr_t { memo_push(lock, MTR_MEMO_SX_LOCK); } - /** Acquire a tablespace S-latch. - @param space tablespace */ - void s_lock_space(fil_space_t *space) - { - ut_ad(space->purpose == FIL_TYPE_TEMPORARY || - space->purpose == FIL_TYPE_IMPORT || - space->purpose == FIL_TYPE_TABLESPACE); - memo_push(space, MTR_MEMO_SPACE_S_LOCK); - space->s_lock(); - } - /** Acquire an exclusive tablespace latch. @param space tablespace */ void x_lock_space(fil_space_t *space); @@ -355,9 +344,8 @@ public: /** Check if we are holding tablespace latch @param space tablespace to search for - @param shared whether to look for shared latch, instead of exclusive @return whether space.latch is being held */ - bool memo_contains(const fil_space_t& space, bool shared= false) const + bool memo_contains(const fil_space_t& space) const MY_ATTRIBUTE((warn_unused_result)); #ifdef UNIV_DEBUG /** Check if we are holding an rw-latch in this mini-transaction @@ -410,7 +398,7 @@ public: break; case MTR_MEMO_MODIFY: case MTR_MEMO_S_LOCK: case MTR_MEMO_X_LOCK: case MTR_MEMO_SX_LOCK: - case MTR_MEMO_SPACE_X_LOCK: case MTR_MEMO_SPACE_S_LOCK: + case MTR_MEMO_SPACE_X_LOCK: ut_ad("invalid type" == 0); } #endif diff --git a/storage/innobase/include/mtr0types.h b/storage/innobase/include/mtr0types.h index 1de31126a88..19db13a12b6 100644 --- a/storage/innobase/include/mtr0types.h +++ b/storage/innobase/include/mtr0types.h @@ -342,8 +342,6 @@ enum mtr_memo_type_t { MTR_MEMO_SX_LOCK = RW_SX_LATCH << 5, /** wr_lock() on fil_space_t::latch */ - MTR_MEMO_SPACE_X_LOCK = MTR_MEMO_SX_LOCK << 1, - /** rd_lock() on fil_space_t::latch */ - MTR_MEMO_SPACE_S_LOCK = MTR_MEMO_SX_LOCK << 2 + MTR_MEMO_SPACE_X_LOCK = MTR_MEMO_SX_LOCK << 1 }; #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index fd6f70fc0ab..add2f6b123c 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -737,8 +737,10 @@ static struct bool reinit_all() { retry: + log_sys.latch.wr_unlock(); bool fail= false; buf_block_t *free_block= buf_LRU_get_free_block(false); + log_sys.latch.wr_lock(SRW_LOCK_CALL); mysql_mutex_lock(&recv_sys.mutex); for (auto d= defers.begin(); d != defers.end(); ) @@ -2573,7 +2575,8 @@ inline recv_sys_t::parse_mtr_result recv_sys_t::parse(store_t store, source &l) /* The entire undo tablespace will be reinitialized by innodb_undo_log_truncate=ON. Discard old log for all pages. */ trim({space_id, 0}, lsn); - truncated_undo_spaces[space_id - srv_undo_space_id_start]= page_no; + truncated_undo_spaces[space_id - srv_undo_space_id_start]= + { lsn, page_no }; if (undo_space_trunc) undo_space_trunc(space_id); #endif @@ -3458,23 +3461,43 @@ void recv_sys_t::apply(bool last_batch) for (auto id= srv_undo_tablespaces_open; id--;) { - if (unsigned pages= truncated_undo_spaces[id]) + const trunc& t= truncated_undo_spaces[id]; + if (t.lsn) { - if (fil_space_t *space= fil_space_get(id + srv_undo_space_id_start)) + /* The entire undo tablespace will be reinitialized by + innodb_undo_log_truncate=ON. Discard old log for all pages. + Even though we recv_sys_t::parse() already invoked trim(), + this will be needed in case recovery consists of multiple batches + (there was an invocation with !last_batch). */ + trim({id + srv_undo_space_id_start, 0}, t.lsn); + if (fil_space_t *space = fil_space_get(id + srv_undo_space_id_start)) { ut_ad(UT_LIST_GET_LEN(space->chain) == 1); fil_node_t *file= UT_LIST_GET_FIRST(space->chain); ut_ad(file->is_open()); os_file_truncate(file->name, file->handle, - os_offset_t{pages} << srv_page_size_shift, true); + os_offset_t{t.pages} << srv_page_size_shift, true); } } } fil_system.extend_to_recv_size(); + /* We must release log_sys.latch and recv_sys.mutex before + invoking buf_LRU_get_free_block(). Allocating a block may initiate + a redo log write and therefore acquire log_sys.latch. To avoid + deadlocks, log_sys.latch must not be acquired while holding + recv_sys.mutex. */ + mysql_mutex_unlock(&mutex); + if (!last_batch) + log_sys.latch.wr_unlock(); + buf_block_t *free_block= buf_LRU_get_free_block(false); + if (!last_batch) + log_sys.latch.wr_lock(SRW_LOCK_CALL); + mysql_mutex_lock(&mutex); + for (map::iterator p= pages.begin(); p != pages.end(); ) { const page_id_t page_id= p->first; @@ -3520,7 +3543,11 @@ erase_for_space: { next_free_block: mysql_mutex_unlock(&mutex); + if (!last_batch) + log_sys.latch.wr_unlock(); free_block= buf_LRU_get_free_block(false); + if (!last_batch) + log_sys.latch.wr_lock(SRW_LOCK_CALL); mysql_mutex_lock(&mutex); break; } @@ -4244,6 +4271,11 @@ read_only_recovery: || recv_sys.is_corrupt_fs()) { goto err_exit; } + + /* In case of multi-batch recovery, + redo log for the last batch is not + applied yet. */ + ut_d(recv_sys.after_apply = false); } } else { ut_ad(recv_sys.pages.empty()); diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 851d37c9d92..611670555ba 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -52,9 +52,6 @@ void mtr_memo_slot_t::release() const static_cast(object)->set_committed_size(); static_cast(object)->x_unlock(); break; - case MTR_MEMO_SPACE_S_LOCK: - static_cast(object)->s_unlock(); - break; default: buf_page_t *bpage= static_cast(object); ut_d(const auto s=) @@ -263,9 +260,6 @@ void mtr_t::release_unlogged() static_cast(slot.object)->set_committed_size(); static_cast(slot.object)->x_unlock(); break; - case MTR_MEMO_SPACE_S_LOCK: - static_cast(slot.object)->s_unlock(); - break; case MTR_MEMO_X_LOCK: case MTR_MEMO_SX_LOCK: static_cast(slot.object)-> @@ -406,9 +400,6 @@ void mtr_t::commit() static_cast(slot.object)->set_committed_size(); static_cast(slot.object)->x_unlock(); break; - case MTR_MEMO_SPACE_S_LOCK: - static_cast(slot.object)->s_unlock(); - break; case MTR_MEMO_X_LOCK: case MTR_MEMO_SX_LOCK: static_cast(slot.object)-> @@ -1200,18 +1191,14 @@ bool mtr_t::have_u_or_x_latch(const buf_block_t &block) const /** Check if we are holding exclusive tablespace latch @param space tablespace to search for -@param shared whether to look for shared latch, instead of exclusive @return whether space.latch is being held */ -bool mtr_t::memo_contains(const fil_space_t& space, bool shared) const +bool mtr_t::memo_contains(const fil_space_t& space) const { - const mtr_memo_type_t type= shared - ? MTR_MEMO_SPACE_S_LOCK : MTR_MEMO_SPACE_X_LOCK; - for (const mtr_memo_slot_t &slot : m_memo) { - if (slot.object == &space && slot.type == type) + if (slot.object == &space && slot.type == MTR_MEMO_SPACE_X_LOCK) { - ut_ad(shared || space.is_owner()); + ut_ad(space.is_owner()); return true; } } diff --git a/storage/maria/ha_s3.cc b/storage/maria/ha_s3.cc index c00f42d74c1..8c105522c11 100644 --- a/storage/maria/ha_s3.cc +++ b/storage/maria/ha_s3.cc @@ -233,7 +233,7 @@ ha_create_table_option s3_table_option_list[]= ha_s3::ha_s3(handlerton *hton, TABLE_SHARE *table_arg) - :ha_maria(hton, table_arg), in_alter_table(S3_NO_ALTER) + :ha_maria(hton, table_arg), in_alter_table(S3_NO_ALTER), open_args(NULL) { /* Remove things that S3 doesn't support */ int_table_flags&= ~(HA_BINLOG_ROW_CAPABLE | HA_BINLOG_STMT_CAPABLE | diff --git a/storage/maria/s3_func.c b/storage/maria/s3_func.c index 491a8e0a323..3d18ba8800d 100644 --- a/storage/maria/s3_func.c +++ b/storage/maria/s3_func.c @@ -351,7 +351,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket, if (display) printf("Copying frm file %s\n", filename); - end= strmov(aws_path_end,"/frm"); + strmov(aws_path_end,"/frm"); convert_frm_to_s3_format(alloc_block); /* Note that frm is not compressed! */ @@ -1232,7 +1232,7 @@ static void convert_index_to_s3_format(uchar *header, ulong block_size, uchar *base_pos; uint base_offset; - memcpy(state.header.file_version, header, sizeof(state.header)); + memcpy(&state.header, header, sizeof(state.header)); base_offset= mi_uint2korr(state.header.base_pos); base_pos= header + base_offset; @@ -1251,7 +1251,7 @@ static void convert_index_to_disk_format(uchar *header) uchar *base_pos; uint base_offset; - memcpy(state.header.file_version, header, sizeof(state.header)); + memcpy(&state.header, header, sizeof(state.header)); base_offset= mi_uint2korr(state.header.base_pos); base_pos= header + base_offset;