Merge 10.6 into 10.8

mysql-test/main/debug_sync.test

@@ -231,15 +231,12 @@ SHOW VARIABLES LIKE 'DEBUG_SYNC';
 #       immediately after setting of the DEBUG_SYNC variable.
 #       So it is executed before the SET statement ends.
 #
-# NOTE: There is only one global signal (say "signal post" or "flag mast").
+# NOTE: There can be multiple active signals at the same time.
-#       A SIGNAL action writes its signal into it ("sets a flag").
+#       A SIGNAL action appends its signal into signals set.
-#       The signal persists until explicitly overwritten.
+#       The signal persists until waited on.
 #       To avoid confusion for later tests, it is recommended to clear
-#       the signal by signalling "empty" ("setting the 'empty' flag"):
+#       the signal set by running
-#       SET DEBUG_SYNC= 'now SIGNAL empty';
-#       Preferably you can reset the whole facility with:
 #       SET DEBUG_SYNC= 'RESET';
-#       The signal is then '' (really empty) which connot be done otherwise.
 #
 
 #

mysql-test/main/empty_table.result

@@ -16,3 +16,61 @@ ERROR 42S02: Table 'test.t2' doesn't exist
 show status like "Empty_queries";
 Variable_name	Value
 Empty_queries	2
+# End of 4.1 tests
+#
+# MDEV-30333 Wrong result with not_null_range_scan and LEFT JOIN with empty table
+#
+set @save_optimizer_switch=@@optimizer_switch;
+CREATE TABLE t1 (a INT, b INT) ENGINE=MyISAM;
+INSERT INTO t1 (b) VALUES (1),(2);
+CREATE TABLE t2 (c INT) ENGINE=MyISAM;
+SET optimizer_switch= 'not_null_range_scan=off';
+explain extended SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t2	system	NULL	NULL	NULL	NULL	0	0.00	Const row not found
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where; Using filesort
+Warnings:
+Note	1003	select `test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` is null order by `test`.`t1`.`b`
+SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+b
+1
+2
+SET optimizer_switch = 'not_null_range_scan=on';
+explain extended SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t2	system	NULL	NULL	NULL	NULL	0	0.00	Const row not found
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where; Using filesort
+Warnings:
+Note	1003	select `test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` is null order by `test`.`t1`.`b`
+SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+b
+1
+2
+flush tables;
+SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+b
+1
+2
+drop table t1,t2;
+# Second test in MDEV-30333
+CREATE TABLE t1 (a int, b varchar(10)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (69,'foo'),(71,'bar');
+CREATE TABLE t2 (c int) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1),(2);
+CREATE TABLE t3 (d int, e int, KEY(e)) ENGINE=MyISAM;
+SELECT * FROM t1 LEFT JOIN t2 LEFT JOIN t3 ON t3.e = t3.d ON 1;
+a	b	c	d	e
+69	foo	1	NULL	NULL
+71	bar	1	NULL	NULL
+69	foo	2	NULL	NULL
+71	bar	2	NULL	NULL
+SET optimizer_switch = 'not_null_range_scan=on';
+SELECT * FROM t1 LEFT JOIN t2 LEFT JOIN t3 ON t3.e = t3.d ON 1;
+a	b	c	d	e
+69	foo	1	NULL	NULL
+71	bar	1	NULL	NULL
+69	foo	2	NULL	NULL
+71	bar	2	NULL	NULL
+DROP TABLE t1, t2, t3;
+set @@optimizer_switch=@save_optimizer_switch;
+End of 10.5 tests

mysql-test/main/empty_table.test

@@ -21,4 +21,38 @@ drop table t1;
 select * from t2;
 show status like "Empty_queries";
 
-# End of 4.1 tests
+--echo # End of 4.1 tests
+
+--echo #
+--echo # MDEV-30333 Wrong result with not_null_range_scan and LEFT JOIN with empty table
+--echo #
+
+set @save_optimizer_switch=@@optimizer_switch;
+CREATE TABLE t1 (a INT, b INT) ENGINE=MyISAM;
+INSERT INTO t1 (b) VALUES (1),(2);
+CREATE TABLE t2 (c INT) ENGINE=MyISAM;
+SET optimizer_switch= 'not_null_range_scan=off'; # Default
+explain extended SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+SET optimizer_switch = 'not_null_range_scan=on';
+explain extended SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+flush tables;
+SELECT b FROM t1 LEFT JOIN t2 ON t2.c = a WHERE a IS NULL ORDER BY b;
+drop table t1,t2;
+
+--echo # Second test in MDEV-30333
+
+CREATE TABLE t1 (a int, b varchar(10)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (69,'foo'),(71,'bar');
+CREATE TABLE t2 (c int) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1),(2);
+CREATE TABLE t3 (d int, e int, KEY(e)) ENGINE=MyISAM;
+SELECT * FROM t1 LEFT JOIN t2 LEFT JOIN t3 ON t3.e = t3.d ON 1;
+SET optimizer_switch = 'not_null_range_scan=on';
+SELECT * FROM t1 LEFT JOIN t2 LEFT JOIN t3 ON t3.e = t3.d ON 1;
+DROP TABLE t1, t2, t3;
+set @@optimizer_switch=@save_optimizer_switch;
+
+--echo End of 10.5 tests
+

mysql-test/main/fetch_first.result

@@ -1378,3 +1378,31 @@ a
 bar
 foo
 DROP TABLE t;
+#
+# MDEV-30324: Wrong result upon SELECT DISTINCT .. WITH TIES using index
+#
+CREATE TABLE t1 (a int, b char(3), KEY (a));
+INSERT INTO t1 VALUES (2,'foo'),(3,'bar'),(3,'bar'),(3,'zzz');
+EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 1 ROWS WITH TIES;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	NULL	a	5	NULL	1	Using temporary
+SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 1 ROWS WITH TIES;
+a	b
+2	foo
+EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 2 ROWS WITH TIES;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	NULL	a	5	NULL	2	Using temporary
+SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 2 ROWS WITH TIES;
+a	b
+2	foo
+3	bar
+3	zzz
+EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 3 ROWS WITH TIES;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	4	Using temporary; Using filesort
+SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 3 ROWS WITH TIES;
+a	b
+2	foo
+3	bar
+3	zzz
+DROP TABLE t1;

mysql-test/main/fetch_first.test

@@ -1059,3 +1059,22 @@ SELECT a FROM t ORDER BY a FETCH FIRST 2 ROWS WITH TIES;
 
 # Cleanup
 DROP TABLE t;
+
+--echo #
+--echo # MDEV-30324: Wrong result upon SELECT DISTINCT .. WITH TIES using index
+--echo #
+CREATE TABLE t1 (a int, b char(3), KEY (a));
+INSERT INTO t1 VALUES (2,'foo'),(3,'bar'),(3,'bar'),(3,'zzz');
+
+EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 1 ROWS WITH TIES;
+--sorted_result
+SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 1 ROWS WITH TIES;
+EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 2 ROWS WITH TIES;
+--sorted_result
+SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 2 ROWS WITH TIES;
+EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 3 ROWS WITH TIES;
+--sorted_result
+SELECT DISTINCT a, b FROM t1 ORDER BY a FETCH FIRST 3 ROWS WITH TIES;
+ 
+# Cleanup
+DROP TABLE t1;

mysql-test/main/func_json.result

@@ -1623,6 +1623,21 @@ id	doc
 {"$oid":"611c0a463b150154132f6636"}	{ "_id" : { "$oid" : "611c0a463b150154132f6636" }, "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : [ { "a" : 1.0 } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] } ] }
 DROP TABLE arrNestTest;
 #
+# MDEV-30412 JSON_OBJECTAGG doesn't escape double quote in key
+#
+SELECT JSON_OBJECTAGG('"', 1);
+JSON_OBJECTAGG('"', 1)
+{"\"":1}
+SELECT JSON_OBJECTAGG('\"', 1);
+JSON_OBJECTAGG('\"', 1)
+{"\"":1}
+SELECT JSON_OBJECTAGG('\\', 1);
+JSON_OBJECTAGG('\\', 1)
+{"\\":1}
+#
+# End of 10.5 tests
+#
+#
 # MDEV-26054 Server crashes in Item_func_json_arrayagg::get_str_from_field
 #
 CREATE TABLE t (a VARCHAR(8));
@@ -1649,5 +1664,5 @@ JSON_INSERT(JSON_OBJECT(l1, l2, l3, l4), '$.k3', 'v3')	JSON_SET(JSON_OBJECT(l1,
 {"k1": "v1", "k2": "v2", "k3": "v3"}	{"k1": "v1", "k2": "new v2"}	{"k1": "v1", "k2": "new v2"}
 DROP TABLE t;
 #
-# End of 10.5 tests
+# End of 10.6 tests
 #

mysql-test/main/func_json.test

@@ -1067,6 +1067,18 @@ INSERT INTO test.arrNestTest (doc) VALUES ('{ "_id" : { "$oid" : "611c0a463b1501
 SELECT * FROM arrNestTest;
 DROP TABLE arrNestTest;
 
+--echo #
+--echo # MDEV-30412 JSON_OBJECTAGG doesn't escape double quote in key
+--echo #
+
+SELECT JSON_OBJECTAGG('"', 1);
+SELECT JSON_OBJECTAGG('\"', 1);
+SELECT JSON_OBJECTAGG('\\', 1);
+
+--echo #
+--echo # End of 10.5 tests
+--echo #
+
 --echo #
 --echo # MDEV-26054 Server crashes in Item_func_json_arrayagg::get_str_from_field
 --echo #
@@ -1078,7 +1090,6 @@ SELECT JSON_ARRAYAGG(a) AS f FROM v;
 DROP VIEW v;
 DROP TABLE t;
 
-
 --echo #
 --echo # MDEV-29264 JSON functions overflow error based ON LONGTEXT field
 --echo #
@@ -1090,6 +1101,5 @@ SELECT JSON_INSERT(JSON_OBJECT(l1, l2, l3, l4), '$.k3', 'v3'),JSON_SET(JSON_OBJE
 DROP TABLE t;
 
 --echo #
---echo # End of 10.5 tests
+--echo # End of 10.6 tests
 --echo #
-

mysql-test/suite/innodb/r/alter_copy.result

@@ -51,7 +51,7 @@ ADD INDEX(a,b,d), ADD INDEX(a,d,b), ADD INDEX(b,c,d), ADD INDEX(b,d,c),
 ALGORITHM=COPY;
 connection default;
 SET DEBUG_SYNC='now WAIT_FOR hung';
-# restart: --innodb-force-recovery=3
+# restart: --innodb-force-recovery=3 --debug_dbug=+d,recv_ran_out_of_buffer
 disconnect hang;
 FTS_INDEX_1.ibd
 FTS_INDEX_2.ibd

mysql-test/suite/innodb/r/recovery_memory.result

@@ -1,3 +1,7 @@
+call mtr.add_suppression("InnoDB: The change buffer is corrupted");
+call mtr.add_suppression("InnoDB: Plugin initialization aborted at srv0start.cc");
+call mtr.add_suppression("Plugin 'InnoDB' init function returned error");
+call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed.");
 CREATE TABLE t1(c TEXT, KEY(c(3072)))ENGINE=InnoDB;
 CREATE PROCEDURE dorepeat()
 LOOP
@@ -10,3 +14,19 @@ CALL dorepeat();
 connection default;
 # restart: --innodb_buffer_pool_size=5242880
 DROP TABLE t1;
+DROP PROCEDURE dorepeat;
+#
+# MDEV-30552  InnoDB recovery crashes when error
+#        handling scenario
+#
+SET DEBUG_DBUG="+d,ib_log_checkpoint_avoid_hard";
+CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB;
+INSERT INTO t1 SELECT * FROM seq_1_to_65536;
+# restart: --innodb_buffer_pool_size=5242880 --debug_dbug=+d,ibuf_init_corrupt
+# restart
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `f1` int(11) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+DROP TABLE t1;

mysql-test/suite/innodb/t/alter_copy.test

@@ -57,7 +57,7 @@ ALTER TABLE t ADD INDEX(b,c,d,a),ADD INDEX(b,c,a,d),ADD INDEX(b,a,c,d),ADD INDEX
 connection default;
 SET DEBUG_SYNC='now WAIT_FOR hung';
 let $shutdown_timeout=0;
---let $restart_parameters= --innodb-force-recovery=3
+--let $restart_parameters= --innodb-force-recovery=3 --debug_dbug="+d,recv_ran_out_of_buffer"
 --source include/restart_mysqld.inc
 disconnect hang;
 let $shutdown_timeout=;

mysql-test/suite/innodb/t/recovery_memory.test

@@ -1,5 +1,10 @@
 --source include/have_innodb.inc
 --source include/big_test.inc
+--source include/have_sequence.inc
+call mtr.add_suppression("InnoDB: The change buffer is corrupted");
+call mtr.add_suppression("InnoDB: Plugin initialization aborted at srv0start.cc");
+call mtr.add_suppression("Plugin 'InnoDB' init function returned error");
+call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed.");
 CREATE TABLE t1(c TEXT, KEY(c(3072)))ENGINE=InnoDB;
 DELIMITER |;
 
@@ -19,3 +24,19 @@ let $shutdown_timeout=0;
 let $restart_parameters=--innodb_buffer_pool_size=5242880;
 --source include/restart_mysqld.inc
 DROP TABLE t1;
+DROP PROCEDURE dorepeat;
+
+--echo #
+--echo # MDEV-30552  InnoDB recovery crashes when error
+--echo #        handling scenario
+--echo #
+SET DEBUG_DBUG="+d,ib_log_checkpoint_avoid_hard";
+CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB;
+INSERT INTO t1 SELECT * FROM seq_1_to_65536;
+let $shutdown_timeout=0;
+let $restart_parameters=--innodb_buffer_pool_size=5242880 --debug_dbug="+d,ibuf_init_corrupt";
+--source include/restart_mysqld.inc
+let $restart_parameters=;
+--source include/restart_mysqld.inc
+SHOW CREATE TABLE t1;
+DROP TABLE t1;

sql/debug_sync.cc

@@ -1344,7 +1344,8 @@ static bool debug_sync_eval_action(THD *thd, char *action_str, char *action_end)
   /*
     Try NO_CLEAR_EVENT.
   */
-  if (!my_strcasecmp(system_charset_info, token, "NO_CLEAR_EVENT")) {
+  if (!my_strcasecmp(system_charset_info, token, "NO_CLEAR_EVENT"))
+  {
     action->clear_event= false;
     /* Get next token. If none follows, set action. */
     if (!(ptr = debug_sync_token(&token, &token_length, ptr, action_end))) goto set_action;
@@ -1634,8 +1635,8 @@ static void debug_sync_execute(THD *thd, st_debug_sync_action *action)
 
 
       /*
-        Wait until global signal string matches the wait_for string.
+        Wait until the signal set contains the wait_for string.
-        Interrupt when thread or query is killed or facility disabled.
+        Interrupt when thread or query is killed or facility is disabled.
         The facility can become disabled when some thread cannot get
         the required dynamic memory allocated.
       */

sql/item_jsonfunc.cc

@@ -4143,7 +4143,7 @@ bool Item_func_json_objectagg::add()
     result.append(STRING_WITH_LEN(", "));
 
   result.append('"');
-  result.append(*key);
+  st_append_escaped(&result,key);
   result.append(STRING_WITH_LEN("\":"));
 
   buf.length(0);

sql/sql_hset.h

@@ -40,7 +40,7 @@ public:
 
   Hash_set(PSI_memory_key psi_key, CHARSET_INFO *charset, ulong default_array_elements,
            size_t key_offset, size_t key_length, my_hash_get_key get_key,
-           void (*free_element)(void*),uint flags)
+           void (*free_element)(void*), uint flags)
   {
     my_hash_init(psi_key, &m_hash, charset, default_array_elements, key_offset,
                  key_length, get_key, free_element, flags);

sql/sql_select.cc

@@ -4166,7 +4166,7 @@ JOIN::optimize_distinct()
   }
 
   /* Optimize "select distinct b from t1 order by key_part_1 limit #" */
-  if (order && skip_sort_order)
+  if (order && skip_sort_order && !unit->lim.is_with_ties())
   {
     /* Should already have been optimized away */
     DBUG_ASSERT(ordered_index_usage == ordered_index_order_by);
@@ -30088,7 +30088,6 @@ void JOIN::make_notnull_conds_for_range_scans()
 {
   DBUG_ENTER("JOIN::make_notnull_conds_for_range_scans");
 
-
   if (impossible_where ||
       !optimizer_flag(thd, OPTIMIZER_SWITCH_NOT_NULL_RANGE_SCAN))
   {
@@ -30168,7 +30167,6 @@ bool build_notnull_conds_for_range_scans(JOIN *join, Item *cond,
                                          table_map allowed)
 {
   THD *thd= join->thd;
-
   DBUG_ENTER("build_notnull_conds_for_range_scans");
 
   for (JOIN_TAB *s= join->join_tab;
@@ -30176,13 +30174,13 @@ bool build_notnull_conds_for_range_scans(JOIN *join, Item *cond,
   {
     /* Clear all needed bitmaps to mark found fields */
     if ((allowed & s->table->map) &&
-        !(s->table->map && join->const_table_map))
+        !(s->table->map & join->const_table_map))
       bitmap_clear_all(&s->table->tmp_set);
   }
 
   /*
     Find all null-rejected fields assuming that cond is null-rejected and
-    only  formulas over tables from 'allowed' are to be taken into account
+    only formulas over tables from 'allowed' are to be taken into account
   */
   if (cond->find_not_null_fields(allowed))
     DBUG_RETURN(true);

storage/innobase/buf/buf0buf.cc

@@ -1940,28 +1940,22 @@ static void buf_relocate(buf_page_t *bpage, buf_page_t *dpage)
   buf_pool.page_hash.replace(chain, bpage, dpage);
 }
 
-/** Register a watch for a page identifier. The caller must hold an
+buf_page_t *buf_pool_t::watch_set(const page_id_t id,
-exclusive page hash latch. The *hash_lock may be released,
+                                  buf_pool_t::hash_chain &chain)
-relocated, and reacquired.
-@param id         page identifier
-@param chain      hash table chain with exclusively held page_hash
-@return a buffer pool block corresponding to id
-@retval nullptr   if the block was not present, and a watch was installed */
-inline buf_page_t *buf_pool_t::watch_set(const page_id_t id,
-                                         buf_pool_t::hash_chain &chain)
 {
   ut_ad(&chain == &page_hash.cell_get(id.fold()));
-  ut_ad(page_hash.lock_get(chain).is_write_locked());
+  page_hash.lock_get(chain).lock();
 
-retry:
+  buf_page_t *bpage= page_hash.get(id, chain);
-  if (buf_page_t *bpage= page_hash.get(id, chain))
+
+  if (bpage)
   {
-    if (!watch_is_sentinel(*bpage))
+got_block:
-      /* The page was loaded meanwhile. */
-      return bpage;
-    /* Add to an existing watch. */
     bpage->fix();
-    return nullptr;
+    if (watch_is_sentinel(*bpage))
+      bpage= nullptr;
+    page_hash.lock_get(chain).unlock();
+    return bpage;
   }
 
   page_hash.lock_get(chain).unlock();
@@ -1990,25 +1984,23 @@ retry:
     w->set_state(buf_page_t::UNFIXED + 1);
     w->id_= id;
 
-    buf_page_t *bpage= page_hash.get(id, chain);
+    page_hash.lock_get(chain).lock();
+    bpage= page_hash.get(id, chain);
     if (UNIV_LIKELY_NULL(bpage))
     {
       w->set_state(buf_page_t::NOT_USED);
-      page_hash.lock_get(chain).lock();
       mysql_mutex_unlock(&mutex);
-      goto retry;
+      goto got_block;
     }
 
-    page_hash.lock_get(chain).lock();
     ut_ad(w->state() == buf_page_t::UNFIXED + 1);
     buf_pool.page_hash.append(chain, w);
     mysql_mutex_unlock(&mutex);
+    page_hash.lock_get(chain).unlock();
     return nullptr;
   }
 
   ut_error;
-  mysql_mutex_unlock(&mutex);
-  return nullptr;
 }
 
 /** Stop watching whether a page has been read in.
@@ -2456,16 +2448,13 @@ loop:
 	case BUF_PEEK_IF_IN_POOL:
 		return nullptr;
 	case BUF_GET_IF_IN_POOL_OR_WATCH:
-		/* We cannot easily use a memory transaction here. */
+		/* Buffer-fixing inside watch_set() will prevent eviction */
-		hash_lock.lock();
 		block = reinterpret_cast<buf_block_t*>
 			(buf_pool.watch_set(page_id, chain));
-		/* buffer-fixing will prevent eviction */
-		state = block ? block->page.fix() : 0;
-		hash_lock.unlock();
 
 		if (block) {
-			goto got_block;
+			state = block->page.state();
+			goto got_block_fixed;
 		}
 
 		return nullptr;
@@ -2504,6 +2493,7 @@ loop:
 got_block:
 	ut_ad(!block->page.in_zip_hash);
 	state++;
+got_block_fixed:
 	ut_ad(state > buf_page_t::FREED);
 
 	if (state > buf_page_t::READ_FIX && state < buf_page_t::WRITE_FIX) {
@@ -2713,24 +2703,19 @@ re_evict:
 		const bool evicted = buf_LRU_free_page(&block->page, true);
 		space->release();
 
+		if (!evicted) {
+			block->fix();
+		}
+
+		mysql_mutex_unlock(&buf_pool.mutex);
+
 		if (evicted) {
-			page_hash_latch& hash_lock
+			if (mode == BUF_GET_IF_IN_POOL_OR_WATCH) {
-				= buf_pool.page_hash.lock_get(chain);
+				buf_pool.watch_set(page_id, chain);
-			hash_lock.lock();
+			}
-			mysql_mutex_unlock(&buf_pool.mutex);
-			/* We may set the watch, as it would have
-			been set if the page were not in the
-			buffer pool in the first place. */
-			block= reinterpret_cast<buf_block_t*>(
-				mode == BUF_GET_IF_IN_POOL_OR_WATCH
-				? buf_pool.watch_set(page_id, chain)
-				: buf_pool.page_hash.get(page_id, chain));
-			hash_lock.unlock();
 			return(NULL);
 		}
 
-		block->fix();
-		mysql_mutex_unlock(&buf_pool.mutex);
 		buf_flush_sync();
 
 		state = block->page.state();
@@ -3438,8 +3423,7 @@ or decrypt/decompress just failed.
 @retval	DB_SUCCESS		if page has been read and is not corrupted
 @retval	DB_PAGE_CORRUPTED	if page based on checksum check is corrupted
 @retval	DB_DECRYPTION_FAILED	if page post encryption checksum matches but
-after decryption normal page checksum does not match.
+after decryption normal page checksum does not match. */
-@retval	DB_TABLESPACE_DELETED	if accessed tablespace is not found */
 static dberr_t buf_page_check_corrupt(buf_page_t *bpage,
                                       const fil_node_t &node)
 {
@@ -3496,7 +3480,8 @@ static dberr_t buf_page_check_corrupt(buf_page_t *bpage,
 @param node     data file
 @return whether the operation succeeded
 @retval DB_PAGE_CORRUPTED    if the checksum fails
-@retval DB_DECRYPTION_FAILED if the page cannot be decrypted */
+@retval DB_DECRYPTION_FAILED if the page cannot be decrypted
+@retval DB_FAIL              if the page contains the wrong ID */
 dberr_t buf_page_t::read_complete(const fil_node_t &node)
 {
   const page_id_t expected_id{id()};

storage/innobase/buf/buf0lru.cc

@@ -397,6 +397,11 @@ buf_block_t *buf_LRU_get_free_block(bool have_mutex)
 		mysql_mutex_assert_owner(&buf_pool.mutex);
 		goto got_mutex;
 	}
+	DBUG_EXECUTE_IF("recv_ran_out_of_buffer",
+			if (recv_recovery_is_on()
+			    && recv_sys.apply_log_recs) {
+				goto flush_lru;
+			});
 	mysql_mutex_lock(&buf_pool.mutex);
 got_mutex:
 	buf_LRU_check_size_of_non_data_objects();
@@ -480,7 +485,9 @@ not_found:
 	removing the block from buf_pool.page_hash and buf_pool.LRU is fairly
 	involved (particularly in case of ROW_FORMAT=COMPRESSED pages). We
 	can do that in a separate patch sometime in future. */
-
+#ifndef DBUG_OFF
+flush_lru:
+#endif
 	if (!buf_flush_LRU(innodb_lru_flush_size)) {
 		MONITOR_INC(MONITOR_LRU_SINGLE_FLUSH_FAILURE_COUNT);
 		++flush_failures;

storage/innobase/buf/buf0rea.cc

@@ -331,6 +331,9 @@ nothing_read:
 		/* The i/o was already completed in space->io() */
 		*err = bpage->read_complete(*fio.node);
 		space->release();
+		if (*err == DB_FAIL) {
+			*err = DB_PAGE_CORRUPTED;
+		}
 	}
 
 	return true;

storage/innobase/dict/dict0defrag_bg.cc

@@ -314,7 +314,7 @@ btr_get_size_and_reserved(
 		return ULINT_UNDEFINED;
 	}
 
-	mtr->s_lock_space(index->table->space);
+	mtr->x_lock_space(index->table->space);
 
 	ulint n = fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_LEAF
 					+ root->page.frame, used, mtr);

storage/innobase/dict/dict0stats.cc

@@ -1490,7 +1490,7 @@ invalid:
 			goto invalid;
 		}
 
-		mtr.s_lock_space(index->table->space);
+		mtr.x_lock_space(index->table->space);
 
 		ulint dummy, size;
 		index->stat_index_size
@@ -2696,7 +2696,7 @@ empty_index:
 	}
 
 	uint16_t root_level = btr_page_get_level(root->page.frame);
-	mtr.s_lock_space(index->table->space);
+	mtr.x_lock_space(index->table->space);
 	ulint dummy, size;
 	result.index_size
 		= fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_LEAF

storage/innobase/fil/fil0crypt.cc

@@ -1678,8 +1678,9 @@ fil_crypt_get_page_throttle(
 		return NULL;
 	}
 
-	if (DB_SUCCESS_LOCKED_REC
+	if (offset % (zip_size ? zip_size : srv_page_size)
-	    != fseg_page_is_allocated(space, state->offset)) {
+	    && DB_SUCCESS_LOCKED_REC
+	    != fseg_page_is_allocated(space, offset)) {
 		/* page is already freed */
 		return NULL;
 	}

storage/innobase/fsp/fsp0fsp.cc

@@ -2650,7 +2650,7 @@ dberr_t fseg_page_is_allocated(fil_space_t *space, unsigned page)
 
   mtr.start();
   if (!space->is_owner())
-    mtr.s_lock_space(space);
+    mtr.x_lock_space(space);
 
   if (page >= space->free_limit || page >= space->size_in_header);
   else if (const buf_block_t *b=

storage/innobase/ibuf/ibuf0ibuf.cc

@@ -426,6 +426,10 @@ err_exit:
 		goto err_exit;
 	}
 
+	DBUG_EXECUTE_IF("ibuf_init_corrupt",
+			err = DB_CORRUPTION;
+			goto err_exit;);
+
 	if (page_is_comp(root) || fil_page_get_type(root) != FIL_PAGE_INDEX
 	    || btr_page_get_index_id(root) != DICT_IBUF_ID_MIN) {
 		err = DB_CORRUPTION;

storage/innobase/include/buf0buf.h

@@ -769,7 +769,8 @@ public:
   @param node     data file
   @return whether the operation succeeded
   @retval DB_PAGE_CORRUPTED    if the checksum fails
-  @retval DB_DECRYPTION_FAILED if the page cannot be decrypted */
+  @retval DB_DECRYPTION_FAILED if the page cannot be decrypted
+  @retval DB_FAIL              if the page contains the wrong ID */
   dberr_t read_complete(const fil_node_t &node);
 
   /** Note that a block is no longer dirty, while not removing
@@ -1465,14 +1466,12 @@ public:
     return !watch_is_sentinel(*page_hash.get(id, chain));
   }
 
-  /** Register a watch for a page identifier. The caller must hold an
+  /** Register a watch for a page identifier.
-  exclusive page hash latch. The *hash_lock may be released,
-  relocated, and reacquired.
   @param id         page identifier
-  @param chain      hash table chain with exclusively held page_hash
+  @param chain      page_hash.cell_get(id.fold())
-  @return a buffer pool block corresponding to id
+  @return a buffer page corresponding to id
-  @retval nullptr   if the block was not present, and a watch was installed */
+  @retval nullptr   if the block was not present in page_hash */
-  inline buf_page_t *watch_set(const page_id_t id, hash_chain &chain);
+  buf_page_t *watch_set(const page_id_t id, hash_chain &chain);
 
   /** Stop watching whether a page has been read in.
   watch_set(id) must have returned nullptr before.

storage/innobase/include/log0recv.h

@@ -243,9 +243,15 @@ private:
   @param lsn     log sequence number of the shrink operation */
   inline void trim(const page_id_t page_id, lsn_t lsn);
 
-  /** Truncated undo tablespace size for which truncate has been logged
+  /** Undo tablespaces for which truncate has been logged
-  (indexed by page_id_t::space() - srv_undo_space_id_start), or 0 */
+  (indexed by page_id_t::space() - srv_undo_space_id_start) */
-  unsigned truncated_undo_spaces[127];
+  struct trunc
+  {
+    /** log sequence number of FILE_CREATE, or 0 if none */
+    lsn_t lsn;
+    /** truncated size of the tablespace, or 0 if not truncated */
+    unsigned pages;
+  } truncated_undo_spaces[127];
 
 public:
   /** The contents of the doublewrite buffer */

storage/innobase/include/mtr0mtr.h

@@ -283,17 +283,6 @@ struct mtr_t {
     memo_push(lock, MTR_MEMO_SX_LOCK);
   }
 
-  /** Acquire a tablespace S-latch.
-  @param space  tablespace */
-  void s_lock_space(fil_space_t *space)
-  {
-    ut_ad(space->purpose == FIL_TYPE_TEMPORARY ||
-          space->purpose == FIL_TYPE_IMPORT ||
-          space->purpose == FIL_TYPE_TABLESPACE);
-    memo_push(space, MTR_MEMO_SPACE_S_LOCK);
-    space->s_lock();
-  }
-
   /** Acquire an exclusive tablespace latch.
   @param space  tablespace */
   void x_lock_space(fil_space_t *space);
@@ -355,9 +344,8 @@ public:
 
   /** Check if we are holding tablespace latch
   @param space  tablespace to search for
-  @param shared whether to look for shared latch, instead of exclusive
   @return whether space.latch is being held */
-  bool memo_contains(const fil_space_t& space, bool shared= false) const
+  bool memo_contains(const fil_space_t& space) const
     MY_ATTRIBUTE((warn_unused_result));
 #ifdef UNIV_DEBUG
   /** Check if we are holding an rw-latch in this mini-transaction
@@ -410,7 +398,7 @@ public:
       break;
     case MTR_MEMO_MODIFY:
     case MTR_MEMO_S_LOCK: case MTR_MEMO_X_LOCK: case MTR_MEMO_SX_LOCK:
-    case MTR_MEMO_SPACE_X_LOCK: case MTR_MEMO_SPACE_S_LOCK:
+    case MTR_MEMO_SPACE_X_LOCK:
       ut_ad("invalid type" == 0);
     }
 #endif

storage/innobase/include/mtr0types.h

@@ -342,8 +342,6 @@ enum mtr_memo_type_t {
 	MTR_MEMO_SX_LOCK = RW_SX_LATCH << 5,
 
 	/** wr_lock() on fil_space_t::latch */
-	MTR_MEMO_SPACE_X_LOCK = MTR_MEMO_SX_LOCK << 1,
+	MTR_MEMO_SPACE_X_LOCK = MTR_MEMO_SX_LOCK << 1
-	/** rd_lock() on fil_space_t::latch */
-	MTR_MEMO_SPACE_S_LOCK = MTR_MEMO_SX_LOCK << 2
 };
 #endif /* !UNIV_INNOCHECKSUM */

storage/innobase/log/log0recv.cc

@@ -737,8 +737,10 @@ static struct
   bool reinit_all()
   {
 retry:
+    log_sys.latch.wr_unlock();
     bool fail= false;
     buf_block_t *free_block= buf_LRU_get_free_block(false);
+    log_sys.latch.wr_lock(SRW_LOCK_CALL);
     mysql_mutex_lock(&recv_sys.mutex);
 
     for (auto d= defers.begin(); d != defers.end(); )
@@ -2573,7 +2575,8 @@ inline recv_sys_t::parse_mtr_result recv_sys_t::parse(store_t store, source &l)
           /* The entire undo tablespace will be reinitialized by
           innodb_undo_log_truncate=ON. Discard old log for all pages. */
           trim({space_id, 0}, lsn);
-          truncated_undo_spaces[space_id - srv_undo_space_id_start]= page_no;
+          truncated_undo_spaces[space_id - srv_undo_space_id_start]=
+            { lsn, page_no };
           if (undo_space_trunc)
             undo_space_trunc(space_id);
 #endif
@@ -3458,23 +3461,43 @@ void recv_sys_t::apply(bool last_batch)
 
     for (auto id= srv_undo_tablespaces_open; id--;)
     {
-      if (unsigned pages= truncated_undo_spaces[id])
+      const trunc& t= truncated_undo_spaces[id];
+      if (t.lsn)
       {
-        if (fil_space_t *space= fil_space_get(id + srv_undo_space_id_start))
+        /* The entire undo tablespace will be reinitialized by
+        innodb_undo_log_truncate=ON. Discard old log for all pages.
+        Even though we recv_sys_t::parse() already invoked trim(),
+        this will be needed in case recovery consists of multiple batches
+        (there was an invocation with !last_batch). */
+        trim({id + srv_undo_space_id_start, 0}, t.lsn);
+        if (fil_space_t *space = fil_space_get(id + srv_undo_space_id_start))
         {
           ut_ad(UT_LIST_GET_LEN(space->chain) == 1);
           fil_node_t *file= UT_LIST_GET_FIRST(space->chain);
           ut_ad(file->is_open());
           os_file_truncate(file->name, file->handle,
-                           os_offset_t{pages} << srv_page_size_shift, true);
+                           os_offset_t{t.pages} << srv_page_size_shift, true);
         }
       }
     }
 
     fil_system.extend_to_recv_size();
 
+    /* We must release log_sys.latch and recv_sys.mutex before
+    invoking buf_LRU_get_free_block(). Allocating a block may initiate
+    a redo log write and therefore acquire log_sys.latch. To avoid
+    deadlocks, log_sys.latch must not be acquired while holding
+    recv_sys.mutex. */
+    mysql_mutex_unlock(&mutex);
+    if (!last_batch)
+      log_sys.latch.wr_unlock();
+
     buf_block_t *free_block= buf_LRU_get_free_block(false);
 
+    if (!last_batch)
+      log_sys.latch.wr_lock(SRW_LOCK_CALL);
+    mysql_mutex_lock(&mutex);
+
     for (map::iterator p= pages.begin(); p != pages.end(); )
     {
       const page_id_t page_id= p->first;
@@ -3520,7 +3543,11 @@ erase_for_space:
         {
 next_free_block:
           mysql_mutex_unlock(&mutex);
+          if (!last_batch)
+            log_sys.latch.wr_unlock();
           free_block= buf_LRU_get_free_block(false);
+          if (!last_batch)
+            log_sys.latch.wr_lock(SRW_LOCK_CALL);
           mysql_mutex_lock(&mutex);
           break;
         }
@@ -4244,6 +4271,11 @@ read_only_recovery:
 			    || recv_sys.is_corrupt_fs()) {
 				goto err_exit;
 			}
+
+			/* In case of multi-batch recovery,
+			redo log for the last batch is not
+			applied yet. */
+			ut_d(recv_sys.after_apply = false);
 		}
 	} else {
 		ut_ad(recv_sys.pages.empty());

storage/innobase/mtr/mtr0mtr.cc

@@ -52,9 +52,6 @@ void mtr_memo_slot_t::release() const
     static_cast<fil_space_t*>(object)->set_committed_size();
     static_cast<fil_space_t*>(object)->x_unlock();
     break;
-  case MTR_MEMO_SPACE_S_LOCK:
-    static_cast<fil_space_t*>(object)->s_unlock();
-    break;
   default:
     buf_page_t *bpage= static_cast<buf_page_t*>(object);
     ut_d(const auto s=)
@@ -263,9 +260,6 @@ void mtr_t::release_unlogged()
       static_cast<fil_space_t*>(slot.object)->set_committed_size();
       static_cast<fil_space_t*>(slot.object)->x_unlock();
       break;
-    case MTR_MEMO_SPACE_S_LOCK:
-      static_cast<fil_space_t*>(slot.object)->s_unlock();
-      break;
     case MTR_MEMO_X_LOCK:
     case MTR_MEMO_SX_LOCK:
       static_cast<index_lock*>(slot.object)->
@@ -406,9 +400,6 @@ void mtr_t::commit()
           static_cast<fil_space_t*>(slot.object)->set_committed_size();
           static_cast<fil_space_t*>(slot.object)->x_unlock();
           break;
-        case MTR_MEMO_SPACE_S_LOCK:
-          static_cast<fil_space_t*>(slot.object)->s_unlock();
-          break;
         case MTR_MEMO_X_LOCK:
         case MTR_MEMO_SX_LOCK:
           static_cast<index_lock*>(slot.object)->
@@ -1200,18 +1191,14 @@ bool mtr_t::have_u_or_x_latch(const buf_block_t &block) const
 
 /** Check if we are holding exclusive tablespace latch
 @param space  tablespace to search for
-@param shared whether to look for shared latch, instead of exclusive
 @return whether space.latch is being held */
-bool mtr_t::memo_contains(const fil_space_t& space, bool shared) const
+bool mtr_t::memo_contains(const fil_space_t& space) const
 {
-  const mtr_memo_type_t type= shared
-    ? MTR_MEMO_SPACE_S_LOCK : MTR_MEMO_SPACE_X_LOCK;
-
   for (const mtr_memo_slot_t &slot : m_memo)
   {
-    if (slot.object == &space && slot.type == type)
+    if (slot.object == &space && slot.type == MTR_MEMO_SPACE_X_LOCK)
     {
-      ut_ad(shared || space.is_owner());
+      ut_ad(space.is_owner());
       return true;
     }
   }

storage/maria/ha_s3.cc

@@ -233,7 +233,7 @@ ha_create_table_option s3_table_option_list[]=
 
 
 ha_s3::ha_s3(handlerton *hton, TABLE_SHARE *table_arg)
-  :ha_maria(hton, table_arg), in_alter_table(S3_NO_ALTER)
+  :ha_maria(hton, table_arg), in_alter_table(S3_NO_ALTER), open_args(NULL)
 {
   /* Remove things that S3 doesn't support */
   int_table_flags&= ~(HA_BINLOG_ROW_CAPABLE | HA_BINLOG_STMT_CAPABLE |

storage/maria/s3_func.c

@@ -351,7 +351,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
       if (display)
         printf("Copying frm file %s\n", filename);
 
-      end= strmov(aws_path_end,"/frm");
+      strmov(aws_path_end,"/frm");
       convert_frm_to_s3_format(alloc_block);
 
       /* Note that frm is not compressed! */
@@ -1232,7 +1232,7 @@ static void convert_index_to_s3_format(uchar *header, ulong block_size,
   uchar *base_pos;
   uint  base_offset;
 
-  memcpy(state.header.file_version, header, sizeof(state.header));
+  memcpy(&state.header, header, sizeof(state.header));
   base_offset= mi_uint2korr(state.header.base_pos);
   base_pos= header + base_offset;
 
@@ -1251,7 +1251,7 @@ static void convert_index_to_disk_format(uchar *header)
   uchar *base_pos;
   uint  base_offset;
 
-  memcpy(state.header.file_version, header, sizeof(state.header));
+  memcpy(&state.header, header, sizeof(state.header));
   base_offset= mi_uint2korr(state.header.base_pos);
   base_pos= header + base_offset;