MDEV-13481 Merge new release of InnoDB MySQL 5.7.19 to 10.2

Only a relevant subset of the InnoDB changes was merged.
In particular, two follow-up bug fixes for the bugs that
were introduced in 5.7.18 but not MariaDB 10.2.7 were omitted.
Because MariaDB 10.2.7 omitted the risky change

Bug#23481444 OPTIMISER CALL ROW_SEARCH_MVCC() AND READ THE INDEX
APPLIED BY UNCOMMITTED ROWS

we do not need the follow-up fixes that were introduced in
MySQL 5.6.37 and MySQL 5.7.19:

Bug#25175249 ASSERTION: (TEMPL->IS_VIRTUAL && !FIELD) || ...
Bug#25793677 INNODB: FAILING ASSERTION: CLUST_TEMPL_FOR_SEC || LEN

mysql-test/suite/gcol/r/gcol_bugfixes.result

@@ -576,6 +576,19 @@ SELECT 1 FROM t WHERE c GROUP BY b;
 COMMIT;
 DROP TABLE t;
 #
+# Bug #25793677	INNODB: FAILING ASSERTION: CLUST_TEMPL_FOR_SEC || LEN ....
+#
+CREATE TABLE v (
+a INT,
+c INT,
+b CHAR(2) GENERATED ALWAYS AS (a IN (1)) VIRTUAL,
+KEY(c,b(1))) charset utf8mb4;
+INSERT INTO v (a,c) VALUES (1,1);
+SELECT (SELECT MAX(c) FROM v);
+(SELECT MAX(c) FROM v)
+1
+DROP TABLE v;
+#
 # MDEV-9255 Add generation_expression to information_schema.columns.
 #
 CREATE TABLE gcol_t1 (

mysql-test/suite/gcol/t/gcol_bugfixes.test

@@ -537,6 +537,19 @@ SELECT 1 FROM t WHERE c GROUP BY b;
 COMMIT;
 DROP TABLE t;
 
+--echo #
+--echo # Bug #25793677	INNODB: FAILING ASSERTION: CLUST_TEMPL_FOR_SEC || LEN ....
+--echo #
+
+CREATE TABLE v (
+a INT,
+c INT,
+b CHAR(2) GENERATED ALWAYS AS (a IN (1)) VIRTUAL,
+KEY(c,b(1))) charset utf8mb4;
+INSERT INTO v (a,c) VALUES (1,1);
+SELECT (SELECT MAX(c) FROM v);
+DROP TABLE v;
+
 --echo #
 --echo # MDEV-9255 Add generation_expression to information_schema.columns.
 --echo #

mysql-test/suite/innodb/r/rename_table.result

@@ -0,0 +1,20 @@
+CREATE DATABASE test_jfg;
+CREATE DATABASE test_jfg2;
+CREATE TABLE test_jfg.test (a int unsigned PRIMARY KEY) ENGINE=InnoDB;
+RENAME TABLE test_jfg.test TO test_jfg2.test;
+SELECT REPLACE(path,'\\','/') path
+FROM INFORMATION_SCHEMA.INNODB_SYS_DATAFILES WHERE PATH LIKE '%test%';
+path
+./test_jfg2/test.ibd
+DROP DATABASE test_jfg;
+DROP DATABASE test_jfg2;
+CREATE DATABASE abc_def;
+CREATE DATABASE abc_def2;
+CREATE TABLE abc_def.test (a int unsigned PRIMARY KEY) ENGINE=InnoDB;
+RENAME TABLE abc_def.test TO abc_def2.test1;
+SELECT REPLACE(path,'\\','/') path
+FROM INFORMATION_SCHEMA.INNODB_SYS_DATAFILES WHERE PATH LIKE '%test%';
+path
+./abc_def2/test1.ibd
+DROP DATABASE abc_def;
+DROP DATABASE abc_def2;

mysql-test/suite/innodb/t/rename_table.opt

@@ -0,0 +1,2 @@
+--innodb
+--innodb-sys-datafiles

mysql-test/suite/innodb/t/rename_table.test

@@ -0,0 +1,31 @@
+--source include/have_innodb.inc
+--source include/not_embedded.inc
+
+CREATE DATABASE test_jfg;
+CREATE DATABASE test_jfg2;
+CREATE TABLE test_jfg.test (a int unsigned PRIMARY KEY) ENGINE=InnoDB;
+RENAME TABLE test_jfg.test TO test_jfg2.test;
+
+SELECT REPLACE(path,'\\','/') path
+FROM INFORMATION_SCHEMA.INNODB_SYS_DATAFILES WHERE PATH LIKE '%test%';
+
+DROP DATABASE test_jfg;
+
+--source include/restart_mysqld.inc
+
+DROP DATABASE test_jfg2;
+
+CREATE DATABASE abc_def;
+CREATE DATABASE abc_def2;
+
+CREATE TABLE abc_def.test (a int unsigned PRIMARY KEY) ENGINE=InnoDB;
+RENAME TABLE abc_def.test TO abc_def2.test1;
+
+SELECT REPLACE(path,'\\','/') path
+FROM INFORMATION_SCHEMA.INNODB_SYS_DATAFILES WHERE PATH LIKE '%test%';
+
+DROP DATABASE abc_def;
+
+--source include/restart_mysqld.inc
+
+DROP DATABASE abc_def2;

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -3074,7 +3074,7 @@ READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	OPTIONAL
 VARIABLE_NAME	INNODB_VERSION
 SESSION_VALUE	NULL
-GLOBAL_VALUE	5.7.18
+GLOBAL_VALUE	5.7.19
 GLOBAL_VALUE_ORIGIN	COMPILE-TIME
 DEFAULT_VALUE	NULL
 VARIABLE_SCOPE	GLOBAL

storage/innobase/fil/fil0fil.cc

@@ -3292,15 +3292,17 @@ fil_prepare_for_truncate(
 
 /** Reinitialize the original tablespace header with the same space id
 for single tablespace
-@param[in]      id              space id of the tablespace
+@param[in]      table		table belongs to tablespace
 @param[in]      size            size in blocks
 @param[in]      trx             Transaction covering truncate */
 void
-fil_reinit_space_header(
+fil_reinit_space_header_for_table(
-	ulint		id,
+	dict_table_t*	table,
 	ulint		size,
 	trx_t*		trx)
 {
+	ulint	id = table->space;
+
 	ut_a(!is_system_tablespace(id));
 
 	/* Invalidate in the buffer pool all pages belonging
@@ -3309,6 +3311,9 @@ fil_reinit_space_header(
 	and the dict operation lock during the scan and aquire
 	it again after the buffer pool scan.*/
 
+	/* Release the lock on the indexes too. So that
+	they won't violate the latch ordering. */
+	dict_table_x_unlock_indexes(table);
 	row_mysql_unlock_data_dictionary(trx);
 
 	/* Lock the search latch in shared mode to prevent user
@@ -3317,8 +3322,11 @@ fil_reinit_space_header(
 	DEBUG_SYNC_C("buffer_pool_scan");
 	buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_ALL_NO_WRITE, 0);
 	btr_search_s_unlock_all();
+
 	row_mysql_lock_data_dictionary(trx);
 
+	dict_table_x_lock_indexes(table);
+
 	/* Remove all insert buffer entries for the tablespace */
 	ibuf_delete_for_discarded_space(id);
 

storage/innobase/handler/ha_innodb.cc

@@ -15637,10 +15637,14 @@ get_foreign_key_info(
 
 		if (ref_table == NULL) {
 
-			ib::info() << "Foreign Key referenced table "
+			if (!thd_test_options(
+				thd, OPTION_NO_FOREIGN_KEY_CHECKS)) {
+				ib::info()
+					<< "Foreign Key referenced table "
 					<< foreign->referenced_table_name
 					<< " not found for foreign table "
 					<< foreign->foreign_table_name;
+			}
 		} else {
 
 			dict_table_close(ref_table, TRUE, FALSE);

storage/innobase/include/fil0fil.h

@@ -963,12 +963,12 @@ fil_prepare_for_truncate(
 
 /** Reinitialize the original tablespace header with the same space id
 for single tablespace
-@param[in]	id		space id of the tablespace
+@param[in]	table		table belongs to the tablespace
 @param[in]	size            size in blocks
 @param[in]	trx		Transaction covering truncate */
 void
-fil_reinit_space_header(
+fil_reinit_space_header_for_table(
-	ulint		id,
+	dict_table_t*	table,
 	ulint		size,
 	trx_t*		trx);
 

storage/innobase/include/univ.i

@@ -41,7 +41,7 @@ Created 1/20/1994 Heikki Tuuri
 
 #define INNODB_VERSION_MAJOR	5
 #define INNODB_VERSION_MINOR	7
-#define INNODB_VERSION_BUGFIX	18
+#define INNODB_VERSION_BUGFIX	19
 
 /* The following is the InnoDB version as shown in
 SELECT plugin_version FROM information_schema.plugins;

storage/innobase/row/row0log.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -2419,6 +2419,9 @@ row_log_table_apply_op(
 		next_mrec = mrec + rec_offs_data_size(offsets);
 
 		if (log->table->n_v_cols) {
+			if (next_mrec + 2 > mrec_end) {
+				return(NULL);
+			}
 			next_mrec += mach_read_from_2(next_mrec);
 		}
 
@@ -2457,7 +2460,7 @@ row_log_table_apply_op(
 		rec_init_offsets_temp(mrec, new_index, offsets);
 		next_mrec = mrec + rec_offs_data_size(offsets) + ext_size;
 		if (log->table->n_v_cols) {
-			if (next_mrec + 2 >= mrec_end) {
+			if (next_mrec + 2 > mrec_end) {
 				return(NULL);
 			}
 

storage/innobase/row/row0merge.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2005, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2014, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -1997,6 +1997,8 @@ row_merge_read_clustered_index(
 			goto func_exit;
 		}
 
+		mem_heap_empty(row_heap);
+
 		page_cur_move_to_next(cur);
 
 		stage->n_pk_recs_inc();
@@ -2676,7 +2678,6 @@ write_buffers:
 			goto func_exit;
 		}
 
-		mem_heap_empty(row_heap);
 		if (v_heap) {
 			mem_heap_empty(v_heap);
 		}

storage/innobase/row/row0mysql.cc

@@ -4595,6 +4595,15 @@ row_rename_table_for_mysql(
 	    && dict_table_is_file_per_table(table)) {
 		/* Make a new pathname to update SYS_DATAFILES. */
 		char*	new_path = row_make_new_pathname(table, new_name);
+		char*	old_path = fil_space_get_first_path(table->space);
+
+		/* If old path and new path are the same means tablename
+		has not changed and only the database name holding the table
+		has changed so we need to make the complete filepath again. */
+		if (!dict_tables_have_same_db(old_name, new_name)) {
+			ut_free(new_path);
+			new_path = fil_make_filepath(NULL, new_name, IBD, false);
+		}
 
 		info = pars_info_create();
 
@@ -4614,6 +4623,7 @@ row_rename_table_for_mysql(
 				   "END;\n"
 				   , FALSE, trx);
 
+		ut_free(old_path);
 		ut_free(new_path);
 	}
 	if (err != DB_SUCCESS) {

storage/innobase/row/row0sel.cc

@@ -2981,8 +2981,6 @@ row_sel_field_store_in_mysql_format_func(
 @param[in]	field_no		templ->rec_field_no or
 					templ->clust_rec_field_no
 					or templ->icp_rec_field_no
-					or sec field no if clust_templ_for_sec
-					is TRUE
 @param[in]	templ			row template
 */
 static MY_ATTRIBUTE((warn_unused_result))
@@ -3141,10 +3139,6 @@ be needed in the query.
 @param[in]	rec_clust		whether the rec in the clustered index
 @param[in]	index			index of rec
 @param[in]	offsets			array returned by rec_get_offsets(rec)
-@param[in]	clust_templ_for_sec	TRUE if rec belongs to secondary index
-					but the prebuilt->template is in
-					clustered index format and it is
-					used only for end range comparison
 @return TRUE on success, FALSE if not all columns could be retrieved */
 static MY_ATTRIBUTE((warn_unused_result))
 ibool

storage/innobase/row/row0trunc.cc

@@ -2018,7 +2018,7 @@ row_truncate_table_for_mysql(
 			space_size -= ib_vector_size(table->fts->indexes);
 		}
 
-		fil_reinit_space_header(table->space, space_size, trx);
+		fil_reinit_space_header_for_table(table, space_size, trx);
 	}
 
 	DBUG_EXECUTE_IF("ib_trunc_crash_with_intermediate_log_checkpoint",

storage/innobase/row/row0upd.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2015, 2016, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -256,6 +256,9 @@ row_upd_check_references_constraints(
 		row_mysql_freeze_data_dictionary(trx);
 	}
 
+	DEBUG_SYNC_C_IF_THD(thr_get_trx(thr)->mysql_thd,
+			    "foreign_constraint_check_for_insert");
+
 	for (dict_foreign_set::iterator it = table->referenced_set.begin();
 	     it != table->referenced_set.end();
 	     ++it) {
@@ -283,6 +286,34 @@ row_upd_check_references_constraints(
 					FALSE, FALSE, DICT_ERR_IGNORE_NONE);
 			}
 
+			/* dict_operation_lock is held both here
+			(UPDATE or DELETE with FOREIGN KEY) and by TRUNCATE
+			TABLE operations.
+			If a TRUNCATE TABLE operation is in progress,
+			there can be 2 possible conditions:
+			1) row_truncate_table_for_mysql() is not yet called.
+			2) Truncate releases dict_operation_lock
+			during eviction of pages from buffer pool
+			for a file-per-table tablespace.
+
+			In case of (1), truncate will wait for FK operation
+			to complete.
+			In case of (2), truncate will be rolled forward even
+			if it is interrupted. So if the foreign table is
+			undergoing a truncate, ignore the FK check. */
+
+			if (foreign_table) {
+				mutex_enter(&fil_system->mutex);
+				const fil_space_t* space = fil_space_get_by_id(
+					foreign_table->space);
+				const bool being_truncated = space
+					&& space->is_being_truncated;
+				mutex_exit(&fil_system->mutex);
+				if (being_truncated) {
+					continue;
+				}
+			}
+
 			/* NOTE that if the thread ends up waiting for a lock
 			we will release dict_operation_lock temporarily!
 			But the counter on the table protects 'foreign' from