Merge 10.1 into 10.2

extra/mariabackup/fil_cur.cc

@@ -272,6 +272,96 @@ xb_fil_cur_open(
 	return(XB_FIL_CUR_SUCCESS);
 }
 
+static bool page_is_corrupted(byte *page, ulint page_no, xb_fil_cur_t *cursor,
+			      fil_space_t *space)
+{
+	byte tmp_frame[UNIV_PAGE_SIZE_MAX];
+	byte tmp_page[UNIV_PAGE_SIZE_MAX];
+	const ulint page_size = cursor->page_size.physical();
+	ulint page_type = mach_read_from_2(page + FIL_PAGE_TYPE);
+
+	/* We ignore the doublewrite buffer pages.*/
+	if (cursor->space_id == TRX_SYS_SPACE
+	    && page_no >= FSP_EXTENT_SIZE
+	    && page_no < FSP_EXTENT_SIZE * 3) {
+		return false;
+	}
+
+	/* Validate page number. */
+	if (mach_read_from_4(page + FIL_PAGE_OFFSET) != page_no
+	    && cursor->space_id != TRX_SYS_SPACE) {
+		/* On pages that are not all zero, the
+		page number must match.
+
+		There may be a mismatch on tablespace ID,
+		because files may be renamed during backup.
+		We disable the page number check
+		on the system tablespace, because it may consist
+		of multiple files, and here we count the pages
+		from the start of each file.)
+
+		The first 38 and last 8 bytes are never encrypted. */
+		const ulint* p = reinterpret_cast<ulint*>(page);
+		const ulint* const end = reinterpret_cast<ulint*>(
+			page + page_size);
+		do {
+			if (*p++) {
+				return true;
+			}
+		} while (p != end);
+
+		/* Whole zero page is valid. */
+		return false;
+	}
+
+	/* Validate encrypted pages. */
+	if (mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
+	    && (space->crypt_data
+		&& space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED)) {
+
+		if (!fil_space_verify_crypt_checksum(page, cursor->page_size))
+			return true;
+
+		/* Compressed encrypted need to be decrypted
+		and decompressed for verification. */
+		if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED
+		    && !opt_extended_validation)
+			return false;
+
+		memcpy(tmp_page, page, page_size);
+
+		bool decrypted = false;
+		if (!fil_space_decrypt(space, tmp_frame, tmp_page, &decrypted)) {
+			return true;
+		}
+
+		if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+			return buf_page_is_corrupted(true, tmp_page,
+						     cursor->page_size, space);
+		}
+	}
+
+	if (page_type == FIL_PAGE_PAGE_COMPRESSED) {
+		memcpy(tmp_page, page, page_size);
+	}
+
+	if (page_type == FIL_PAGE_PAGE_COMPRESSED
+	    || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+		ulint decomp = fil_page_decompress(tmp_frame, tmp_page);
+		page_type = mach_read_from_2(tmp_page + FIL_PAGE_TYPE);
+
+		return (!decomp
+			|| (decomp != srv_page_size
+			    && cursor->page_size.is_compressed())
+			|| page_type == FIL_PAGE_PAGE_COMPRESSED
+			|| page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED
+			|| buf_page_is_corrupted(true, tmp_page,
+						 cursor->page_size, space));
+	}
+
+	return buf_page_is_corrupted(true, page, cursor->page_size, space);
+}
+
 /************************************************************************
 Reads and verifies the next block of pages from the source
 file. Positions the cursor after the last read non-corrupted page.
@@ -290,8 +380,6 @@ xb_fil_cur_read(
 	xb_fil_cur_result_t	ret;
 	ib_int64_t		offset;
 	ib_int64_t		to_read;
-	byte			tmp_frame[UNIV_PAGE_SIZE_MAX];
-	byte			tmp_page[UNIV_PAGE_SIZE_MAX];
 	const ulint		page_size = cursor->page_size.physical();
 	xb_ad(!cursor->is_system() || page_size == UNIV_PAGE_SIZE);
 
@@ -358,103 +446,29 @@ read_retry:
 	for (page = cursor->buf, i = 0; i < npages;
 	     page += page_size, i++) {
 		ulint page_no = cursor->buf_page_no + i;
-		ulint page_type = mach_read_from_2(page + FIL_PAGE_TYPE);
 
-		if (cursor->space_id == TRX_SYS_SPACE
+		if (page_is_corrupted(page, page_no, cursor, space)){
-		    && page_no >= FSP_EXTENT_SIZE
+			retry_count--;
-		    && page_no < FSP_EXTENT_SIZE * 3) {
-			/* We ignore the doublewrite buffer pages */
-                } else if (mach_read_from_4(page + FIL_PAGE_OFFSET) != page_no
-			   && space->id != TRX_SYS_SPACE) {
-			/* On pages that are not all zero, the
-			page number must match.
 
-			There may be a mismatch on tablespace ID,
+			if (retry_count == 0) {
-			because files may be renamed during backup.
+				msg("[%02u] mariabackup: "
-			We disable the page number check
+				    "Error: failed to read page after "
-			on the system tablespace, because it may consist
+				    "10 retries. File %s seems to be "
-			of multiple files, and here we count the pages
+				    "corrupted.\n", cursor->thread_n,
-			from the start of each file.)
+				    cursor->abs_path);
-
+				ret = XB_FIL_CUR_ERROR;
-			The first 38 and last 8 bytes are never encrypted. */
+				break;
-			const ulint* p = reinterpret_cast<ulint*>(page);
-			const ulint* const end = reinterpret_cast<ulint*>(
-				page + page_size);
-			do {
-				if (*p++) {
-					goto corrupted;
-				}
-			} while (p != end);
-		} else if (mach_read_from_4(
-				   page
-				   + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
-			   && space->crypt_data
-			   && space->crypt_data->type
-			   != CRYPT_SCHEME_UNENCRYPTED
-			   && fil_space_verify_crypt_checksum(
-                                   page, cursor->page_size)) {
-			bool decrypted = false;
-
-			memcpy(tmp_page, page, page_size);
-
-			if (!fil_space_decrypt(space, tmp_frame,
-					       tmp_page, &decrypted)) {
-				goto corrupted;
 			}
+			msg("[%02u] mariabackup: "
+			    "Database page corruption detected at page "
+			    ULINTPF ", retrying...\n", cursor->thread_n,
+			    page_no);
 
-			if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+			os_thread_sleep(100000);
-				goto page_decomp;
+			goto read_retry;
-			}
+		}
-
+		cursor->buf_read += page_size;
-			if (buf_page_is_corrupted(true, tmp_page,
+		cursor->buf_npages++;
-						  cursor->page_size, space)) {
-				goto corrupted;
-			}
-		} else if (page_type == FIL_PAGE_PAGE_COMPRESSED) {
-			memcpy(tmp_page, page, page_size);
-page_decomp:
-			ulint decomp = fil_page_decompress(tmp_frame, tmp_page);
-			page_type = mach_read_from_2(tmp_page + FIL_PAGE_TYPE);
-
-			if (!decomp
-			    || (decomp != srv_page_size
-				&& cursor->page_size.is_compressed())
-			    || page_type == FIL_PAGE_PAGE_COMPRESSED
-			    || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED
-			    || buf_page_is_corrupted(true, tmp_page,
-						     cursor->page_size,
-						     space)) {
-				goto corrupted;
-			}
-
-		} else if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED
-			   || buf_page_is_corrupted(true, page,
-						    cursor->page_size,
-						    space)) {
-corrupted:
-                        retry_count--;
-                        if (retry_count == 0) {
-                                msg("[%02u] mariabackup: "
-                                    "Error: failed to read page after "
-                                    "10 retries. File %s seems to be "
-                                    "corrupted.\n", cursor->thread_n,
-                                    cursor->abs_path);
-                                ret = XB_FIL_CUR_ERROR;
-                                break;
-                        }
-
-                        if (retry_count == 9) {
-                                msg("[%02u] mariabackup: "
-                                    "Database page corruption detected at page "
-                                    ULINTPF ", retrying...\n",
-                                    cursor->thread_n, page_no);
-                        }
-
-                        os_thread_sleep(100000);
-                        goto read_retry;
-                }
-                cursor->buf_read += page_size;
-                cursor->buf_npages++;
 	}
 
 	posix_fadvise(cursor->file, offset, to_read, POSIX_FADV_DONTNEED);

extra/mariabackup/xtrabackup.cc

@@ -200,6 +200,7 @@ static char*	log_ignored_opt;
 
 extern my_bool opt_use_ssl;
 my_bool opt_ssl_verify_server_cert;
+my_bool opt_extended_validation;
 
 /* === metadata of backup === */
 #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints"
@@ -760,6 +761,7 @@ enum options_xtrabackup
   OPT_XTRA_DATABASES,
   OPT_XTRA_DATABASES_FILE,
   OPT_XTRA_PARALLEL,
+  OPT_XTRA_EXTENDED_VALIDATION,
   OPT_XTRA_STREAM,
   OPT_XTRA_COMPRESS,
   OPT_XTRA_COMPRESS_THREADS,
@@ -1220,6 +1222,14 @@ struct my_option xb_server_options[] =
    (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT,
    REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
 
+  {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION,
+   "Enable extended validation for Innodb data pages during backup phase."
+   "Will slow down backup considerably, in case encryption is used.",
+   (G_PTR*)&opt_extended_validation,
+   (G_PTR*)&opt_extended_validation,
+   0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0},
+
+
    {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
    (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0,
    GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},

extra/mariabackup/xtrabackup.h

@@ -111,6 +111,7 @@ extern my_bool		opt_noversioncheck;
 extern my_bool		opt_no_backup_locks;
 extern my_bool		opt_decompress;
 extern my_bool		opt_remove_original;
+extern my_bool		opt_extended_validation;
 extern my_bool		opt_lock_ddl_per_table;
 
 extern char		*opt_incremental_history_name;

mysql-test/r/grant5.result

@@ -18,3 +18,10 @@ ERROR 42000: Access denied for user 'test'@'%' to database 'mysql'
 connection default;
 drop user test, foo;
 drop role foo;
+CREATE TABLE t1 (a INT);
+LOCK TABLE t1 WRITE;
+REVOKE EXECUTE ON PROCEDURE sp FROM u;
+ERROR HY000: Table 'user' was not locked with LOCK TABLES
+REVOKE PROCESS ON *.* FROM u;
+ERROR HY000: Table 'user' was not locked with LOCK TABLES
+DROP TABLE t1;

mysql-test/suite/encryption/r/innodb-missing-key.result

@@ -1,6 +1,6 @@
 call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\.");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
-call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[12].ibd looks corrupted; key_version=1");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file .*test.t[12].ibd looks corrupted; key_version=1");
 # Start server with keys2.txt
 CREATE TABLE t1(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=19;
 CREATE TABLE t2(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=1;

mysql-test/suite/encryption/t/innodb-missing-key.test

@@ -9,7 +9,7 @@
 
 call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\.");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
-call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[12].ibd looks corrupted; key_version=1");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file .*test.t[12].ibd looks corrupted; key_version=1");
 
 --echo # Start server with keys2.txt
 -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt

mysql-test/suite/mariabackup/encrypted_page_compressed.result

@@ -2,5 +2,5 @@ CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=I
 insert into t1(b, c) values("mariadb", "mariabackup");
 # Corrupt the table
 # xtrabackup backup
-FOUND 1 /Database page corruption detected/ in backup.log
+FOUND 1 /Database page corruption detected.*/ in backup.log
 drop table t1;

mysql-test/suite/mariabackup/encrypted_page_compressed.test

@@ -38,7 +38,7 @@ let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log;
 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $backuplog;
 --enable_result_log
 
---let SEARCH_PATTERN=Database page corruption detected
+--let SEARCH_PATTERN=Database page corruption detected.*
 --let SEARCH_FILE=$backuplog
 --source include/search_pattern_in_file.inc
 remove_file $backuplog;

mysql-test/suite/mariabackup/encrypted_page_corruption.result

@@ -3,5 +3,5 @@ CREATE TABLE t1(c VARCHAR(128)) ENGINE INNODB, encrypted=yes;
 insert into t1 select repeat('a',100);
 # Corrupt the table
 # xtrabackup backup
-FOUND 1 /Database page corruption detected/ in backup.log
+FOUND 1 /Database page corruption detected.*/ in backup.log
 drop table t1;

mysql-test/suite/mariabackup/encrypted_page_corruption.test

@@ -50,14 +50,20 @@ let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log;
 
 --disable_result_log
 --error 1
-exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $backuplog;
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --extended-validation --target-dir=$targetdir > $backuplog;
 --enable_result_log
 
 
---let SEARCH_PATTERN=Database page corruption detected
+--let SEARCH_PATTERN=Database page corruption detected.*
 --let SEARCH_FILE=$backuplog
 --source include/search_pattern_in_file.inc
 remove_file $backuplog;
+rmdir $targetdir;
+
+# Due to very constructed nature of the "corruption" (faking checksums), the "corruption" won't be found without --extended-validation
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir;
+--enable_result_log
 
 drop table t1;
 rmdir $targetdir;

mysql-test/suite/mariabackup/unencrypted_page_compressed.result

@@ -4,5 +4,5 @@ insert into t1(b, c) values("mariadb", "mariabackup");
 InnoDB		0 transactions not purged
 # Corrupt the table
 # xtrabackup backup
-FOUND 1 /Database page corruption detected/ in backup.log
+FOUND 1 /Database page corruption detected.*/ in backup.log
 drop table t1;

mysql-test/suite/mariabackup/unencrypted_page_compressed.test

@@ -40,7 +40,7 @@ let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log;
 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $backuplog;
 --enable_result_log
 
---let SEARCH_PATTERN=Database page corruption detected
+--let SEARCH_PATTERN=Database page corruption detected.*
 --let SEARCH_FILE=$backuplog
 --source include/search_pattern_in_file.inc
 remove_file $backuplog;

mysql-test/t/grant5.test

@@ -23,3 +23,13 @@ show grants for foo@'%'; # user
 drop user test, foo;
 drop role foo;
 
+#
+# MDEV-17975 Assertion `! is_set()' or `!is_set() || (m_status == DA_OK_BULK && is_bulk_op())' failed upon REVOKE under LOCK TABLE
+#
+CREATE TABLE t1 (a INT);
+LOCK TABLE t1 WRITE;
+--error ER_TABLE_NOT_LOCKED
+REVOKE EXECUTE ON PROCEDURE sp FROM u;
+--error ER_TABLE_NOT_LOCKED
+REVOKE PROCESS ON *.* FROM u;
+DROP TABLE t1;

sql/sql_acl.cc

@@ -6668,7 +6668,8 @@ bool mysql_routine_grant(THD *thd, TABLE_LIST *table_list, bool is_proc,
 {
   List_iterator <LEX_USER> str_list (user_list);
   LEX_USER *Str, *tmp_Str;
-  bool create_new_users= 0, result;
+  bool create_new_users= 0;
+  int result;
   char *db_name, *table_name;
   DBUG_ENTER("mysql_routine_grant");
 
@@ -7106,7 +7107,8 @@ bool mysql_grant(THD *thd, const char *db, List <LEX_USER> &list,
   List_iterator <LEX_USER> str_list (list);
   LEX_USER *Str, *tmp_Str, *proxied_user= NULL;
   char tmp_db[SAFE_NAME_LEN+1];
-  bool create_new_users=0, result;
+  bool create_new_users=0;
+  int result;
   DBUG_ENTER("mysql_grant");
 
   if (lower_case_table_names && db)

storage/innobase/fil/fil0pagecompress.cc

@@ -334,14 +334,14 @@ ulint fil_page_decompress(byte* tmp_buf, byte* buf)
 	case PAGE_ZLIB_ALGORITHM:
 		{
 			uLong len = srv_page_size;
-			if (Z_OK != uncompress(tmp_buf, &len,
+			if (Z_OK == uncompress(tmp_buf, &len,
 					       buf + header_len,
 					       uLong(actual_size))
-			    && len != srv_page_size) {
+			    && len == srv_page_size) {
-				return 0;
+				break;
 			}
 		}
-		break;
+		return 0;
 #ifdef HAVE_LZ4
 	case PAGE_LZ4_ALGORITHM:
 		if (LZ4_decompress_safe(reinterpret_cast<const char*>(buf)

storage/xtradb/fil/fil0pagecompress.cc

@@ -341,14 +341,14 @@ UNIV_INTERN ulint fil_page_decompress(byte* tmp_buf, byte* buf)
 	case PAGE_ZLIB_ALGORITHM:
 		{
 			uLong len = srv_page_size;
-			if (Z_OK != uncompress(tmp_buf, &len,
+			if (Z_OK == uncompress(tmp_buf, &len,
 					       buf + header_len,
 					       uLong(actual_size))
-			    && len != srv_page_size) {
+			    && len == srv_page_size) {
-				return 0;
+				break;
 			}
 		}
-		break;
+		return 0;
 #ifdef HAVE_LZ4
 	case PAGE_LZ4_ALGORITHM:
 		if (LZ4_decompress_safe(reinterpret_cast<const char*>(buf)