MDEV-24695 Encryption modifies a freed page

During recovery, InnoDB fails if it tries to apply a FREE_PAGE
and WRITE record to the page. InnoDB encryption thread accesses
the freed page and writes redo log for it.

This is similar to commit deadec4e68 (MDEV-24569)
InnoDB is missing buf_page_free() while freeing the segment.
To avoid accessing of freed page in buffer pool, InnoDB should
mark the pages as FREED while freeing the segment. Also to
avoid reading of freed page, InnoDB should check the
allocation bitmap page.

fseg_free_step(): Mark the page in buffer pool as FREED

fseg_free_step_not_header(): Mark the page in buffer pool as FREED

buf_dump(): Ignore the freed pages while dumping the buffer pool content

fil_crypt_get_page_throttle_func(): Skip the rotation for FREED page
to avoid the assert failure during recovery

fil_crypt_rotate_page(): Skip the rotation for the freed page

Reviewed-by: Marko Mäkelä

mysql-test/suite/encryption/r/innodb_encrypt_freed.result

@@ -0,0 +1,96 @@
+SHOW VARIABLES LIKE 'innodb_encrypt%';
+Variable_name	Value
+innodb_encrypt_log	ON
+innodb_encrypt_tables	ON
+innodb_encrypt_temporary_tables	OFF
+innodb_encryption_rotate_key_age	1
+innodb_encryption_rotation_iops	100
+innodb_encryption_threads	1
+SET GLOBAL innodb_encrypt_tables = ON;
+CREATE TABLE t1(f1 BIGINT PRIMARY KEY, f2 int not null,
+f3 int not null, index(f1), index idx_1(f2),
+index(f2, f3)) ENGINE=InnoDB;
+# Wait max 10 min for key encryption threads to encrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+CREATE TABLE t2 (f1 int not null)engine=innodb;
+# restart: --debug=d,ib_log_checkpoint_avoid
+connect con1,localhost,root,,,;
+begin;
+insert into t2 values(1);
+connection default;
+set global innodb_encrypt_tables = OFF;
+# Wait max 10 min for key encryption threads to decrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+test/t2
+alter table t1 drop index idx_1;
+set global innodb_encrypt_tables = ON;
+# Wait max 10 min for key encryption threads to encrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+test/t2
+disconnect con1;
+# restart: --debug=d,ib_log_checkpoint_avoid
+drop table t1, t2;
+CREATE TABLE t1(f1 BIGINT PRIMARY KEY, f2 int not null,
+f3 int not null, index(f1), index idx_1(f2),
+index(f2, f3)) ENGINE=InnoDB;
+# Wait max 10 min for key encryption threads to encrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+CREATE TABLE t2 (f1 int not null)engine=innodb;
+# restart: --debug=d,ib_log_checkpoint_avoid
+connect con1,localhost,root,,,;
+begin;
+insert into t2 values(1);
+connection default;
+set global innodb_encrypt_tables = OFF;
+# Wait max 10 min for key encryption threads to decrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+test/t2
+alter table t1 drop index idx_1;
+disconnect con1;
+# restart: --debug=d,ib_log_checkpoint_avoid
+connect con1,localhost,root,,,;
+begin;
+insert into t2 values(1);
+connection default;
+set global innodb_encrypt_tables = ON;
+# Wait max 10 min for key encryption threads to encrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+test/t2
+disconnect con1;
+drop table t2, t1;

mysql-test/suite/encryption/t/innodb_encrypt_freed.opt

@@ -0,0 +1,4 @@
+--innodb-encrypt-tables
+--innodb-encrypt-log
+--innodb-encryption-threads=1
+--innodb-tablespaces-encryption

mysql-test/suite/encryption/t/innodb_encrypt_freed.test

@@ -0,0 +1,117 @@
+--source include/have_innodb.inc
+--source include/have_example_key_management_plugin.inc
+--source include/have_debug.inc
+--source include/not_embedded.inc
+
+SHOW VARIABLES LIKE 'innodb_encrypt%';
+
+SET GLOBAL innodb_encrypt_tables = ON;
+
+CREATE TABLE t1(f1 BIGINT PRIMARY KEY, f2 int not null,
+		f3 int not null, index(f1), index idx_1(f2),
+		index(f2, f3)) ENGINE=InnoDB;
+--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+
+--echo # Wait max 10 min for key encryption threads to encrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+
+CREATE TABLE t2 (f1 int not null)engine=innodb;
+let $restart_parameters="--debug=d,ib_log_checkpoint_avoid";
+--source include/restart_mysqld.inc
+
+# Stop the purge
+connect(con1,localhost,root,,,);
+begin;
+insert into t2 values(1);
+
+connection default;
+
+--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+set global innodb_encrypt_tables = OFF;
+
+--echo # Wait max 10 min for key encryption threads to decrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+
+# Free the index `idx_1`
+alter table t1 drop index idx_1;
+
+set global innodb_encrypt_tables = ON;
+
+--echo # Wait max 10 min for key encryption threads to encrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+
+disconnect con1;
+let $shutdown_timeout=0;
+--source include/restart_mysqld.inc
+
+drop table t1, t2;
+
+#
+#
+CREATE TABLE t1(f1 BIGINT PRIMARY KEY, f2 int not null,
+		f3 int not null, index(f1), index idx_1(f2),
+		index(f2, f3)) ENGINE=InnoDB;
+--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+
+--echo # Wait max 10 min for key encryption threads to encrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+
+CREATE TABLE t2 (f1 int not null)engine=innodb;
+--source include/restart_mysqld.inc
+
+# Stop the purge
+connect(con1,localhost,root,,,);
+begin;
+insert into t2 values(1);
+
+connection default;
+
+--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+set global innodb_encrypt_tables = OFF;
+
+--echo # Wait max 10 min for key encryption threads to decrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+
+# Free the index `idx_1`
+alter table t1 drop index idx_1;
+
+disconnect con1;
+--source include/restart_mysqld.inc
+
+# Stop the purge
+connect(con1,localhost,root,,,);
+begin;
+insert into t2 values(1);
+
+connection default;
+set global innodb_encrypt_tables = ON;
+
+--echo # Wait max 10 min for key encryption threads to encrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+
+disconnect con1;
+drop table t2, t1;

mysql-test/suite/encryption/t/innodb_encryption.test

@@ -14,7 +14,7 @@ SHOW VARIABLES LIKE 'innodb_encrypt%';
 
 SET GLOBAL innodb_encrypt_tables = ON;
 
---let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+--let $tables_count= `select count(*) + @@global.innodb_undo_tablespaces + 1 from information_schema.tables where engine = 'InnoDB'`
 
 --echo # Wait max 10 min for key encryption threads to encrypt all spaces
 --let $wait_timeout= 600

storage/innobase/buf/buf0dump.cc

@@ -332,6 +332,10 @@ buf_dump(
 			continue;
 		}
 
+		if (bpage->status == buf_page_t::FREED) {
+			continue;
+		}
+
 		dump[j++] = id;
 	}
 

storage/innobase/fil/fil0crypt.cc

@@ -1771,6 +1771,11 @@ fil_crypt_get_page_throttle_func(
 		return NULL;
 	}
 
+	if (fseg_page_is_free(space, state->offset)) {
+		/* page is already freed */
+		return NULL;
+	}
+
 	state->crypt_stat.pages_read_from_disk++;
 
 	const ulonglong start = my_interval_timer();
@@ -1874,6 +1879,9 @@ fil_crypt_rotate_page(
 			some dummy pages will be allocated, with 0 in
 			the FIL_PAGE_TYPE. Those pages should be
 			skipped from key rotation forever. */
+		} else if (block->page.status == buf_page_t::FREED) {
+			/* Do not modify freed pages to avoid an assertion
+			failure on recovery.*/
 		} else if (fil_crypt_needs_rotation(
 				crypt_data,
 				kv,

storage/innobase/fsp/fsp0fsp.cc

@@ -2705,10 +2705,11 @@ fseg_free_step(
 		DBUG_RETURN(true);
 	}
 
-	fseg_free_page_low(
-		inode, iblock, space,
-		fseg_get_nth_frag_page_no(inode, n),
-		mtr);
+	page_no_t page_no = fseg_get_nth_frag_page_no(inode, n);
+
+	fseg_free_page_low(inode, iblock, space, page_no, mtr);
+
+	buf_page_free(space, page_no, mtr, __FILE__, __LINE__);
 
 	n = fseg_find_last_used_frag_page_slot(inode);
 
@@ -2770,6 +2771,7 @@ fseg_free_step_not_header(
 	}
 
 	fseg_free_page_low(inode, iblock, space, page_no, mtr);
+	buf_page_free(space, page_no, mtr, __FILE__, __LINE__);
 	return false;
 }