Merge 10.5 into 10.6

extra/mariabackup/xtrabackup.cc

@@ -3445,7 +3445,9 @@ static void xb_load_single_table_tablespace(const char *dirname,
 	if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) {
 		space = fil_space_t::create(
 			file->space_id(), file->flags(),
-			FIL_TYPE_TABLESPACE, NULL/* TODO: crypt_data */);
+			FIL_TYPE_TABLESPACE, nullptr/* TODO: crypt_data */,
+			FIL_ENCRYPTION_DEFAULT,
+			file->handle() != OS_FILE_CLOSED);
 
 		ut_a(space != NULL);
 		fil_node_t* node= space->add(
@@ -5312,7 +5314,8 @@ exit:
 	ut_ad(fil_space_t::physical_size(flags) == info.page_size);
 
 	if (fil_space_t::create(info.space_id, flags,
-				FIL_TYPE_TABLESPACE, 0)) {
+				FIL_TYPE_TABLESPACE, 0, FIL_ENCRYPTION_DEFAULT,
+				true)) {
 		*success = xb_space_create_file(real_name, info.space_id,
 						flags, &file);
 	} else {

mysql-test/suite/encryption/t/innodb-key-rotation-disable.test

@@ -31,8 +31,10 @@ insert into t8 values (1, 'publicmessage');
 insert into t9 values (1, 'pugliccompressedaaaaaaaaabbbbbbbbbbbbbbccccccccccccccc');
 
 --echo # should list tables t1-t6
+--sorted_result
 SELECT NAME,ENCRYPTION_SCHEME,CURRENT_KEY_ID FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'enctests%';
 --echo # should list tables t7-t9
+--sorted_result
 SELECT NAME,ENCRYPTION_SCHEME,CURRENT_KEY_ID FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 and NAME LIKE 'enctests%';
 
 --let $MYSQLD_DATADIR=`select @@datadir`

mysql-test/suite/encryption/t/innodb_encryption_is.test

@@ -9,6 +9,7 @@ INSERT INTO t2 VALUES ('foobar');
 #
 # MDEV-9640: Add used key_id to INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION
 #
+--sorted_result
 SELECT NAME, ENCRYPTION_SCHEME, MIN_KEY_VERSION, CURRENT_KEY_VERSION,
 CURRENT_KEY_ID
 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION

storage/innobase/fil/fil0fil.cc

@@ -122,6 +122,9 @@ bool fil_space_t::try_to_close(bool print_info)
     }
 
     node->close();
+
+    fil_system.move_closed_last_to_space_list(node->space);
+
     return true;
   }
 
@@ -392,13 +395,7 @@ static bool fil_node_open_file_low(fil_node_t *node)
 
   ut_ad(node->is_open());
 
-  if (UNIV_LIKELY(!fil_system.freeze_space_list))
-  {
-    /* Move the file last in fil_system.space_list, so that
-    fil_space_t::try_to_close() should close it as a last resort. */
-    fil_system.space_list.erase(space_list_t::iterator(node->space));
-    fil_system.space_list.push_back(*node->space);
-  }
+  fil_system.move_opened_last_to_space_list(node->space);
 
   fil_system.n_open++;
   return true;
@@ -795,7 +792,17 @@ pfs_os_file_t fil_system_t::detach(fil_space_t *space, bool detach_handle)
     space->is_in_default_encrypt= false;
     default_encrypt_tables.remove(*space);
   }
-  space_list.erase(space_list_t::iterator(space));
+
+  {
+    space_list_t::iterator s= space_list_t::iterator(space);
+    if (space_list_last_opened == space)
+    {
+      space_list_t::iterator prev= s;
+      space_list_last_opened= &*--prev;
+    }
+    space_list.erase(s);
+  }
+
   if (space == sys_space)
     sys_space= nullptr;
   else if (space == temp_space)
@@ -913,12 +920,14 @@ fil_space_free(
 @param purpose    tablespace purpose
 @param crypt_data encryption information
 @param mode       encryption mode
+@param opened     true if space files are opened
 @return pointer to created tablespace, to be filled in with add()
 @retval nullptr on failure (such as when the same tablespace exists) */
 fil_space_t *fil_space_t::create(ulint id, ulint flags,
                                  fil_type_t purpose,
 				 fil_space_crypt_t *crypt_data,
-				 fil_encryption_t mode)
+				 fil_encryption_t mode,
+				 bool opened)
 {
 	fil_space_t*	space;
 
@@ -971,7 +980,10 @@ fil_space_t *fil_space_t::create(ulint id, ulint flags,
 
 	HASH_INSERT(fil_space_t, hash, &fil_system.spaces, id, space);
 
-	fil_system.space_list.push_back(*space);
+	if (opened)
+	  fil_system.add_opened_last_to_space_list(space);
+	else
+          fil_system.space_list.push_back(*space);
 
 	switch (id) {
 	case 0:
@@ -1296,6 +1308,15 @@ void fil_system_t::close()
 #endif /* __linux__ */
 }
 
+void fil_system_t::add_opened_last_to_space_list(fil_space_t *space)
+{
+  if (UNIV_LIKELY(space_list_last_opened != nullptr))
+    space_list.insert(space_list_t::iterator(space_list_last_opened), *space);
+  else
+    space_list.push_back(*space);
+  space_list_last_opened= space;
+}
+
 /** Extend all open data files to the recovered size */
 ATTRIBUTE_COLD void fil_system_t::extend_to_recv_size()
 {
@@ -1963,7 +1984,7 @@ err_exit:
 
 	if (fil_space_t* space = fil_space_t::create(space_id, flags,
 						     FIL_TYPE_TABLESPACE,
-						     crypt_data, mode)) {
+						     crypt_data, mode, true)) {
 		fil_node_t* node = space->add(path, file, size, false, true);
 		IF_WIN(node->find_metadata(), node->find_metadata(file, true));
 		mtr.start();

storage/innobase/include/fil0fil.h

@@ -958,11 +958,13 @@ public:
   @param purpose    tablespace purpose
   @param crypt_data encryption information
   @param mode       encryption mode
+  @param opened     true if space files are opened
   @return pointer to created tablespace, to be filled in with add()
   @retval nullptr on failure (such as when the same tablespace exists) */
   static fil_space_t *create(ulint id, ulint flags,
                              fil_type_t purpose, fil_space_crypt_t *crypt_data,
-                             fil_encryption_t mode= FIL_ENCRYPTION_DEFAULT);
+                             fil_encryption_t mode= FIL_ENCRYPTION_DEFAULT,
+                             bool opened= false);
 
   MY_ATTRIBUTE((warn_unused_result))
   /** Acquire a tablespace reference.
@@ -1447,6 +1449,11 @@ struct fil_system_t {
 
 private:
   bool m_initialised;
+
+  /** Points to the last opened space in space_list. Protected with
+  fil_system.mutex. */
+  fil_space_t *space_list_last_opened= nullptr;
+
 #ifdef __linux__
   /** available block devices that reside on non-rotational storage */
   std::vector<dev_t> ssd;
@@ -1492,8 +1499,10 @@ public:
   /** nonzero if fil_node_open_file_low() should avoid moving the tablespace
   to the end of space_list, for FIFO policy of try_to_close() */
   ulint freeze_space_list;
+  /** List of all file spaces, opened spaces should be at the top of the list
+  to optimize try_to_close() execution. Protected with fil_system.mutex. */
   ilist<fil_space_t, space_list_tag_t> space_list;
-					/*!< list of all file spaces */
+
   ilist<fil_space_t, named_spaces_tag_t> named_spaces;
 					/*!< list of all file spaces
 					for which a FILE_MODIFY
@@ -1509,6 +1518,49 @@ public:
 					has issued a warning about
 					potential space_id reuse */
 
+  /** Add the file to the end of opened spaces list in
+  fil_system.space_list, so that fil_space_t::try_to_close() should close
+  it as a last resort.
+  @param space space to add */
+  void add_opened_last_to_space_list(fil_space_t *space);
+
+  /** Move the file to the end of opened spaces list in
+  fil_system.space_list, so that fil_space_t::try_to_close() should close
+  it as a last resort.
+  @param space space to move */
+  inline void move_opened_last_to_space_list(fil_space_t *space)
+  {
+    /* In the case when several files of the same space are added in a
+    row, there is no need to remove and add a space to the same position
+    in space_list. It can be for system or temporary tablespaces. */
+    if (freeze_space_list || space_list_last_opened == space)
+      return;
+
+    space_list.erase(space_list_t::iterator(space));
+    add_opened_last_to_space_list(space);
+  }
+
+  /** Move closed file last in fil_system.space_list, so that
+  fil_space_t::try_to_close() iterates opened files first in FIFO order,
+  i.e. first opened, first closed.
+  @param space space to move */
+  void move_closed_last_to_space_list(fil_space_t *space)
+  {
+    if (UNIV_UNLIKELY(freeze_space_list))
+      return;
+
+    space_list_t::iterator s= space_list_t::iterator(space);
+
+    if (space_list_last_opened == space)
+    {
+      space_list_t::iterator prev= s;
+      space_list_last_opened= &*--prev;
+    }
+
+    space_list.erase(s);
+    space_list.push_back(*space);
+  }
+
   /** Return the next tablespace from default_encrypt_tables list.
   @param space   previous tablespace (nullptr to start from the start)
   @param recheck whether the removal condition needs to be rechecked after

storage/innobase/srv/srv0start.cc

@@ -556,7 +556,8 @@ err_exit:
   fil_set_max_space_id_if_bigger(space_id);
 
   fil_space_t *space= fil_space_t::create(space_id, fsp_flags,
-					  FIL_TYPE_TABLESPACE, NULL);
+					  FIL_TYPE_TABLESPACE, nullptr,
+					  FIL_ENCRYPTION_DEFAULT, true);
   ut_a(fil_validate());
   ut_a(space);
 
@@ -905,9 +906,7 @@ static lsn_t srv_prepare_to_delete_redo_log_file(bool old_exists)
 {
   DBUG_ENTER("srv_prepare_to_delete_redo_log_file");
 
-  /* Disable checkpoints in the page cleaner. */
-  ut_ad(!recv_sys.recovery_on);
-  recv_sys.recovery_on= true;
+  ut_ad(recv_sys.recovery_on);
 
   /* Clean the buffer pool. */
   buf_flush_sync();
@@ -1589,10 +1588,10 @@ file_checked:
 			}
 		}
 
-		recv_sys.debug_free();
-
 		if (srv_operation == SRV_OPERATION_RESTORE
 		    || srv_operation == SRV_OPERATION_RESTORE_EXPORT) {
+			buf_flush_sync();
+			recv_sys.debug_free();
 			/* After applying the redo log from
 			SRV_OPERATION_BACKUP, flush the changes
 			to the data files and truncate or delete the log.
@@ -1684,6 +1683,8 @@ file_checked:
 				return(srv_init_abort(err));
 			}
 		}
+
+		recv_sys.debug_free();
 	}
 
 	ut_ad(err == DB_SUCCESS);