MDEV-35398 Improve shrinking of system tablespace

Problem:
========
From 10.6.13(86767bcc0f) version,
InnoDB purge thread does free the TRX_UNDO_CACHED undo segment.
Pre-10.6.13 version data directory can contain TRX_UNDO_CACHED
undo segment in system tablespace even though it
has external undo tablespace.

During slow shutdown, InnoDB collects the segment from tables
that exist in system tablespace and cached undo segment in
the system tablespace as used segment exist in system tablespace.
While shrinking the system tablespace, last used extent can be
used by undo cached segment. This extent blocks the shrinking of
system tablespace in a effective way.

Solution:
========
While freeing the unused segment, InnoDB should free the cached
undo segment header page exists in system tablespace and
reset the TRX_RSEG_UNDO_SLOTS to 0xff for the rollback segment
header page exists in system tablespace. This could improve
the shrinking of system tablespace further.

mysql-test/suite/innodb/r/shrink_cached_undo.result

@@ -0,0 +1,23 @@
+#
+# MDEV-35398 Improve shrinking of system tablespace
+#
+# restart: --debug_dbug=d,skip_cached_undo
+SET GLOBAL innodb_file_per_table= 0;
+Warnings:
+Warning	1287	'@@innodb_file_per_table' is deprecated and will be removed in a future release
+CREATE TABLE t1(f1 INT PRIMARY KEY)ENGINE=InnoDB;
+CREATE TABLE t2(f1 INT PRIMARY KEY)ENGINE=InnoDB;
+SET STATEMENT unique_checks=0,foreign_key_checks=0 FOR
+INSERT INTO t1 SELECT seq FROM seq_1_to_24576;
+SET STATEMENT unique_checks=0,foreign_key_checks=0 FOR
+INSERT INTO t2 SELECT seq FROM seq_1_to_24576;
+# Insert 34 transaction which has undo cached records
+DROP TABLE t2, t1;
+SET GLOBAL innodb_fast_shutdown=0;
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	79691776
+# restart
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+NAME	FILE_SIZE
+innodb_system	12582912

mysql-test/suite/innodb/t/shrink_cached_undo.opt

@@ -0,0 +1,2 @@
+--innodb_undo_tablespaces=0
+--innodb_sys_tablespaces

mysql-test/suite/innodb/t/shrink_cached_undo.test

@@ -0,0 +1,47 @@
+--source include/have_innodb.inc
+--source include/have_sequence.inc
+--source include/have_debug.inc
+--source include/not_embedded.inc
+--source include/big_test.inc
+--echo #
+--echo # MDEV-35398 Improve shrinking of system tablespace
+--echo #
+# Make pre-10.6.13 version data directory which skips the
+# purging of cached undo log records
+let $restart_parameters=--debug_dbug=d,skip_cached_undo;
+--source include/restart_mysqld.inc
+SET GLOBAL innodb_file_per_table= 0;
+CREATE TABLE t1(f1 INT PRIMARY KEY)ENGINE=InnoDB;
+CREATE TABLE t2(f1 INT PRIMARY KEY)ENGINE=InnoDB;
+
+SET STATEMENT unique_checks=0,foreign_key_checks=0 FOR
+INSERT INTO t1 SELECT seq FROM seq_1_to_24576;
+SET STATEMENT unique_checks=0,foreign_key_checks=0 FOR
+INSERT INTO t2 SELECT seq FROM seq_1_to_24576;
+
+--echo # Insert 34 transaction which has undo cached records
+--disable_query_log
+let $c = 34;
+while ($c)
+{
+  connect (con$c,localhost,root,,);
+  START TRANSACTION;
+  eval INSERT INTO t1 SELECT seq+$c*32768 FROM seq_1_to_3753;
+  dec $c;
+}
+
+connection default;
+let $c = 34;
+while ($c)
+{
+  disconnect con$c;
+  dec $c;
+}
+--enable_query_log
+
+DROP TABLE t2, t1;
+SET GLOBAL innodb_fast_shutdown=0;
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;
+let $restart_parameters=;
+--source include/restart_mysqld.inc
+SELECT NAME, FILE_SIZE FROM information_schema.innodb_sys_tablespaces WHERE SPACE = 0;

storage/innobase/fsp/fsp0fsp.cc

@@ -3851,22 +3851,6 @@ static dberr_t fsp_get_sys_used_segment(inode_info *inodes, mtr_t *mtr)
                               TRX_RSEG_FSEG_HEADER))
         err= DB_CORRUPTION;
       block->page.unfix();
-
-      /* Even after slow shutdown, there is a possiblity that
-      cached undo log can exist. So store the segment as used one */
-      for (trx_undo_t *undo= UT_LIST_GET_FIRST(rseg->undo_cached);
-           undo && err == DB_SUCCESS;
-           undo= UT_LIST_GET_NEXT(undo_list, undo))
-      {
-        block= buf_pool.page_fix(page_id_t{0, undo->hdr_page_no}, &err,
-                                 buf_pool_t::FIX_WAIT_READ);
-        if (!block)
-	  return err;
-	if (!inodes->insert_seg(block->page.frame + TRX_UNDO_SEG_HDR +
-                                TRX_UNDO_FSEG_HEADER))
-          err= DB_CORRUPTION;
-        block->page.unfix();
-      }
     }
   }
 
@@ -4010,6 +3994,33 @@ dberr_t fil_space_t::garbage_collect(bool shutdown)
     return err;
   }
 
+  /* Reset the undo log segments slots in the rollback segment header
+  which exist in system tablespace. Undo cached segment will be
+  treated as unused file segment. These segments will be freed as a
+  part of inode_info::free_segs  */
+  mtr.start();
+  mtr.x_lock_space(fil_system.sys_space);
+  for (trx_rseg_t &rseg : trx_sys.rseg_array)
+  {
+    if (rseg.space == fil_system.sys_space &&
+        UT_LIST_GET_LEN(rseg.undo_cached))
+    {
+      buf_block_t *block=
+        buf_page_get_gen(page_id_t{0, rseg.page_no}, 0,
+                         RW_X_LATCH, nullptr, BUF_GET, &mtr, &err);
+      if (!block)
+      {
+        mtr.commit();
+        return err;
+      }
+
+      mtr.memset(block, TRX_RSEG_UNDO_SLOTS + TRX_RSEG,
+                 TRX_RSEG_N_SLOTS * TRX_RSEG_SLOT_SIZE, 0xff);
+      rseg.reinit(rseg.page_no);
+    }
+  }
+  mtr.commit();
+
   return unused_inodes.free_segs();
 }
 

storage/innobase/trx/trx0purge.cc

@@ -497,6 +497,7 @@ loop:
       if (UNIV_UNLIKELY(mach_read_from_4(TRX_RSEG + TRX_RSEG_FORMAT +
                                          rseg_hdr->page.frame)))
         trx_rseg_format_upgrade(rseg_hdr, &mtr);
+      DBUG_EXECUTE_IF("skip_cached_undo", goto skip_purge_free;);
       if (UNIV_LIKELY(undo != nullptr))
       {
         UT_LIST_REMOVE(rseg.undo_cached, undo);