MDEV-35394 Innochecksum misinterprets freed pages

- Innochecksum misinterprets the freed pages as active one.
This leads the user to think there are too many valid
pages exist.

- To avoid this confusion, innochecksum introduced one
more option --skip-freed-pages and -r to avoid the freed
pages while dumping or printing the summary of the tablespace.

- Innochecksum can safely assume the page is freed if
the respective extent doesn't belong to a segment and marked as
freed in XDES_BITMAP in extent descriptor page.

- Innochecksum shouldn't assume that zero-filled page as extent
descriptor page.

Reviewed-by: Marko Mäkelä

extra/innochecksum.cc

@@ -75,6 +75,8 @@ static my_bool do_leaf;
 static my_bool per_page_details;
 static ulint n_merge;
 static ulint physical_page_size;  /* Page size in bytes on disk. */
+static ulint extent_size;
+static ulint xdes_size;
 ulong srv_page_size;
 ulong srv_page_size_shift;
 /* Current page number (0 based). */
@@ -99,7 +101,7 @@ char*				log_filename = NULL;
 FILE*				log_file = NULL;
 /* Enabled for log write option. */
 static bool			is_log_enabled = false;
-
+static bool			skip_freed_pages;
 static byte field_ref_zero_buf[UNIV_PAGE_SIZE_MAX];
 const byte *field_ref_zero = field_ref_zero_buf;
 
@@ -267,6 +269,8 @@ static void init_page_size(const byte* buf)
 		srv_page_size_shift = UNIV_ZIP_SIZE_SHIFT_MIN - 1 + ssize;
 		srv_page_size = 512U << ssize;
 		physical_page_size = srv_page_size;
+		extent_size = FSP_EXTENT_SIZE;
+		xdes_size = XDES_SIZE;
 		return;
 	}
 
@@ -278,6 +282,8 @@ static void init_page_size(const byte* buf)
 
 	srv_page_size = fil_space_t::logical_size(flags);
 	physical_page_size = fil_space_t::physical_size(flags);
+	extent_size = FSP_EXTENT_SIZE;
+	xdes_size = XDES_SIZE;
 }
 
 #ifdef _WIN32
@@ -555,8 +561,8 @@ bool
 is_page_doublewritebuffer(
 	const byte*	page)
 {
-	if ((cur_page_num >= FSP_EXTENT_SIZE)
+	if ((cur_page_num >= extent_size)
-		&& (cur_page_num < FSP_EXTENT_SIZE * 3)) {
+		&& (cur_page_num < extent_size * 3)) {
 		/* page is doublewrite buffer. */
 		return (true);
 	}
@@ -757,8 +763,8 @@ static inline bool is_page_free(const byte *xdes, ulint physical_page_size,
 {
   const byte *des=
       xdes + XDES_ARR_OFFSET +
-      XDES_SIZE * ((page_no & (physical_page_size - 1)) / FSP_EXTENT_SIZE);
+      xdes_size * ((page_no & (physical_page_size - 1)) / extent_size);
-  return xdes_is_free(des, page_no % FSP_EXTENT_SIZE);
+  return xdes_is_free(des, page_no % extent_size);
 }
 
 /*
@@ -786,6 +792,16 @@ parse_page(
 
 	/* Check whether page is doublewrite buffer. */
 	str = skip_page ? "Double_write_buffer" : "-";
+	page_no = mach_read_from_4(page + FIL_PAGE_OFFSET);
+	if (skip_freed_pages) {
+		const byte *des= xdes + XDES_ARR_OFFSET +
+			xdes_size * ((page_no & (physical_page_size - 1))
+				     / extent_size);
+		if (mach_read_from_4(des) != XDES_FSEG &&
+		    xdes_is_free(des, page_no % extent_size)) {
+			return;
+		}
+	}
 
 	switch (fil_page_get_type(page)) {
 
@@ -1211,6 +1227,9 @@ static struct my_option innochecksum_options[] = {
     &do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"merge", 'm', "leaf page count if merge given number of consecutive pages",
    &n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, 0, 0, (longlong)10L, 0, 1, 0},
+  {"skip-freed-pages", 'r', "skip freed pages for the tablespace",
+   &skip_freed_pages, &skip_freed_pages, 0, GET_BOOL, NO_ARG,
+   0, 0, 0, 0, 0, 0},
 
   {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
@@ -1234,7 +1253,7 @@ static void usage(void)
 	print_version();
 	puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000"));
 	printf("InnoDB offline file checksum utility.\n");
-	printf("Usage: %s [-c] [-s <start page>] [-e <end page>] "
+	printf("Usage: %s [-c] [-r] [-s <start page>] [-e <end page>] "
 		"[-p <page>] [-i] [-v]  [-a <allow mismatches>] [-n] "
 		"[-S] [-D <page type dump>] "
 		"[-l <log>] [-l] [-m <merge pages>] <filename or [-]>\n", my_progname);
@@ -1247,8 +1266,8 @@ static void usage(void)
 extern "C" my_bool
 innochecksum_get_one_option(
 	const struct my_option	*opt,
-	const char		*argument MY_ATTRIBUTE((unused)),
+	const char		*IF_DBUG(argument,),
-        const char *)
+	const char *)
 {
 	switch (opt->id) {
 #ifndef DBUG_OFF
@@ -1273,15 +1292,6 @@ innochecksum_get_one_option(
 		my_end(0);
 		exit(EXIT_SUCCESS);
 		break;
-	case 'n':
-		no_check = true;
-		break;
-	case 'a':
-	case 'S':
-		break;
-	case 'w':
-		do_write = true;
-		break;
 	case 'D':
 		page_type_dump = true;
 		break;
@@ -1328,8 +1338,8 @@ get_options(
 */
 static bool check_encryption(const char* filename, const byte* page)
 {
-	ulint offset = FSP_HEADER_OFFSET + XDES_ARR_OFFSET + XDES_SIZE *
+	ulint offset = FSP_HEADER_OFFSET + XDES_ARR_OFFSET + xdes_size *
-		physical_page_size / FSP_EXTENT_SIZE;
+		physical_page_size / extent_size;
 
 	if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) {
 		return false;
@@ -1861,7 +1871,7 @@ first_non_zero:
 				printf("page " UINT32PF " ", cur_page_num);
 			}
 
-			if (page_get_page_no(buf) % physical_page_size == 0) {
+			if (cur_page_num % physical_page_size == 0) {
 				memcpy(xdes, buf, physical_page_size);
 			}
 

mysql-test/suite/innodb/r/innochecksum_undo_page.result

@@ -0,0 +1,10 @@
+SET GLOBAL INNODB_FILE_PER_TABLE= 0;
+CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB;
+INSERT INTO t1 VALUES(1);
+DROP TABLE t1;
+SET GLOBAL innodb_fast_shutdown=0;
+# Run the innochecksum to display undo log pages
+FOUND 1 /Undo page state: 0 active, [0-9]+ cached, [0-9]+ to_purge, [0-9]+ prepared, [0-9]+ other/ in result.log
+# Run the innochecksum with --skip-freed-pages
+FOUND 1 /Undo page state: 0 active, 0 cached, 0 to_purge, 0 prepared, 0 other/ in result.log
+# restart

mysql-test/suite/innodb/t/innochecksum_undo_page.opt

@@ -0,0 +1 @@
+--innodb_undo_tablespaces=0

mysql-test/suite/innodb/t/innochecksum_undo_page.test

@@ -0,0 +1,27 @@
+--source include/have_innodb.inc
+--source include/not_embedded.inc
+let MYSQLD_DATADIR= `SELECT @@datadir`;
+
+SET GLOBAL INNODB_FILE_PER_TABLE= 0;
+CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB;
+INSERT INTO t1 VALUES(1);
+DROP TABLE t1;
+SET GLOBAL innodb_fast_shutdown=0;
+--source include/shutdown_mysqld.inc
+
+--echo # Run the innochecksum to display undo log pages
+let $resultlog=$MYSQLTEST_VARDIR/tmp/result.log;
+let SEARCH_FILE = $MYSQLTEST_VARDIR/tmp/result.log;
+let SEARCH_ABORT = NOT FOUND;
+exec $INNOCHECKSUM -S $MYSQLD_DATADIR/ibdata1 > $resultlog;
+# Expected > 0 cached undo log pages, but can't guarantee it because
+# the writes of freed pages may be optimized while flushing
+let SEARCH_PATTERN= Undo page state: 0 active, [0-9]+ cached, [0-9]+ to_purge, [0-9]+ prepared, [0-9]+ other;
+--source include/search_pattern_in_file.inc
+
+--echo # Run the innochecksum with --skip-freed-pages
+exec $INNOCHECKSUM -S -r $MYSQLD_DATADIR/ibdata1 > $resultlog;
+let SEARCH_PATTERN= Undo page state: 0 active, 0 cached, 0 to_purge, 0 prepared, 0 other;
+--source include/search_pattern_in_file.inc
+--remove_file $resultlog
+--source include/start_mysqld.inc

mysql-test/suite/innodb_zip/r/innochecksum_2.result

@@ -34,6 +34,7 @@ per-page-details                  FALSE
 log                               (No default value)
 leaf                              FALSE
 merge                             0
+skip-freed-pages                  FALSE
 [1]:# check the both short and long options for "help"
 [2]:# Run the innochecksum when file isn't provided.
 # It will print the innochecksum usage similar to --help option.
@@ -41,7 +42,7 @@ innochecksum Ver #.#.#
 Copyright (c) YEAR, YEAR , Oracle, MariaDB Corporation Ab and others.
 
 InnoDB offline file checksum utility.
-Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-i] [-v]  [-a <allow mismatches>] [-n] [-S] [-D <page type dump>] [-l <log>] [-l] [-m <merge pages>] <filename or [-]>
+Usage: innochecksum [-c] [-r] [-s <start page>] [-e <end page>] [-p <page>] [-i] [-v]  [-a <allow mismatches>] [-n] [-S] [-D <page type dump>] [-l <log>] [-l] [-m <merge pages>] <filename or [-]>
 See https://mariadb.com/kb/en/library/innochecksum/ for usage hints.
   -?, --help          Displays this help and exits.
   -I, --info          Synonym for --help.
@@ -66,6 +67,8 @@ See https://mariadb.com/kb/en/library/innochecksum/ for usage hints.
   -f, --leaf          Examine leaf index pages
   -m, --merge=#       leaf page count if merge given number of consecutive
                       pages
+  -r, --skip-freed-pages 
+                      skip freed pages for the tablespace
 
 Variables (--variable-name=value)
 and boolean options {FALSE|TRUE}  Value (after reading options)
@@ -84,6 +87,7 @@ per-page-details                  FALSE
 log                               (No default value)
 leaf                              FALSE
 merge                             0
+skip-freed-pages                  FALSE
 [3]:# check the both short and long options for "count" and exit
 Number of pages:#
 Number of pages:#

mysql-test/suite/innodb_zip/r/innochecksum_3.result

@@ -139,6 +139,7 @@ per-page-details                  FALSE
 log                               (No default value)
 leaf                              FALSE
 merge                             0
+skip-freed-pages                  FALSE
 [5]: Page type dump for with shortform for tab1.ibd