diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index a949a996284..c9a18f048bd 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -517,7 +517,7 @@ static bool is_page_corrupted(byte *buf, bool is_encrypted, uint32_t flags) normal method. */ if (is_encrypted && key_version != 0) { is_corrupted = use_full_crc32 - ? buf_page_is_corrupted(true, buf, flags) + ? !!buf_page_is_corrupted(false, buf, flags) : !fil_space_verify_crypt_checksum(buf, zip_size); if (is_corrupted && log_file) { diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index be871e4a65f..647432b15d4 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -283,7 +283,7 @@ static bool page_is_corrupted(const byte *page, ulint page_no, } if (space->full_crc32()) { - return buf_page_is_corrupted(true, page, space->flags); + return buf_page_is_corrupted(false, page, space->flags); } /* Validate encrypted pages. The first page is never encrypted. @@ -316,7 +316,7 @@ static bool page_is_corrupted(const byte *page, ulint page_no, } if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return buf_page_is_corrupted(true, tmp_page, + return buf_page_is_corrupted(false, tmp_page, space->flags); } } @@ -336,11 +336,11 @@ static bool page_is_corrupted(const byte *page, ulint page_no, && cursor->zip_size) || page_type == FIL_PAGE_PAGE_COMPRESSED || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || buf_page_is_corrupted(true, tmp_page, + || buf_page_is_corrupted(false, tmp_page, space->flags)); } - return buf_page_is_corrupted(true, page, space->flags); + return buf_page_is_corrupted(false, page, space->flags); } PRAGMA_REENABLE_CHECK_STACK_FRAME diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 411bb2479d1..e9f6876f30b 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -3912,7 +3912,7 @@ static void xb_load_single_table_tablespace(const char *dirname, for (int i = 0; i < 10; i++) { file->m_defer = false; - err = file->validate_first_page(); + err = file->validate_first_page(file->get_first_page()); if (file->m_defer) { if (defer_space_id) { @@ -3954,7 +3954,7 @@ static void xb_load_single_table_tablespace(const char *dirname, skip_node_page0 ? file->detach() : pfs_os_file_t(), 0, false, false); node->deferred= defer; - if (!space->read_page0()) + if (!space->read_page0(nullptr, true)) err = DB_CANNOT_OPEN_FILE; mysql_mutex_unlock(&fil_system.mutex); @@ -6852,8 +6852,10 @@ error: goto error; } - ok = fil_system.sys_space->open(false) - && xtrabackup_apply_deltas(); + mysql_mutex_lock(&recv_sys.mutex); + ok = fil_system.sys_space->open(false); + mysql_mutex_unlock(&recv_sys.mutex); + if (ok) ok = xtrabackup_apply_deltas(); xb_data_files_close(); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index e69d427e5d2..79e46fef8b4 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -3,8 +3,10 @@ call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` has an unreadable root page"); call mtr.add_suppression("Table .*t[12].* is corrupted"); call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted; key_version=1"); -call mtr.add_suppression("failed to read \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]"); +call mtr.add_suppression("InnoDB: Failed to read page [1-9][0-9]* from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t1.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t[12]` is corrupted"); call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); # Start server with keys2.txt diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result index c02c543ae7e..99d71da9582 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result @@ -3,7 +3,9 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page n call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded\\."); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("InnoDB: Cannot delete tablespace .* because it is not found in the tablespace memory cache"); call mtr.add_suppression("InnoDB: ALTER TABLE `test`\\.`t1` DISCARD TABLESPACE failed to find tablespace"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result index 40b377e69b2..7017a75b5ed 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result @@ -2,7 +2,9 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` (has an unreadable root pa call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t1\\.ibd' cannot be decrypted; key_version=1"); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); # restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt SET GLOBAL innodb_file_per_table = ON; diff --git a/mysql-test/suite/encryption/r/innodb-compressed-blob.result b/mysql-test/suite/encryption/r/innodb-compressed-blob.result index c7696c3799a..dbc037d893c 100644 --- a/mysql-test/suite/encryption/r/innodb-compressed-blob.result +++ b/mysql-test/suite/encryption/r/innodb-compressed-blob.result @@ -1,7 +1,7 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted; key_version=1"); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Unable to decompress ..test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test/t[12]\\.ibd' page \\[page id: space=[1-9][0-9]*, page number=3\\]"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); call mtr.add_suppression("Table `test`\\.`t[12]` is corrupted"); # Restart mysqld --file-key-management-filename=keys2.txt diff --git a/mysql-test/suite/encryption/r/innodb-encryption-disable.result b/mysql-test/suite/encryption/r/innodb-encryption-disable.result index 86c6d63649f..13bd8ac4a34 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-disable.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-disable.result @@ -2,6 +2,8 @@ call mtr.add_suppression("Table `test`\\.`t[15]` (has an unreadable root page|is call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[15].ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[15]\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t[15]\\.ibd' is corrupted"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); # restart: --innodb-encrypt-tables=ON --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt create table t5 ( diff --git a/mysql-test/suite/encryption/r/innodb-force-corrupt.result b/mysql-test/suite/encryption/r/innodb-force-corrupt.result index 16cce1d2723..8ef0b897f6b 100644 --- a/mysql-test/suite/encryption/r/innodb-force-corrupt.result +++ b/mysql-test/suite/encryption/r/innodb-force-corrupt.result @@ -1,7 +1,10 @@ -call mtr.add_suppression("Table `test`\\.`t[13]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("Table `test`\\.`t[123]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("InnoDB: File '.*test/t[123]\\.ibd' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[13]\\.ibd'"); +call mtr.add_suppression("InnoDB: Failed to read page 6 from file '.*test/t2\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version="); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption in an InnoDB type table"); -call mtr.add_suppression("\\[ERROR\\] (mysqld|mariadbd).*: Index for table 't2' is corrupt; try to repair it"); +call mtr.add_suppression("\\[ERROR\\] mariadbd.*: Index for table 't2' is corrupt; try to repair it"); SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; # Create and populate tables to be corrupted diff --git a/mysql-test/suite/encryption/r/innodb-missing-key.result b/mysql-test/suite/encryption/r/innodb-missing-key.result index 83c9166d05b..14fb0275f1c 100644 --- a/mysql-test/suite/encryption/r/innodb-missing-key.result +++ b/mysql-test/suite/encryption/r/innodb-missing-key.result @@ -2,7 +2,9 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` has an unreadable root pag call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file .*test.t[12].ibd looks corrupted; key_version=1"); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); # Start server with keys2.txt # restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt CREATE TABLE t1(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=19; diff --git a/mysql-test/suite/encryption/r/innodb-redo-badkey.result b/mysql-test/suite/encryption/r/innodb-redo-badkey.result index 34fd043a7bd..16035bd3b15 100644 --- a/mysql-test/suite/encryption/r/innodb-redo-badkey.result +++ b/mysql-test/suite/encryption/r/innodb-redo-badkey.result @@ -24,6 +24,7 @@ insert into t2 select * from t1; insert into t3 select * from t1; insert into t4 select * from t1; commit; +set global innodb_log_checkpoint_now=on; SET GLOBAL innodb_flush_log_at_trx_commit=1; begin; update t1 set c = repeat('secret3', 20); diff --git a/mysql-test/suite/encryption/t/corrupted_during_recovery.test b/mysql-test/suite/encryption/t/corrupted_during_recovery.test index e4a31a0b478..dabf06dd047 100644 --- a/mysql-test/suite/encryption/t/corrupted_during_recovery.test +++ b/mysql-test/suite/encryption/t/corrupted_during_recovery.test @@ -5,11 +5,11 @@ call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); call mtr.add_suppression("InnoDB: Failed to read page [123] from file '.*test.t1\\.ibd': Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("InnoDB: The page \\[page id: space=\\d+, page number=3\\] in file '.*test.t1\\.ibd' cannot be decrypted"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table in tablespace \\d+ encrypted. However key management plugin or used key_version \\d+ is not found or used encryption algorithm or method does not match. Can't continue opening the table."); -call mtr.add_suppression("InnoDB: (Unable to apply log to|Discarding log for) corrupted page "); +call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page "); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index e95378473d8..0c897b2c936 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -13,8 +13,10 @@ call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` has an unreadable root page"); call mtr.add_suppression("Table .*t[12].* is corrupted"); call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted; key_version=1"); -call mtr.add_suppression("failed to read \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]"); +call mtr.add_suppression("InnoDB: Failed to read page [1-9][0-9]* from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t1.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t[12]` is corrupted"); call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found"); # for innodb_checksum_algorithm=full_crc32 only call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test index 9ae3c722d98..1d31328b09d 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test @@ -14,7 +14,9 @@ call mtr.add_suppression("InnoDB: Recovery failed to read page"); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded\\."); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("InnoDB: Cannot delete tablespace .* because it is not found in the tablespace memory cache"); call mtr.add_suppression("InnoDB: ALTER TABLE `test`\\.`t1` DISCARD TABLESPACE failed to find tablespace"); # for innodb_checksum_algorithm=full_crc32 only diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test index c37eb365e7c..a0d601c65d4 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test @@ -12,7 +12,9 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page n call mtr.add_suppression("InnoDB: Recovery failed to read page"); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); # for innodb_checksum_algorithm=full_crc32 only call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); diff --git a/mysql-test/suite/encryption/t/innodb-compressed-blob.test b/mysql-test/suite/encryption/t/innodb-compressed-blob.test index 6fcc4fdeeae..2780567a89f 100644 --- a/mysql-test/suite/encryption/t/innodb-compressed-blob.test +++ b/mysql-test/suite/encryption/t/innodb-compressed-blob.test @@ -7,7 +7,7 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted; key_version=1"); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Unable to decompress ..test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test/t[12]\\.ibd' page \\[page id: space=[1-9][0-9]*, page number=3\\]"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); call mtr.add_suppression("Table `test`\\.`t[12]` is corrupted"); diff --git a/mysql-test/suite/encryption/t/innodb-encryption-disable.test b/mysql-test/suite/encryption/t/innodb-encryption-disable.test index 7a7e590e304..676edd54f98 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-disable.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-disable.test @@ -11,6 +11,8 @@ call mtr.add_suppression("Table `test`\\.`t[15]` (has an unreadable root page|is call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[15].ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[15]\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t[15]\\.ibd' is corrupted"); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); diff --git a/mysql-test/suite/encryption/t/innodb-force-corrupt.test b/mysql-test/suite/encryption/t/innodb-force-corrupt.test index 861cc78cc7f..8d0d2f7deb8 100644 --- a/mysql-test/suite/encryption/t/innodb-force-corrupt.test +++ b/mysql-test/suite/encryption/t/innodb-force-corrupt.test @@ -7,10 +7,13 @@ # Don't test under embedded -- source include/not_embedded.inc -call mtr.add_suppression("Table `test`\\.`t[13]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("Table `test`\\.`t[123]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("InnoDB: File '.*test/t[123]\\.ibd' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[13]\\.ibd'"); +call mtr.add_suppression("InnoDB: Failed to read page 6 from file '.*test/t2\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version="); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption in an InnoDB type table"); -call mtr.add_suppression("\\[ERROR\\] (mysqld|mariadbd).*: Index for table 't2' is corrupt; try to repair it"); +call mtr.add_suppression("\\[ERROR\\] mariadbd.*: Index for table 't2' is corrupt; try to repair it"); SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.test b/mysql-test/suite/encryption/t/innodb-missing-key.test index 0c7a1df9ae2..07e22484113 100644 --- a/mysql-test/suite/encryption/t/innodb-missing-key.test +++ b/mysql-test/suite/encryption/t/innodb-missing-key.test @@ -11,7 +11,9 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` has an unreadable root pag call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file .*test.t[12].ibd looks corrupted; key_version=1"); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); --echo # Start server with keys2.txt -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-redo-badkey.test b/mysql-test/suite/encryption/t/innodb-redo-badkey.test index 393ca4ad375..ed92a267a91 100644 --- a/mysql-test/suite/encryption/t/innodb-redo-badkey.test +++ b/mysql-test/suite/encryption/t/innodb-redo-badkey.test @@ -54,6 +54,8 @@ insert into t3 select * from t1; insert into t4 select * from t1; commit; +set global innodb_log_checkpoint_now=on; + --source ../../suite/innodb/include/no_checkpoint_start.inc # diff --git a/mysql-test/suite/innodb/r/corrupted_during_recovery.result b/mysql-test/suite/innodb/r/corrupted_during_recovery.result index 2cab795f6a1..593943b4951 100644 --- a/mysql-test/suite/innodb/r/corrupted_during_recovery.result +++ b/mysql-test/suite/innodb/r/corrupted_during_recovery.result @@ -8,8 +8,10 @@ INSERT INTO t2 VALUES(1); # Corrupt the pages SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' +FOUND 1 /InnoDB: Page \[page id: space=[1-9][0-9]*, page number=3\] log sequence number 1311768467463790320 is in the future!/ in mysqld.1.err SELECT * FROM t1; -Got one of the listed errors +a +1 SELECT * FROM t2; a 1 @@ -27,7 +29,7 @@ SET GLOBAL innodb_flush_log_at_trx_commit=1; DELETE FROM t1 WHERE pk=3; # Kill the server disconnect con1; -# Corrupt the pages +# Corrupt the page SELECT * FROM t1; pk 1 diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result index 65ff203a28e..667c65d7b7b 100644 --- a/mysql-test/suite/innodb/r/doublewrite.result +++ b/mysql-test/suite/innodb/r/doublewrite.result @@ -21,8 +21,8 @@ connection default; flush table t1 for export; # Kill the server # restart -FOUND 1 /InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile/ in mysqld.1.err -FOUND 1 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=3\]/ in mysqld.1.err +FOUND 1 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=0\]/ in mysqld.1.err +# restart XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text @@ -44,7 +44,7 @@ connection default; flush table t1 for export; # Kill the server # restart -FOUND 1 /InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile/ in mysqld.1.err +FOUND 4 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=[03]\]/ in mysqld.1.err XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text diff --git a/mysql-test/suite/innodb/r/doublewrite_debug.result b/mysql-test/suite/innodb/r/doublewrite_debug.result index aa141c18a6e..a743217f34e 100644 --- a/mysql-test/suite/innodb/r/doublewrite_debug.result +++ b/mysql-test/suite/innodb/r/doublewrite_debug.result @@ -37,7 +37,7 @@ set global innodb_buf_flush_list_now = 1; # Make the 1st page (page_no=0) and 2nd page (page_no=1) # of the system tablespace all zeroes. # restart -FOUND 1 /InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile/ in mysqld.1.err +FOUND 1 /InnoDB: Recovered page \[page id: space=0, page number=0\]/ in mysqld.1.err FOUND 1 /InnoDB: Recovered page \[page id: space=0, page number=1\]/ in mysqld.1.err check table t1; Table Op Msg_type Msg_text @@ -66,7 +66,7 @@ set global innodb_buf_flush_list_now = 1; # Kill the server # Corrupt the 1st page (page_no=0) and 2nd page of the system tablespace. # restart -FOUND 2 /InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile/ in mysqld.1.err +FOUND 2 /InnoDB: Recovered page \[page id: space=0, page number=0\]/ in mysqld.1.err FOUND 2 /InnoDB: Recovered page \[page id: space=0, page number=1\]/ in mysqld.1.err check table t1; Table Op Msg_type Msg_text diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result index e7af2d9469e..97da5374ca3 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result @@ -9,8 +9,8 @@ call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS="); call mtr.add_suppression("InnoDB: Unknown index id .* on page"); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; SET GLOBAL innodb_file_per_table = 1; CREATE TABLE t1 (c1 INT) ENGINE = InnoDB; diff --git a/mysql-test/suite/innodb/r/page_id_innochecksum.result b/mysql-test/suite/innodb/r/page_id_innochecksum.result index bde986c07ef..4ffbf2f1453 100644 --- a/mysql-test/suite/innodb/r/page_id_innochecksum.result +++ b/mysql-test/suite/innodb/r/page_id_innochecksum.result @@ -6,3 +6,4 @@ FOUND 1 /page id mismatch/ in result.log InnoDB 0 transactions not purged drop table t1; call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd': Page read from tablespace is corrupted\\."); +call mtr.add_suppression("InnoDB: File '.*t1\\.ibd' is corrupted"); diff --git a/mysql-test/suite/innodb/r/undo_space_dblwr.result b/mysql-test/suite/innodb/r/undo_space_dblwr.result index 2172ce53cb7..d7c1cb125a9 100644 --- a/mysql-test/suite/innodb/r/undo_space_dblwr.result +++ b/mysql-test/suite/innodb/r/undo_space_dblwr.result @@ -12,7 +12,7 @@ set global innodb_fil_make_page_dirty_debug = 1; SET GLOBAL innodb_buf_flush_list_now = 1; # Kill the server # restart: --debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0 -FOUND 1 /Restoring page \[page id: space=1, page number=0\] of datafile '.*undo001' from the doublewrite buffer./ in mysqld.1.err +FOUND 1 /Recovered page \[page id: space=1, page number=0\] to '.*undo001' from the doublewrite buffer\./ in mysqld.1.err check table t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb/t/corrupted_during_recovery.test b/mysql-test/suite/innodb/t/corrupted_during_recovery.test index 8324054992c..f62215a7207 100644 --- a/mysql-test/suite/innodb/t/corrupted_during_recovery.test +++ b/mysql-test/suite/innodb/t/corrupted_during_recovery.test @@ -4,15 +4,18 @@ call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); -call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test.t1\\.ibd': Page read from tablespace is corrupted."); -call mtr.add_suppression("InnoDB: (Unable to apply log to|Discarding log for) corrupted page .*, page number=3\\]"); -call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate."); +call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page 3 in file .*test.t1\\.ibd"); +call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate\\."); call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: A long wait .* was observed for dict_sys"); +call mtr.add_suppression("InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 1311768467463790320 is in the future!"); +call mtr.add_suppression("InnoDB: Your database may be corrupt"); +call mtr.add_suppression("InnoDB: MySQL-8\\.0 tablespace in .*test/t2\\.ibd"); +call mtr.add_suppression("InnoDB: Restart in MySQL for migration/recovery\\."); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let ALGO=`select @@innodb_checksum_algorithm`; CREATE TABLE t1(a BIGINT PRIMARY KEY) ENGINE=InnoDB; INSERT INTO t1 VALUES(1); # Force a redo log checkpoint. @@ -30,15 +33,32 @@ INSERT INTO t2 VALUES(1); --echo # Corrupt the pages perl; +do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; +my $polynomial = 0x82f63b78; # CRC-32C +my $algo = $ENV{ALGO}; my $ps = $ENV{INNODB_PAGE_SIZE}; - my $file = "$ENV{MYSQLD_DATADIR}/test/t1.ibd"; open(FILE, "+<$file") || die "Unable to open $file"; binmode FILE; sysseek(FILE, 3*$ps, 0) || die "Unable to seek $file\n"; die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps; -# Replace the a=1 with a=0. -$page =~ s/\x80\x0\x0\x0\x0\x0\x0\x1/\x80\x0\x0\x0\x0\x0\x0\x0/; +# Assign a future FIL_PAGE_LSN +substr($page, 16, 8) = pack("NN", 0x12345678, 0x9abcdef0); +substr($page, $ps - 8, 8) = pack("NN", 0x9abcdef0, 0x9abcdef0); +if ($algo =~ /full_crc32/) +{ + my $ck = mycrc32(substr($page, 0, $ps - 4), 0, $polynomial); + substr($page, $ps - 4, 4) = pack("N", $ck); +} +else +{ + # Replace the innodb_checksum_algorithm=crc32 checksum + my $ck= pack("N", + mycrc32(substr($page, 4, 22), 0, $polynomial) ^ + mycrc32(substr($page, 38, $ps - 38 - 8), 0, $polynomial)); + substr ($page, 0, 4) = $ck; + substr ($page, $ps - 8, 4) = $ck; +} sysseek(FILE, 3*$ps, 0) || die "Unable to seek $file\n"; syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n"; close FILE or die "close"; @@ -46,20 +66,23 @@ close FILE or die "close"; $file = "$ENV{MYSQLD_DATADIR}/test/t2.ibd"; open(FILE, "+<$file") || die "Unable to open $file"; binmode FILE; -# Corrupt pages 1 to 3. MLOG_INIT_FILE_PAGE2 should protect us! -# Unfortunately, we are not immune to page 0 corruption. -seek (FILE, $ps, SEEK_SET) or die "seek"; -print FILE chr(0xff) x ($ps * 3); +# Corrupt pages 0 to 3. INIT_PAGE should protect us! +print FILE chr(0xff) x ($ps * 4); close FILE or die "close"; EOF --source include/start_mysqld.inc --error ER_UNKNOWN_STORAGE_ENGINE SELECT * FROM t1; + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 1311768467463790320 is in the future!; +--source include/search_pattern_in_file.inc + let $restart_parameters=--innodb_force_recovery=1; --source include/restart_mysqld.inc ---error ER_NO_SUCH_TABLE_IN_ENGINE,ER_TABLE_CORRUPT +--error 0,ER_NO_SUCH_TABLE_IN_ENGINE SELECT * FROM t1; SELECT * FROM t2; CHECK TABLE t2; @@ -81,13 +104,36 @@ DELETE FROM t1 WHERE pk=3; --source ../include/no_checkpoint_end.inc disconnect con1; ---echo # Corrupt the pages +--echo # Corrupt the page perl; +do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; +my $polynomial = 0x82f63b78; # CRC-32C +my $algo = $ENV{ALGO}; +my $ps = $ENV{INNODB_PAGE_SIZE}; my $file = "$ENV{MYSQLD_DATADIR}/test/t1.ibd"; open(FILE, "+<$file") || die "Unable to open $file"; binmode FILE; -seek (FILE, $ENV{INNODB_PAGE_SIZE} * 3, SEEK_SET) or die "seek"; -print FILE "junk"; +sysseek(FILE, $ps * 3, SEEK_SET) or die "seek"; +sysread(FILE, $page, $ps)==$ps||die "Unable to read $file\n"; +# Set FIL_PAGE_LSN to the maximum +substr($page, 16, 8) = chr(255) x 8; +substr($page, $ps - 8, 8) = chr(255) x 8; +if ($algo =~ /full_crc32/) +{ + my $ck = mycrc32(substr($page, 0, $ps - 4), 0, $polynomial); + substr($page, $ps - 4, 4) = pack("N", $ck); +} +else +{ + # Replace the innodb_checksum_algorithm=crc32 checksum + my $ck= pack("N", + mycrc32(substr($page, 4, 22), 0, $polynomial) ^ + mycrc32(substr($page_, 38, $ps - 38 - 8), 0, $polynomial)); + substr ($page, 0, 4) = $ck; + substr ($page, $ps - 8, 4) = $ck; +} +sysseek(FILE, $ps * 3, SEEK_SET) or die "seek"; +syswrite(FILE, $page); close FILE or die "close"; EOF --source include/start_mysqld.inc diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index d73009908db..70832584228 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -18,6 +18,9 @@ call mtr.add_suppression("InnoDB: Checksum mismatch in datafile: "); call mtr.add_suppression("InnoDB: Inconsistent tablespace ID in .*t1\\.ibd"); call mtr.add_suppression("\\[Warning\\] Found 1 prepared XA transactions"); call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile:"); +call mtr.add_suppression("InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 18446744073709551615 is in the future!"); +call mtr.add_suppression("InnoDB: Your database may be corrupt or you may have copied the InnoDB tablespace but not the ib_logfile0"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; @@ -71,7 +74,26 @@ syswrite(FILE, chr(0) x ($page_size/2)); sysseek(FILE, 3*$page_size, 0); sysread(FILE, $page, $page_size)==$page_size||die "Unable to read $name\n"; sysseek(FILE, 3*$page_size, 0)||die "Unable to seek $fname\n"; -syswrite(FILE, chr(0) x ($page_size/2)); +my $corrupted = $page; +# Set FIL_PAGE_LSN to the maximum +substr($corrupted, 16, 8) = chr(255) x 8; +substr($corrupted, $page_size - 8, 8) = chr(255) x 8; +if ($algo =~ /full_crc32/) +{ + my $ck = mycrc32(substr($corrupted, 0, $page_size - 4), 0, $polynomial); + substr($corrupted, $page_size - 4, 4) = pack("N", $ck); +} +else +{ + # Replace the innodb_checksum_algorithm=crc32 checksum + my $ck= pack("N", + mycrc32(substr($corrupted, 4, 22), 0, $polynomial) ^ + mycrc32(substr($corrupted_, 38, $page_size - 38 - 8), 0, + $polynomial)); + substr ($corrupted, 0, 4) = $ck; + substr ($corrupted, $page_size - 8, 4) = $ck; +} +syswrite(FILE, $corrupted); close FILE; # Change the flag offset of page 0 in doublewrite buffer @@ -113,10 +135,28 @@ die "Did not find the page in the doublewrite buffer ($d1,$d2)\n"; EOF --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=[1-9][0-9]*, page number=0\\]; --source include/search_pattern_in_file.inc -let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=3\]; +let SEARCH_PATTERN=InnoDB: The log was only scanned up to \\d+, while the current LSN at the time of the latest checkpoint \\d+ was 0 and the maximum LSN on a data page was 18446744073709551615! --source include/search_pattern_in_file.inc +--error ER_XAER_NOTA +XA ROLLBACK 'x'; +let $shutdown_timeout=0; +--source include/shutdown_mysqld.inc +let $shutdown_timeout=; +# Corrupt the file in a better way. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; +my $page_size = $ENV{INNODB_PAGE_SIZE}; +open(FILE, "+<", $fname) or die; +sysseek(FILE, ($page_size/2), 0); +syswrite(FILE, chr(0) x ($page_size/2)); +sysseek(FILE, 3*$page_size, 0); +syswrite(FILE, chr(0) x ($page_size/2)); +close FILE; +EOF +--source include/start_mysqld.inc XA ROLLBACK 'x'; check table t1; select f1, f2 from t1; @@ -144,7 +184,7 @@ close FILE; EOF --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=[1-9][0-9]*, page number=[03]\\]; --source include/search_pattern_in_file.inc XA ROLLBACK 'x'; check table t1; diff --git a/mysql-test/suite/innodb/t/doublewrite_debug.test b/mysql-test/suite/innodb/t/doublewrite_debug.test index b8dcd5068ef..b207823e3d1 100644 --- a/mysql-test/suite/innodb/t/doublewrite_debug.test +++ b/mysql-test/suite/innodb/t/doublewrite_debug.test @@ -81,10 +81,10 @@ EOF let $restart_parameters=; --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=0\\]; --source include/search_pattern_in_file.inc -let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=0, page number=1\]; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=1\\]; --source include/search_pattern_in_file.inc check table t1; @@ -129,10 +129,10 @@ EOF let $restart_parameters=; --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=0\\]; --source include/search_pattern_in_file.inc -let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=0, page number=1\]; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=1\\]; --source include/search_pattern_in_file.inc check table t1; diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test index 2d3b5b3fae4..68330692390 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test @@ -29,8 +29,8 @@ call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS="); call mtr.add_suppression("InnoDB: Unknown index id .* on page"); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491.test b/mysql-test/suite/innodb/t/innodb_bug14147491.test index 10f3c98be9e..d954f2fadea 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491.test +++ b/mysql-test/suite/innodb/t/innodb_bug14147491.test @@ -9,7 +9,7 @@ --disable_query_log call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted\\. Please drop the table and recreate\\."); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); +call mtr.add_suppression("InnoDB: Failed to read page [1-9][0-9]* from file '.*test.t1\\.ibd'"); call mtr.add_suppression("InnoDB: We detected index corruption in an InnoDB type table"); call mtr.add_suppression("Index for table 't1' is corrupt; try to repair it"); call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); diff --git a/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test b/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test index 21d5336528a..026be74373f 100644 --- a/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test +++ b/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test @@ -2,9 +2,8 @@ --source include/have_debug.inc --disable_query_log -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 19 from file '.*test.t1\\.ibd': Page read from tablespace is corrupted\\."); -call mtr.add_suppression("InnoDB: (Unable to apply log to|Discarding log for) corrupted page .*, page number=19\\]"); +call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page 19 in file .*t1\\.ibd"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Plugin initialization aborted at srv0start\\.cc.* with error Data structure corruption"); call mtr.add_suppression("\\[ERROR\\] Plugin 'InnoDB' (init function|registration)"); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption"); diff --git a/mysql-test/suite/innodb/t/page_id_innochecksum.test b/mysql-test/suite/innodb/t/page_id_innochecksum.test index 9d8114d1720..09523cccd1f 100644 --- a/mysql-test/suite/innodb/t/page_id_innochecksum.test +++ b/mysql-test/suite/innodb/t/page_id_innochecksum.test @@ -66,5 +66,6 @@ let $restart_parameters=--innodb-force-recovery=1; --source include/wait_all_purged.inc drop table t1; call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd': Page read from tablespace is corrupted\\."); +call mtr.add_suppression("InnoDB: File '.*t1\\.ibd' is corrupted"); let $restart_parameters=; --source include/restart_mysqld.inc diff --git a/mysql-test/suite/innodb/t/undo_space_dblwr.test b/mysql-test/suite/innodb/t/undo_space_dblwr.test index 4cf4d3b8b6d..33e8ed9d651 100644 --- a/mysql-test/suite/innodb/t/undo_space_dblwr.test +++ b/mysql-test/suite/innodb/t/undo_space_dblwr.test @@ -39,7 +39,7 @@ EOF --source include/start_mysqld.inc let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; -let SEARCH_PATTERN= Restoring page \[page id: space=1, page number=0\] of datafile '.*undo001' from the doublewrite buffer.; +let SEARCH_PATTERN= Recovered page \\[page id: space=1, page number=0\\] to '.*undo001' from the doublewrite buffer\\.; --source include/search_pattern_in_file.inc check table t1; diff --git a/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result b/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result index 62a28a981cd..574ba0b6c2e 100644 --- a/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result +++ b/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result @@ -9,8 +9,8 @@ call mtr.add_suppression("InnoDB: Error number \\d+ means"); call mtr.add_suppression("InnoDB: Cannot open datafile .*t1\\.ibd"); call mtr.add_suppression("InnoDB: Ignoring tablespace for test/t1 "); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; SET SESSION innodb_strict_mode=1; CREATE TABLE t1 (c1 INT) ENGINE = Innodb diff --git a/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test b/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test index 47b53b718f5..de76449c456 100644 --- a/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test +++ b/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test @@ -25,8 +25,8 @@ call mtr.add_suppression("InnoDB: Error number \\d+ means"); call mtr.add_suppression("InnoDB: Cannot open datafile .*t1\\.ibd"); call mtr.add_suppression("InnoDB: Ignoring tablespace for test/t1 "); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; diff --git a/mysql-test/suite/mariabackup/compression_providers_unloaded.result b/mysql-test/suite/mariabackup/compression_providers_unloaded.result index ccf3e0355a4..89cd6ed2158 100644 --- a/mysql-test/suite/mariabackup/compression_providers_unloaded.result +++ b/mysql-test/suite/mariabackup/compression_providers_unloaded.result @@ -1,6 +1,7 @@ call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("Refusing to load corrupted table"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); # # Testing mariabackup with bzip2 compression # diff --git a/mysql-test/suite/mariabackup/compression_providers_unloaded.test b/mysql-test/suite/mariabackup/compression_providers_unloaded.test index 673c16d03cf..1c2b5fa1db8 100644 --- a/mysql-test/suite/mariabackup/compression_providers_unloaded.test +++ b/mysql-test/suite/mariabackup/compression_providers_unloaded.test @@ -2,6 +2,7 @@ let $alg = $MTR_COMBINATIONS; call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("Refusing to load corrupted table"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); if (`select count(*) = 0 from information_schema.plugins where plugin_name = 'provider_$alg' and plugin_status='active'`) { diff --git a/mysql-test/suite/mariabackup/encrypted_page_compressed.result b/mysql-test/suite/mariabackup/encrypted_page_compressed.result index de4c966caf4..31098e45675 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_compressed.result +++ b/mysql-test/suite/mariabackup/encrypted_page_compressed.result @@ -1,5 +1,7 @@ call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); call mtr.add_suppression("InnoDB: Encrypted page .* in file .*test.t1\\.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes; insert into t1(b, c) values("mariadb", "mariabackup"); # Corrupt the table diff --git a/mysql-test/suite/mariabackup/encrypted_page_compressed.test b/mysql-test/suite/mariabackup/encrypted_page_compressed.test index 245fcc31c0d..9faf654d334 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/encrypted_page_compressed.test @@ -1,6 +1,8 @@ source include/have_file_key_management.inc; call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); call mtr.add_suppression("InnoDB: Encrypted page .* in file .*test.t1\\.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes; insert into t1(b, c) values("mariadb", "mariabackup"); diff --git a/mysql-test/suite/mariabackup/encrypted_page_corruption.result b/mysql-test/suite/mariabackup/encrypted_page_corruption.result index b328d361cd6..62f39b561da 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_corruption.result +++ b/mysql-test/suite/mariabackup/encrypted_page_corruption.result @@ -1,4 +1,6 @@ call mtr.add_suppression("\\[ERROR\\] InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=3\\] in file '.*test.t1\\.ibd' cannot be decrypted."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Table `test`\\.`t1` has an unreadable root page"); CREATE TABLE t1(c VARCHAR(128)) ENGINE INNODB, encrypted=yes; insert into t1 select repeat('a',100); diff --git a/mysql-test/suite/mariabackup/encrypted_page_corruption.test b/mysql-test/suite/mariabackup/encrypted_page_corruption.test index 9ba958c68a0..10e4b8c98fc 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_corruption.test +++ b/mysql-test/suite/mariabackup/encrypted_page_corruption.test @@ -2,6 +2,8 @@ --source include/innodb_page_size.inc call mtr.add_suppression("\\[ERROR\\] InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=3\\] in file '.*test.t1\\.ibd' cannot be decrypted."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Table `test`\\.`t1` has an unreadable root page"); CREATE TABLE t1(c VARCHAR(128)) ENGINE INNODB, encrypted=yes; insert into t1 select repeat('a',100); diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.result b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result index dfcf19b6c2b..ededfbb8e8d 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.result +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result @@ -1,3 +1,5 @@ +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB PAGE_COMPRESSED=YES STATS_PERSISTENT=0; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test index 31e8323b8b6..68f22e69e03 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test @@ -1,3 +1,5 @@ +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB PAGE_COMPRESSED=YES STATS_PERSISTENT=0; diff --git a/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff b/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff index 3f449220df5..e48fb5eaa8f 100644 --- a/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff +++ b/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff @@ -1,6 +1,6 @@ --- suite/plugins/r/compression.result +++ suite/plugins/r/compression.reject -@@ -1,13 +1,8 @@ +@@ -1,14 +1,8 @@ # -# Testing bzip2 compression provider with innodb +# Testing lz4 compression provider with mroonga @@ -10,13 +10,14 @@ -call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); -call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); -call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate"); +-call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); -call mtr.add_suppression("Table .*t1.* is compressed with (\\w+), which is not currently loaded. Please load the \\1 provider plugin to open the table"); -create table t1 (a int, b text ) engine = innodb page_compressed = 1; +create table t1 (a int, b text COMMENT 'FLAGS "COLUMN_SCALAR|COMPRESS_LZ4"') engine = mroonga charset = utf8; insert t1 (a, b) values (0, repeat("abc", 100)); insert t1 (a, b) values (1, repeat("def", 1000)); insert t1 (a, b) values (2, repeat("ghi", 10000)); -@@ -16,12 +11,20 @@ +@@ -17,12 +11,20 @@ 0 abcabcabc 300 1 defdefdef 3000 2 ghighighi 30000 diff --git a/mysql-test/suite/plugins/r/compression.result b/mysql-test/suite/plugins/r/compression.result index 07bfbc0b9bb..2319754960c 100644 --- a/mysql-test/suite/plugins/r/compression.result +++ b/mysql-test/suite/plugins/r/compression.result @@ -6,6 +6,7 @@ set global innodb_compression_algorithm = bzip2; call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("Table .*t1.* is compressed with (\\w+), which is not currently loaded. Please load the \\1 provider plugin to open the table"); create table t1 (a int, b text ) engine = innodb page_compressed = 1; insert t1 (a, b) values (0, repeat("abc", 100)); diff --git a/mysql-test/suite/plugins/t/compression.test b/mysql-test/suite/plugins/t/compression.test index 95ae2df9462..e05750f9e99 100644 --- a/mysql-test/suite/plugins/t/compression.test +++ b/mysql-test/suite/plugins/t/compression.test @@ -22,6 +22,7 @@ if ($engine == "innodb") { call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate"); + call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("Table .*t1.* is compressed with (\\w+), which is not currently loaded. Please load the \\1 provider plugin to open the table"); } if ($engine == "mroonga") { diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 7a9558badc2..23b6ddca085 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -510,43 +510,46 @@ buf_page_is_checksum_valid_crc32( return checksum_field1 == crc32; } +#ifndef UNIV_INNOCHECKSUM /** Checks whether the lsn present in the page is lesser than the peek current lsn. -@param[in] check_lsn lsn to check -@param[in] read_buf page. */ -static void buf_page_check_lsn(bool check_lsn, const byte* read_buf) +@param check_lsn lsn to check +@param read_buf page frame +@return whether the FIL_PAGE_LSN is invalid */ +static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) { -#ifndef UNIV_INNOCHECKSUM - if (check_lsn && recv_lsn_checks_on) { - const lsn_t current_lsn = log_sys.get_lsn(); - const lsn_t page_lsn - = mach_read_from_8(read_buf + FIL_PAGE_LSN); + if (!check_lsn) + return false; + lsn_t current_lsn= log_sys.get_lsn(); + if (UNIV_UNLIKELY(current_lsn == log_sys.FIRST_LSN) && + srv_force_recovery == SRV_FORCE_NO_LOG_REDO) + return false; + const lsn_t page_lsn= mach_read_from_8(read_buf + FIL_PAGE_LSN); - /* Since we are going to reset the page LSN during the import - phase it makes no sense to spam the log with error messages. */ - if (current_lsn < page_lsn) { + if (UNIV_LIKELY(current_lsn >= page_lsn)) + return false; - const uint32_t space_id = mach_read_from_4( - read_buf + FIL_PAGE_SPACE_ID); - const uint32_t page_no = mach_read_from_4( - read_buf + FIL_PAGE_OFFSET); + const uint32_t space_id= mach_read_from_4(read_buf + FIL_PAGE_SPACE_ID); + const uint32_t page_no= mach_read_from_4(read_buf + FIL_PAGE_OFFSET); - ib::error() << "Page " << page_id_t(space_id, page_no) - << " log sequence number " << page_lsn - << " is in the future! Current system" - << " log sequence number " - << current_lsn << "."; + sql_print_error("InnoDB: Page " + "[page id: space=" UINT32PF ", page number=" UINT32PF "]" + " log sequence number " LSN_PF + " is in the future! Current system log sequence number " + LSN_PF ".", + space_id, page_no, page_lsn, current_lsn); - ib::error() << "Your database may be corrupt or" - " you may have copied the InnoDB" - " tablespace but not the InnoDB" - " log files. " - << FORCE_RECOVERY_MSG; + if (srv_force_recovery) + return false; - } - } -#endif /* !UNIV_INNOCHECKSUM */ + sql_print_error("InnoDB: Your database may be corrupt or" + " you may have copied the InnoDB" + " tablespace but not the ib_logfile0. %s", + FORCE_RECOVERY_MSG); + + return true; } +#endif /** Check if a buffer is all zeroes. @@ -563,35 +566,36 @@ bool buf_is_zeroes(span buf) @param read_buf database page @param fsp_flags contents of FIL_SPACE_FLAGS @return whether the page is corrupted */ -bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, - uint32_t fsp_flags) +buf_page_is_corrupted_reason +buf_page_is_corrupted(bool check_lsn, const byte *read_buf, uint32_t fsp_flags) { if (fil_space_t::full_crc32(fsp_flags)) { bool compressed = false, corrupted = false; const uint size = buf_page_full_crc32_size( read_buf, &compressed, &corrupted); if (corrupted) { - return true; + return CORRUPTED_OTHER; } const byte* end = read_buf + (size - FIL_PAGE_FCRC32_CHECKSUM); uint crc32 = mach_read_from_4(end); if (!crc32 && size == srv_page_size && buf_is_zeroes(span(read_buf, size))) { - return false; + return NOT_CORRUPTED; } DBUG_EXECUTE_IF( "page_intermittent_checksum_mismatch", { static int page_counter; - if (page_counter++ == 3) { + if (mach_read_from_4(FIL_PAGE_OFFSET + read_buf) + && page_counter++ == 3) { crc32++; } }); if (crc32 != my_crc32c(0, read_buf, size - FIL_PAGE_FCRC32_CHECKSUM)) { - return true; + return CORRUPTED_OTHER; } static_assert(FIL_PAGE_FCRC32_KEY_VERSION == 0, "alignment"); static_assert(FIL_PAGE_LSN % 4 == 0, "alignment"); @@ -603,11 +607,15 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, end - (FIL_PAGE_FCRC32_END_LSN - FIL_PAGE_FCRC32_CHECKSUM), 4)) { - return true; + return CORRUPTED_OTHER; } - buf_page_check_lsn(check_lsn, read_buf); - return false; + return +#ifndef UNIV_INNOCHECKSUM + buf_page_check_lsn(check_lsn, read_buf) + ? CORRUPTED_FUTURE_LSN : +#endif + NOT_CORRUPTED; } const ulint zip_size = fil_space_t::zip_size(fsp_flags); @@ -628,7 +636,13 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, && FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp_flags) #endif ) { - return(false); + check_lsn: + return +#ifndef UNIV_INNOCHECKSUM + buf_page_check_lsn(check_lsn, read_buf) + ? CORRUPTED_FUTURE_LSN : +#endif + NOT_CORRUPTED; } static_assert(FIL_PAGE_LSN % 4 == 0, "alignment"); @@ -641,15 +655,16 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, /* Stored log sequence numbers at the start and the end of page do not match */ - return(true); + return CORRUPTED_OTHER; } - buf_page_check_lsn(check_lsn, read_buf); - /* Check whether the checksum fields have correct values */ if (zip_size) { - return !page_zip_verify_checksum(read_buf, zip_size); + if (!page_zip_verify_checksum(read_buf, zip_size)) { + return CORRUPTED_OTHER; + } + goto check_lsn; } const uint32_t checksum_field1 = mach_read_from_4( @@ -686,7 +701,7 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, } if (all_zeroes) { - return false; + return NOT_CORRUPTED; } } @@ -695,13 +710,17 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, case SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: #endif /* !UNIV_INNOCHECKSUM */ - return !buf_page_is_checksum_valid_crc32( - read_buf, checksum_field1, checksum_field2); + if (!buf_page_is_checksum_valid_crc32(read_buf, + checksum_field1, + checksum_field2)) { + return CORRUPTED_OTHER; + } + goto check_lsn; #ifndef UNIV_INNOCHECKSUM default: if (checksum_field1 == BUF_NO_CHECKSUM_MAGIC && checksum_field2 == BUF_NO_CHECKSUM_MAGIC) { - return false; + goto check_lsn; } const uint32_t crc32 = buf_calc_page_crc32(read_buf); @@ -719,27 +738,35 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, DBUG_EXECUTE_IF( "page_intermittent_checksum_mismatch", { static int page_counter; - if (page_counter++ == 3) return true; + if (mach_read_from_4(FIL_PAGE_OFFSET + + read_buf) + && page_counter++ == 3) + return CORRUPTED_OTHER; }); if ((checksum_field1 != crc32 || checksum_field2 != crc32) && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { - return true; + return CORRUPTED_OTHER; } } switch (checksum_field1) { case 0: case BUF_NO_CHECKSUM_MAGIC: - return false; + break; + default: + if ((checksum_field1 != crc32 + || checksum_field2 != crc32) + && checksum_field1 + != buf_calc_page_new_checksum(read_buf)) { + return CORRUPTED_OTHER; + } } - return (checksum_field1 != crc32 || checksum_field2 != crc32) - && checksum_field1 - != buf_calc_page_new_checksum(read_buf); } #endif /* !UNIV_INNOCHECKSUM */ + goto check_lsn; } #ifndef UNIV_INNOCHECKSUM @@ -3927,6 +3954,7 @@ or decrypt/decompress just failed. @return whether the operation succeeded @retval DB_SUCCESS if page has been read and is not corrupted @retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted +@retval DB_CORRUPTION if the page LSN is in the future @retval DB_DECRYPTION_FAILED if page post encryption checksum matches but after decryption normal page checksum does not match. */ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, @@ -3963,8 +3991,18 @@ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, node.space->is_compressed())) { err = DB_PAGE_CORRUPTED; } - } else if (buf_page_is_corrupted(true, dst_frame, node.space->flags)) { - err = DB_PAGE_CORRUPTED; + } else { + switch (buf_page_is_corrupted(true, dst_frame, + node.space->flags)) { + case NOT_CORRUPTED: + break; + case CORRUPTED_OTHER: + err = DB_PAGE_CORRUPTED; + break; + case CORRUPTED_FUTURE_LSN: + err = DB_CORRUPTION; + break; + } } if (seems_encrypted && err == DB_PAGE_CORRUPTED @@ -3984,9 +4022,8 @@ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, /** Complete a read of a page. @param node data file @return whether the operation succeeded -@retval DB_PAGE_CORRUPTED if the checksum fails -@retval DB_DECRYPTION_FAILED if the page cannot be decrypted -@retval DB_FAIL if the page contains the wrong ID */ +@retval DB_PAGE_CORRUPTED if the checksum or the page ID is incorrect +@retval DB_DECRYPTION_FAILED if the page cannot be decrypted */ dberr_t buf_page_t::read_complete(const fil_node_t &node) { const page_id_t expected_id{id()}; @@ -4014,9 +4051,8 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node) if (!ok) { - ib::info() << "Page " << expected_id << " zip_decompress failure."; err= DB_PAGE_CORRUPTED; - goto database_corrupted; + goto database_corrupted_compressed; } } @@ -4041,15 +4077,21 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node) node.space->crypt_data && node.space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { - ib::error() << "Cannot decrypt " << expected_id; err= DB_DECRYPTION_FAILED; goto release_page; } else { - ib::error() << "Space id and page no stored in the page, read in are " - << read_id << ", should be " << expected_id; - err= DB_PAGE_CORRUPTED; + sql_print_error("InnoDB: Space id and page no stored in the page," + " read in from %s are " + "[page id: space=" UINT32PF ", page number=" UINT32PF + "], should be " + "[page id: space=" UINT32PF ", page number=" UINT32PF + "]", + node.name, + read_id.space(), read_id.page_no(), + expected_id.space(), expected_id.page_no()); + err= DB_FAIL; goto release_page; } } @@ -4059,24 +4101,9 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node) { database_corrupted: if (belongs_to_unzip_LRU()) +database_corrupted_compressed: memset_aligned(frame, 0, srv_page_size); - if (err == DB_PAGE_CORRUPTED) - { - ib::error() << "Database page corruption on disk" - " or a failed read of file '" - << node.name << "' page " << expected_id - << ". You may have to recover from a backup."; - - buf_page_print(read_frame, zip_size()); - - node.space->set_corrupted(); - - ib::info() << " You can use CHECK TABLE to scan" - " your table for corruption. " - << FORCE_RECOVERY_MSG; - } - if (!srv_force_recovery) goto release_page; } @@ -4084,6 +4111,24 @@ database_corrupted: if (err == DB_PAGE_CORRUPTED || err == DB_DECRYPTION_FAILED) { release_page: + if (recv_sys.free_corrupted_page(expected_id, node)); + else if (err == DB_FAIL) + err= DB_PAGE_CORRUPTED; + else + { + sql_print_error("InnoDB: Failed to read page " UINT32PF + " from file '%s': %s", expected_id.page_no(), + node.name, ut_strerr(err)); + + buf_page_print(read_frame, zip_size()); + + if (node.space->set_corrupted() && + !is_predefined_tablespace(node.space->id)) + sql_print_information("InnoDB: You can use CHECK TABLE to scan" + " your table for corruption. %s", + FORCE_RECOVERY_MSG); + } + buf_pool.corrupted_evict(this, buf_page_t::READ_FIX); return err; } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 86e89078dc8..ff2e71b698f 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -33,6 +33,7 @@ Created 2011/12/19 #include "trx0sys.h" #include "fil0crypt.h" #include "fil0pagecompress.h" +#include "log.h" using st_::span; @@ -355,6 +356,9 @@ void buf_dblwr_t::recover() ut_ad(log_sys.last_checkpoint_lsn); if (!is_created()) return; + const lsn_t max_lsn{log_sys.get_lsn()}; + ut_ad(recv_sys.scanned_lsn == max_lsn); + ut_ad(recv_sys.scanned_lsn >= recv_sys.lsn); uint32_t page_no_dblwr= 0; byte *read_buf= static_cast(aligned_malloc(3 * srv_page_size, @@ -364,25 +368,15 @@ void buf_dblwr_t::recover() for (recv_dblwr_t::list::iterator i= recv_sys.dblwr.pages.begin(); i != recv_sys.dblwr.pages.end(); ++i, ++page_no_dblwr) { - byte *page= *i; + const page_t *const page= *i; const uint32_t page_no= page_get_page_no(page); - if (!page_no) /* recovered via recv_dblwr_t::restore_first_page() */ - continue; - const lsn_t lsn= mach_read_from_8(page + FIL_PAGE_LSN); - if (log_sys.last_checkpoint_lsn > lsn) - /* Pages written before the checkpoint are not useful for recovery. */ + if (log_sys.last_checkpoint_lsn > lsn || lsn > recv_sys.lsn) + /* Pages written before or after the recovery range are not usable. */ continue; const uint32_t space_id= page_get_space_id(page); const page_id_t page_id(space_id, page_no); - if (recv_sys.scanned_lsn < lsn) - { - ib::info() << "Ignoring a doublewrite copy of page " << page_id - << " with future log sequence number " << lsn; - continue; - } - fil_space_t *space= fil_space_t::get(space_id); if (!space) @@ -394,10 +388,14 @@ void buf_dblwr_t::recover() /* Do not report the warning for undo tablespaces, because they can be truncated in place. */ if (!srv_is_undo_tablespace(space_id)) - ib::warn() << "A copy of page " << page_no - << " in the doublewrite buffer slot " << page_no_dblwr - << " is beyond the end of " << space->chain.start->name - << " (" << space->size << " pages)"; + sql_print_warning("InnoDB: A copy of page " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " in the doublewrite buffer slot " UINT32PF + " is beyond the end of %s (" UINT32PF " pages)", + page_id.space(), page_id.page_no(), + page_no_dblwr, space->chain.start->name, + space->size); next_page: space->release(); continue; @@ -416,41 +414,48 @@ next_page: physical_size, read_buf); if (UNIV_UNLIKELY(fio.err != DB_SUCCESS)) - { - ib::warn() << "Double write buffer recovery: " << page_id - << " ('" << space->chain.start->name - << "') read failed with error: " << fio.err; - continue; - } - - if (buf_is_zeroes(span(read_buf, physical_size))) + sql_print_warning("InnoDB: Double write buffer recovery: " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " ('%s') read failed with error: %s", + page_id.space(), page_id.page_no(), fio.node->name, + ut_strerr(fio.err)); + else if (buf_is_zeroes(span(read_buf, physical_size))) { /* We will check if the copy in the doublewrite buffer is valid. If not, we will ignore this page (there should be redo log records to initialize it). */ } - else if (recv_sys.dblwr.validate_page(page_id, read_buf, space, buf)) + else if (recv_sys.dblwr.validate_page(page_id, max_lsn, space, + read_buf, buf)) goto next_page; else /* We intentionally skip this message for all-zero pages. */ - ib::info() << "Trying to recover page " << page_id - << " from the doublewrite buffer."; + sql_print_information("InnoDB: Trying to recover page " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " from the doublewrite buffer.", + page_id.space(), page_id.page_no()); - page= recv_sys.dblwr.find_page(page_id, space, buf); + if (const byte *page= + recv_sys.dblwr.find_page(page_id, max_lsn, space, buf)) + { + /* Write the good page from the doublewrite buffer to the intended + position. */ + space->reacquire(); + fio= space->io(IORequestWrite, + os_offset_t{page_id.page_no()} * physical_size, + physical_size, const_cast(page)); - if (!page) - goto next_page; + if (fio.err == DB_SUCCESS) + sql_print_information("InnoDB: Recovered page " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " to '%s' from the doublewrite buffer.", + page_id.space(), page_id.page_no(), + fio.node->name); + } - /* Write the good page from the doublewrite buffer to the intended - position. */ - space->reacquire(); - fio= space->io(IORequestWrite, - os_offset_t{page_id.page_no()} * physical_size, - physical_size, page); - - if (fio.err == DB_SUCCESS) - ib::info() << "Recovered page " << page_id << " to '" << fio.node->name - << "' from the doublewrite buffer."; goto next_page; } diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 2076322179a..0490cb9143e 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1017,7 +1017,7 @@ buf_LRU_block_free_non_file_page( } /** Release a memory block to the buffer pool. */ -ATTRIBUTE_COLD void buf_pool_t::free_block(buf_block_t *block) +ATTRIBUTE_COLD void buf_pool_t::free_block(buf_block_t *block) noexcept { ut_ad(this == &buf_pool); mysql_mutex_lock(&mutex); @@ -1168,13 +1168,12 @@ static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, const page_id_t id, @param bpage x-latched page that was found corrupted @param state expected current state of the page */ ATTRIBUTE_COLD -void buf_pool_t::corrupted_evict(buf_page_t *bpage, uint32_t state) +void buf_pool_t::corrupted_evict(buf_page_t *bpage, uint32_t state) noexcept { const page_id_t id{bpage->id()}; buf_pool_t::hash_chain &chain= buf_pool.page_hash.cell_get(id.fold()); page_hash_latch &hash_lock= buf_pool.page_hash.lock_get(chain); - recv_sys.free_corrupted_page(id); mysql_mutex_lock(&mutex); hash_lock.lock(); diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 6e509b77141..f2ffee0651f 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -326,15 +326,13 @@ buf_read_page_low( dst, bpage); if (UNIV_UNLIKELY(fio.err != DB_SUCCESS)) { + recv_sys.free_corrupted_page(page_id, *space->chain.start); buf_pool.corrupted_evict(bpage, buf_page_t::READ_FIX); } else if (sync) { thd_wait_end(nullptr); /* The i/o was already completed in space->io() */ fio.err = bpage->read_complete(*fio.node); space->release(); - if (fio.err == DB_FAIL) { - fio.err = DB_PAGE_CORRUPTED; - } if (mariadb_timer) { mariadb_increment_pages_read_time(mariadb_timer); } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index b38aef931e3..7af86af7177 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -59,10 +59,14 @@ Created 10/25/1995 Heikki Tuuri #include "bzlib.h" #include "snappy-c.h" -ATTRIBUTE_COLD void fil_space_t::set_corrupted() const +ATTRIBUTE_COLD bool fil_space_t::set_corrupted() const noexcept { if (!is_stopping() && !is_corrupted.test_and_set()) + { sql_print_error("InnoDB: File '%s' is corrupted", chain.start->name); + return true; + } + return false; } /** Try to close a file to adhere to the innodb_open_files limit. @@ -335,11 +339,14 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, return node; } -__attribute__((warn_unused_result, nonnull)) +__attribute__((warn_unused_result, nonnull(1))) /** Open a tablespace file. @param node data file +@param page first page of the tablespace, or nullptr +@param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the file was successfully opened */ -static bool fil_node_open_file_low(fil_node_t *node) +static bool fil_node_open_file_low(fil_node_t *node, const byte *page, + bool no_lsn) { ut_ad(!node->is_open()); ut_ad(node->space->is_closing()); @@ -395,7 +402,7 @@ static bool fil_node_open_file_low(fil_node_t *node) bool comp_algo_invalid = false; if (node->size); - else if (!node->read_page0() || + else if (!node->read_page0(page, no_lsn) || // validate compression algorithm for full crc32 format (node->space->full_crc32() && (comp_algo_invalid = !fil_comp_algo_loaded(comp_algo)))) @@ -426,8 +433,10 @@ static bool fil_node_open_file_low(fil_node_t *node) /** Open a tablespace file. @param node data file +@param page first page of the tablespace, or nullptr +@param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the file was successfully opened */ -static bool fil_node_open_file(fil_node_t *node) +static bool fil_node_open_file(fil_node_t *node, const byte *page, bool no_lsn) { mysql_mutex_assert_owner(&fil_system.mutex); ut_ad(!node->is_open()); @@ -466,7 +475,7 @@ static bool fil_node_open_file(fil_node_t *node) /* The node can be opened beween releasing and acquiring fil_system.mutex in the above code */ - return node->is_open() || fil_node_open_file_low(node); + return node->is_open() || fil_node_open_file_low(node, page, no_lsn); } /** Close the file handle. */ @@ -677,7 +686,8 @@ ATTRIBUTE_COLD bool fil_space_t::prepare_acquired() ut_ad(!id || purpose == FIL_TYPE_TEMPORARY || node == UT_LIST_GET_FIRST(chain)); - const bool is_open= node && (node->is_open() || fil_node_open_file(node)); + const bool is_open= node && + (node->is_open() || fil_node_open_file(node, nullptr, false)); if (!is_open) release(); @@ -1104,8 +1114,10 @@ bool fil_assign_new_space_id(uint32_t *space_id) } /** Read the first page of a data file. +@param dpage copy of a first page, from the doublewrite buffer, or nullptr +@param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the page was found valid */ -bool fil_space_t::read_page0() +bool fil_space_t::read_page0(const byte *dpage, bool no_lsn) noexcept { ut_ad(fil_system.is_initialised()); mysql_mutex_assert_owner(&fil_system.mutex); @@ -1122,32 +1134,26 @@ bool fil_space_t::read_page0() ut_ad("this should not happen" == 0); return false; } - const bool ok= node->is_open() || fil_node_open_file(node); + const bool ok= node->is_open() || fil_node_open_file(node, dpage, no_lsn); release(); return ok; } -/** Look up a tablespace and ensure that its first page has been validated. */ -static fil_space_t *fil_space_get_space(uint32_t id) -{ - if (fil_space_t *space= fil_space_get_by_id(id)) - if (space->read_page0()) - return space; - return nullptr; -} - void fil_space_set_recv_size_and_flags(uint32_t id, uint32_t size, uint32_t flags) { ut_ad(id < SRV_SPACE_ID_UPPER_BOUND); + mysql_mutex_assert_owner(&recv_sys.mutex); mysql_mutex_lock(&fil_system.mutex); - if (fil_space_t *space= fil_space_get_space(id)) - { - if (size) - space->recv_size= size; - if (flags != FSP_FLAGS_FCRC32_MASK_MARKER) - space->flags= flags; - } + if (fil_space_t *space= fil_space_get_by_id(id)) + if (space->read_page0(recv_sys.dblwr.find_page(page_id_t(id, 0), LSN_MAX), + true)) + { + if (size) + space->recv_size= size; + if (flags != FSP_FLAGS_FCRC32_MASK_MARKER) + space->flags= flags; + } mysql_mutex_unlock(&fil_system.mutex); } @@ -1162,12 +1168,16 @@ bool fil_space_t::open(bool create_new_db) bool success= true; bool skip_read= create_new_db; + const page_t *page= skip_read + ? nullptr + : recv_sys.dblwr.find_page(page_id_t{id, 0}, LSN_MAX); + mysql_mutex_lock(&fil_system.mutex); for (fil_node_t *node= UT_LIST_GET_FIRST(chain); node; node= UT_LIST_GET_NEXT(chain, node)) { - if (!node->is_open() && !fil_node_open_file_low(node)) + if (!node->is_open() && !fil_node_open_file_low(node, page, page)) { err_exit: success= false; @@ -1185,7 +1195,7 @@ err_exit: continue; } - if (!node->read_page0()) + if (!node->read_page0(page, true)) { fil_system.n_open--; os_file_close(node->handle); @@ -1194,6 +1204,7 @@ err_exit: } skip_read= true; + page= nullptr; } if (!create_new_db) @@ -2888,6 +2899,7 @@ void IORequest::read_complete(int io_error) const { sql_print_error("InnoDB: Read error %d of page " UINT32PF " in file %s", io_error, id.page_no(), node->name); + recv_sys.free_corrupted_page(id, *node); buf_pool.corrupted_evict(bpage, buf_page_t::READ_FIX); corrupted: if (recv_recovery_is_on() && !srv_force_recovery) @@ -2897,13 +2909,8 @@ void IORequest::read_complete(int io_error) const mysql_mutex_unlock(&recv_sys.mutex); } } - else if (dberr_t err= bpage->read_complete(*node)) - { - if (err != DB_FAIL) - ib::error() << "Failed to read page " << id.page_no() - << " from file '" << node->name << "': " << err; + else if (bpage->read_complete(*node)) goto corrupted; - } node->space->release(); } diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index 62f90f53a54..637d8eef00c 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -234,6 +234,46 @@ Datafile::same_as( #endif /* WIN32 */ } +dberr_t Datafile::read_first_page_flags(const page_t *page) noexcept +{ + ut_ad(m_order == 0); + + if (memcmp_aligned<2>(FIL_PAGE_SPACE_ID + page, + FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4)) + { + sql_print_error("InnoDB: Inconsistent tablespace ID in %s", m_filepath); + return DB_CORRUPTION; + } + + m_space_id= mach_read_from_4(FIL_PAGE_SPACE_ID + page); + m_flags= fsp_header_get_flags(page); + if (!fil_space_t::is_valid_flags(m_flags, m_space_id)) + { + uint32_t cflags= fsp_flags_convert_from_101(m_flags); + if (cflags == UINT32_MAX) + switch (fsp_flags_is_incompatible_mysql(m_flags)) { + case 0: + sql_print_error("InnoDB: Invalid flags 0x%zx in %s", + m_flags, m_filepath); + return DB_CORRUPTION; + case 3: + case 2: + sql_print_error("InnoDB: MySQL-8.0 tablespace in %s", m_filepath); + goto unsupported; + case 1: + sql_print_error("InnoDB: MySQL Encrypted tablespace in %s", + m_filepath); + unsupported: + sql_print_error("InnoDB: Restart in MySQL for migration/recovery."); + return DB_UNSUPPORTED; + } + else + m_flags= cflags; + } + + return DB_SUCCESS; +} + /** Reads a few significant fields from the first page of the first datafile. The Datafile must already be open. @param[in] read_only_mode If true, then readonly mode checks are enforced. @@ -286,56 +326,16 @@ Datafile::read_first_page(bool read_only_mode) } } - if (err != DB_SUCCESS) { - return(err); + if (err == DB_SUCCESS && m_order == 0) { + err = read_first_page_flags(m_first_page); } - if (m_order == 0) { - if (memcmp_aligned<2>(FIL_PAGE_SPACE_ID + m_first_page, - FSP_HEADER_OFFSET + FSP_SPACE_ID - + m_first_page, 4)) { - ib::error() - << "Inconsistent tablespace ID in " - << m_filepath; - return DB_CORRUPTION; - } - - m_space_id = mach_read_from_4(FIL_PAGE_SPACE_ID - + m_first_page); - m_flags = fsp_header_get_flags(m_first_page); - if (!fil_space_t::is_valid_flags(m_flags, m_space_id)) { - uint32_t cflags = fsp_flags_convert_from_101(m_flags); - if (cflags == UINT32_MAX) { - switch (fsp_flags_is_incompatible_mysql(m_flags)) { - case 0: - sql_print_error("InnoDB: Invalid flags 0x%" PRIx32 " in %s", - m_flags, m_filepath); - return DB_CORRUPTION; - case 3: - case 2: - sql_print_error("InnoDB: MySQL-8.0 tablespace in %s", - m_filepath); - break; - case 1: - sql_print_error("InnoDB: MySQL Encrypted tablespace in %s", - m_filepath); - break; - } - sql_print_error("InnoDB: Restart in MySQL for migration/recovery."); - return DB_UNSUPPORTED; - } else { - m_flags = cflags; - } - } - } - - const size_t physical_size = fil_space_t::physical_size(m_flags); - - if (physical_size > page_size) { + if (err == DB_SUCCESS + && fil_space_t::physical_size(m_flags) > page_size) { ib::error() << "File " << m_filepath << " should be longer than " << page_size << " bytes"; - return(DB_CORRUPTION); + err = DB_CORRUPTION; } return(err); @@ -366,7 +366,7 @@ dberr_t Datafile::validate_to_dd(uint32_t space_id, uint32_t flags) /* Validate this single-table-tablespace with the data dictionary, but do not compare the DATA_DIR flag, in case the tablespace was remotely located. */ - err = validate_first_page(); + err = validate_first_page(m_first_page); if (err != DB_SUCCESS) { return(err); } @@ -394,6 +394,72 @@ dberr_t Datafile::validate_to_dd(uint32_t space_id, uint32_t flags) return(DB_ERROR); } +inline +uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file) + const noexcept +{ + os_offset_t file_size= os_file_get_size(file); + if (file_size != (os_offset_t) -1) + { + for (const page_t *page : pages) + { + uint32_t space_id= page_get_space_id(page); + byte *read_page= nullptr; + if (page_get_page_no(page) > 0 || space_id == 0) + { +next_page: + aligned_free(read_page); + continue; + } + uint32_t flags= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + page); + size_t page_size= fil_space_t::physical_size(flags); + if (file_size < 4 * page_size) + goto next_page; + read_page= + static_cast(aligned_malloc(3 * page_size, page_size)); + /* Read 3 pages from the file and match the space id + with the space id which is stored in + doublewrite buffer page. */ + if (os_file_read(IORequestRead, file, read_page, page_size, + 3 * page_size, nullptr) != DB_SUCCESS) + goto next_page; + for (ulint j= 0; j <= 2; j++) + { + byte *cur_page= read_page + j * page_size; + if (buf_is_zeroes(span(cur_page, page_size))) + { + aligned_free(read_page); + return 0; + } + if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 || + memcmp(cur_page + FIL_PAGE_SPACE_ID, + page + FIL_PAGE_SPACE_ID, 4) || + buf_page_is_corrupted(false, cur_page, flags)) + goto next_page; + } + + aligned_free(read_page); + page= find_page(page_id_t{space_id, 0}, LSN_MAX); + + if (!page) + { + /* If the first page of the given user tablespace is not there + in the doublewrite buffer, then the recovery is going to fail + now. Report error only when doublewrite buffer is not empty */ + sql_print_error("InnoDB: Corrupted page " + "[page id: space=" UINT32PF ", page number=0]" + " of datafile '%s' could not be found" + " in the doublewrite buffer", space_id, name); + break; + } + + return space_id; + } + } + return 0; +} + /** Validates this datafile for the purpose of recovery. The file should exist and be successfully opened. We initially open it in read-only mode because we just want to read the SpaceID. However, if the first page is @@ -409,7 +475,7 @@ Datafile::validate_for_recovery() ut_ad(is_open()); ut_ad(!srv_read_only_mode); - err = validate_first_page(); + err = validate_first_page(m_first_page); switch (err) { case DB_TABLESPACE_EXISTS: @@ -426,14 +492,7 @@ Datafile::validate_for_recovery() m_space_id is set in read_first_page(). */ /* fall through */ default: - /* Re-open the file in read-write mode Attempt to restore - page 0 from doublewrite and read the space ID from a survey - of the first few pages. */ - close(); - err = open_read_write(); - if (err != DB_SUCCESS) { - return(err); - } + const page_t *first_page = nullptr; if (!m_space_id) { m_space_id = recv_sys.dblwr.find_first_page( @@ -459,15 +518,17 @@ Datafile::validate_for_recovery() return DB_SUCCESS; /* empty file */ } - if (recv_sys.dblwr.restore_first_page( - m_space_id, m_filepath, m_handle)) { + first_page = recv_sys.dblwr.find_page( + page_id_t(m_space_id, 0), LSN_MAX); + + if (!first_page) { return m_defer ? err : DB_CORRUPTION; } free_first_page: /* Free the previously read first page and then re-validate. */ free_first_page(); m_defer = false; - err = validate_first_page(); + err = validate_first_page(first_page); } return(err); @@ -477,22 +538,24 @@ free_first_page: tablespace is opened. This occurs before the fil_space_t is created so the Space ID found here must not already be open. m_is_valid is set true on success, else false. +@param[in] first_page the contents of the first page @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ -dberr_t Datafile::validate_first_page() +dberr_t Datafile::validate_first_page(const page_t *first_page) noexcept { const char* error_txt = NULL; m_is_valid = true; + ut_ad(first_page || !m_first_page); - if (m_first_page == NULL - && read_first_page(srv_read_only_mode) != DB_SUCCESS) { - + if (first_page) { + if (dberr_t err = read_first_page_flags(first_page)) { + m_is_valid = false; + return err; + } + } else if (read_first_page(srv_read_only_mode) != DB_SUCCESS) { error_txt = "Cannot read first page"; - } - - if (error_txt != NULL) { err_exit: free_first_page(); @@ -508,11 +571,14 @@ err_exit: error_txt, m_filepath, m_space_id, m_flags); m_is_valid = false; return DB_CORRUPTION; + } else { + first_page = m_first_page; + ut_ad(first_page); } /* Check if the whole page is blank. */ if (!m_space_id && !m_flags) { - const byte* b = m_first_page; + const byte* b = first_page; ulint nonzero_bytes = srv_page_size; while (*b == '\0' && --nonzero_bytes != 0) { @@ -550,7 +616,7 @@ err_exit: return(DB_ERROR); } - if (page_get_page_no(m_first_page) != 0) { + if (page_get_page_no(first_page) != 0) { /* First page must be number 0 */ error_txt = "Header page contains inconsistent data"; goto err_exit; @@ -561,8 +627,13 @@ err_exit: goto err_exit; } - if (buf_page_is_corrupted(false, m_first_page, m_flags)) { - /* Look for checksum and other corruptions. */ + switch (buf_page_is_corrupted(false, first_page, m_flags)) { + case NOT_CORRUPTED: + break; + case CORRUPTED_FUTURE_LSN: + error_txt = "LSN is in the future"; + goto err_exit; + case CORRUPTED_OTHER: error_txt = "Checksum mismatch"; goto err_exit; } diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index 4ac9da50281..5771176ec2d 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -583,17 +583,22 @@ inline dberr_t SysTablespace::read_lsn_and_check_flags() /* Check the contents of the first page of the first datafile. */ - err = it->validate_first_page(); + err = it->validate_first_page(it->m_first_page); + const page_t *first_page = it->m_first_page; if (err != DB_SUCCESS) { - if (recv_sys.dblwr.restore_first_page( - it->m_space_id, it->m_filepath, - it->handle())) { - it->close(); - return(err); + mysql_mutex_lock(&recv_sys.mutex); + first_page = recv_sys.dblwr.find_page( + page_id_t(space_id(), 0), LSN_MAX); + mysql_mutex_unlock(&recv_sys.mutex); + if (!first_page) { + err = DB_CORRUPTION; + } else { + err = it->read_first_page_flags(first_page); + if (err == DB_SUCCESS) { + err = it->validate_first_page(first_page); + } } - err = it->read_first_page( - m_ignore_read_only && srv_read_only_mode); } /* Make sure the tablespace space ID matches the @@ -616,7 +621,7 @@ inline dberr_t SysTablespace::read_lsn_and_check_flags() log_sys.latch.wr_lock(SRW_LOCK_CALL); /* Prepare for possible upgrade from 0-sized ib_logfile0. */ log_sys.next_checkpoint_lsn = mach_read_from_8( - it->m_first_page + 26/*FIL_PAGE_FILE_FLUSH_LSN*/); + first_page + 26/*FIL_PAGE_FILE_FLUSH_LSN*/); if (log_sys.next_checkpoint_lsn < 8204) { /* Before MDEV-14425, InnoDB had a minimum LSN of 8192+12=8204. Likewise, mariadb-backup diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index b66f461c9da..f2704273dc5 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -308,13 +308,21 @@ buf_block_modify_clock_inc( @return whether the buffer is all zeroes */ bool buf_is_zeroes(st_::span buf); +/** Reason why buf_page_is_corrupted() fails */ +enum buf_page_is_corrupted_reason +{ + CORRUPTED_FUTURE_LSN= -1, + NOT_CORRUPTED= 0, + CORRUPTED_OTHER +}; + /** Check if a page is corrupt. @param check_lsn whether FIL_PAGE_LSN should be checked @param read_buf database page @param fsp_flags contents of FIL_SPACE_FLAGS @return whether the page is corrupted */ -bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, - uint32_t fsp_flags) +buf_page_is_corrupted_reason +buf_page_is_corrupted(bool check_lsn, const byte *read_buf, uint32_t fsp_flags) MY_ATTRIBUTE((warn_unused_result)); /** Read the key version from the page. In full crc32 format, @@ -778,9 +786,8 @@ public: /** Complete a read of a page. @param node data file @return whether the operation succeeded - @retval DB_PAGE_CORRUPTED if the checksum fails - @retval DB_DECRYPTION_FAILED if the page cannot be decrypted - @retval DB_FAIL if the page contains the wrong ID */ + @retval DB_PAGE_CORRUPTED if the checksum or the page ID is incorrect + @retval DB_DECRYPTION_FAILED if the page cannot be decrypted */ dberr_t read_complete(const fil_node_t &node); /** Release a write fix after a page write was completed. @@ -1362,10 +1369,11 @@ public: /** Release and evict a corrupted page. @param bpage x-latched page that was found corrupted @param state expected current state of the page */ - ATTRIBUTE_COLD void corrupted_evict(buf_page_t *bpage, uint32_t state); + ATTRIBUTE_COLD void corrupted_evict(buf_page_t *bpage, uint32_t state) + noexcept; /** Release a memory block to the buffer pool. */ - ATTRIBUTE_COLD void free_block(buf_block_t *block); + ATTRIBUTE_COLD void free_block(buf_block_t *block) noexcept; #ifdef UNIV_DEBUG /** Find a block that points to a ROW_FORMAT=COMPRESSED page diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 0cc55eff03f..6ac6120a0a9 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -513,8 +513,9 @@ public: written while the space ID is being updated in each page. */ inline void set_imported(); - /** Report the tablespace as corrupted */ - ATTRIBUTE_COLD void set_corrupted() const; + /** Report the tablespace as corrupted + @return whether this was the first call */ + ATTRIBUTE_COLD bool set_corrupted() const noexcept; /** @return whether the storage device is rotational (HDD, not SSD) */ inline bool is_rotational() const; @@ -969,8 +970,10 @@ public: void flush_low(); /** Read the first page of a data file. + @param dpage copy of a first page, from the doublewrite buffer, or nullptr + @param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the page was found valid */ - bool read_page0(); + bool read_page0(const byte *dpage, bool no_lsn) noexcept; /** Determine the next tablespace for encryption key rotation. @param space current tablespace (nullptr to start from the beginning) @@ -1067,8 +1070,10 @@ struct fil_node_t final bool is_open() const { return handle != OS_FILE_CLOSED; } /** Read the first page of a data file. + @param dpage copy of a first page, from the doublewrite buffer, or nullptr + @param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the page was found valid */ - bool read_page0(); + bool read_page0(const byte *dpage, bool no_lsn) noexcept; /** Determine some file metadata when creating or reading the file. @param file the file that is being created, or OS_FILE_CLOSED */ @@ -1543,7 +1548,7 @@ inline uint32_t fil_space_t::get_size() if (!size) { mysql_mutex_lock(&fil_system.mutex); - read_page0(); + read_page0(nullptr, false); mysql_mutex_unlock(&fil_system.mutex); } return size; diff --git a/storage/innobase/include/fsp0file.h b/storage/innobase/include/fsp0file.h index 67e79f1aa81..be32887c8dc 100644 --- a/storage/innobase/include/fsp0file.h +++ b/storage/innobase/include/fsp0file.h @@ -216,10 +216,11 @@ public: tablespace is opened. This occurs before the fil_space_t is created so the Space ID found here must not already be open. m_is_valid is set true on success, else false. + @param first_page the contents of the first page @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ - dberr_t validate_first_page() + dberr_t validate_first_page(const byte *first_page) noexcept MY_ATTRIBUTE((warn_unused_result)); /** Get Datafile::m_filepath. @@ -358,6 +359,13 @@ private: dberr_t read_first_page(bool read_only_mode) MY_ATTRIBUTE((warn_unused_result)); + /** Read m_space_id, m_flags from a page frame. + @param page a copy of the first page of the tablespace + @retval DB_SUCCESS if the page seems to be valid + @retval DB_CORRUPTION if the page looks corrupted + @retval DB_UNSUPPORTED if the page is in an unsupported format */ + dberr_t read_first_page_flags(const byte *page) noexcept; + /** Free the first page from memory when it is no longer needed. */ void free_first_page(); diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 731d7e3ff11..c6d2c3925c4 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -95,32 +95,28 @@ struct recv_dblwr_t /** Validate the page. @param page_id page identifier - @param page page contents + @param max_lsn the maximum allowed LSN @param space the tablespace of the page (not available for page 0) + @param page page contents @param tmp_buf 2*srv_page_size for decrypting and decompressing any page_compressed or encrypted pages @return whether the page is valid */ - bool validate_page(const page_id_t page_id, const byte *page, - const fil_space_t *space, byte *tmp_buf); + bool validate_page(const page_id_t page_id, lsn_t max_lsn, + const fil_space_t *space, + const byte *page, byte *tmp_buf) const noexcept; - /** Find a doublewrite copy of a page. + /** Find a doublewrite copy of a page with the smallest FIL_PAGE_LSN + that is large enough for recovery. @param page_id page identifier - @param space tablespace (not available for page_id.page_no()==0) + @param max_lsn the maximum allowed LSN + @param space tablespace (nullptr for page_id.page_no()==0) @param tmp_buf 2*srv_page_size for decrypting and decompressing any page_compressed or encrypted pages @return page frame - @retval NULL if no valid page for page_id was found */ - byte* find_page(const page_id_t page_id, const fil_space_t *space= NULL, - byte *tmp_buf= NULL); - - /** Restore the first page of the given tablespace from - doublewrite buffer. - @param space_id tablespace identifier - @param name tablespace filepath - @param file tablespace file handle - @return whether the operation failed */ - bool restore_first_page(uint32_t space_id, const char *name, - pfs_os_file_t file); + @retval nullptr if no valid page for page_id was found */ + const byte *find_page(const page_id_t page_id, lsn_t max_lsn, + const fil_space_t *space= nullptr, + byte *tmp_buf= nullptr) const noexcept; /** Restore the first page of the given tablespace from doublewrite buffer. @@ -131,7 +127,8 @@ struct recv_dblwr_t @param name tablespace filepath @param file tablespace file handle @return space_id or 0 in case of error */ - uint32_t find_first_page(const char *name, pfs_os_file_t file); + inline uint32_t find_first_page(const char *name, pfs_os_file_t file) + const noexcept; typedef std::deque > list; @@ -448,20 +445,26 @@ public: inline void free(const void *data); /** Remove records for a corrupted page. - This function should only be called when innodb_force_recovery is set. - @param page_id corrupted page identifier */ - ATTRIBUTE_COLD void free_corrupted_page(page_id_t page_id); + @param page_id corrupted page identifier + @param node file for which an error is to be reported + @return whether an error message was reported */ + ATTRIBUTE_COLD bool free_corrupted_page(page_id_t page_id, + const fil_node_t &node) noexcept; /** Flag data file corruption during recovery. */ - ATTRIBUTE_COLD void set_corrupt_fs(); + ATTRIBUTE_COLD void set_corrupt_fs() noexcept; /** Flag log file corruption during recovery. */ - ATTRIBUTE_COLD void set_corrupt_log(); + ATTRIBUTE_COLD void set_corrupt_log() noexcept; /** @return whether data file corruption was found */ bool is_corrupt_fs() const { return UNIV_UNLIKELY(found_corrupt_fs); } /** @return whether log file corruption was found */ bool is_corrupt_log() const { return UNIV_UNLIKELY(found_corrupt_log); } + /** Check if recovery reached a consistent log sequence number. + @return whether the recovery failed to process enough log */ + inline bool validate_checkpoint() const noexcept; + /** Attempt to initialize a page based on redo log records. @param page_id page identifier @return the recovered block @@ -503,8 +506,3 @@ extern bool recv_needed_recovery; protected by exclusive log_sys.latch. */ extern bool recv_no_log_write; #endif /* UNIV_DEBUG */ - -/** TRUE if buf_page_is_corrupted() should check if the log sequence -number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by -recv_recovery_from_checkpoint_start(). */ -extern bool recv_lsn_checks_on; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index a8c3af695f0..a14b3912e6d 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -66,11 +66,6 @@ Protected by log_sys.latch. */ bool recv_no_log_write = false; #endif /* UNIV_DEBUG */ -/** TRUE if buf_page_is_corrupted() should check if the log sequence -number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by -recv_recovery_from_checkpoint_start(). */ -bool recv_lsn_checks_on; - /** If the following is TRUE, the buffer pool file pages must be invalidated after recovery and no ibuf operations are allowed; this becomes TRUE if the log record hash table becomes too full, and log records must be merged @@ -82,11 +77,6 @@ true means that recovery is running and no operations on the log file are allowed yet: the variable name is misleading. */ bool recv_no_ibuf_operations; -/** The maximum lsn we see for a page during the recovery process. If this -is bigger than the lsn we are able to scan up to, that is an indication that -the recovery failed and the database may be corrupt. */ -static lsn_t recv_max_page_lsn; - /** Stored physical log record */ struct log_phys_t : public log_rec_t { @@ -857,7 +847,7 @@ processed: This is invoked if we found neither a valid first page in the data file nor redo log records that would initialize the first page. */ - void deferred_dblwr() + void deferred_dblwr(lsn_t max_lsn) { for (auto d= defers.begin(); d != defers.end(); ) { @@ -868,7 +858,7 @@ processed: continue; } const page_id_t page_id{d->first, 0}; - const byte *page= recv_sys.dblwr.find_page(page_id); + const byte *page= recv_sys.dblwr.find_page(page_id, max_lsn); if (!page) goto next_item; const uint32_t space_id= mach_read_from_4(page + FIL_PAGE_SPACE_ID); @@ -1442,7 +1432,6 @@ void recv_sys_t::create() progress_time = time(NULL); ut_ad(pages.empty()); pages_it = pages.end(); - recv_max_page_lsn = 0; memset(truncated_undo_spaces, 0, sizeof truncated_undo_spaces); UT_LIST_INIT(blocks, &buf_block_t::unzip_LRU); @@ -1791,6 +1780,8 @@ dberr_t recv_sys_t::find_checkpoint() log_sys.last_checkpoint_lsn= log_sys.next_checkpoint_lsn; log_sys.set_recovered_lsn(log_sys.next_checkpoint_lsn); lsn= file_checkpoint= log_sys.next_checkpoint_lsn; + if (UNIV_LIKELY(lsn != 0)) + scanned_lsn= lsn; log_sys.next_checkpoint_no= 0; return DB_SUCCESS; } @@ -2960,7 +2951,8 @@ restart: l - recs + rlen))) { lsn= start_lsn; - log_sys.set_recovered_lsn(start_lsn); + if (lsn > log_sys.get_lsn()) + log_sys.set_recovered_lsn(start_lsn); l+= rlen; offset= begin.ptr - log_sys.buf; rewind(l, begin); @@ -2972,7 +2964,7 @@ restart: goto restart; } sql_print_information("InnoDB: Multi-batch recovery needed at LSN " - LSN_PF, lsn); + LSN_PF, start_lsn); return GOT_OOM; } } @@ -3325,11 +3317,18 @@ set_start_lsn: mtr.discard_modifications(); mtr.commit(); + fil_space_t* s = space + ? space + : fil_space_t::get(block->page.id().space()); + buf_pool.corrupted_evict(&block->page, block->page.state() & buf_page_t::LRU_MASK); - block = nullptr; - goto done; + if (!space) { + s->release(); + } + + return nullptr; } if (!start_lsn) { @@ -3372,29 +3371,26 @@ set_start_lsn: mtr.discard_modifications(); mtr.commit(); -done: - /* FIXME: do this in page read, protected with recv_sys.mutex! */ - if (recv_max_page_lsn < page_lsn) { - recv_max_page_lsn = page_lsn; - } - return block; } /** Remove records for a corrupted page. -This function should only be called when innodb_force_recovery is set. -@param page_id corrupted page identifier */ -ATTRIBUTE_COLD void recv_sys_t::free_corrupted_page(page_id_t page_id) +@param page_id corrupted page identifier +@param node file for which an error is to be reported +@return whether an error message was reported */ +ATTRIBUTE_COLD +bool recv_sys_t::free_corrupted_page(page_id_t page_id, + const fil_node_t &node) noexcept { if (!recovery_on) - return; + return false; mysql_mutex_lock(&mutex); map::iterator p= pages.find(page_id); if (p == pages.end()) { mysql_mutex_unlock(&mutex); - return; + return false; } p->second.being_processed= -1; @@ -3402,18 +3398,20 @@ ATTRIBUTE_COLD void recv_sys_t::free_corrupted_page(page_id_t page_id) set_corrupt_fs(); mysql_mutex_unlock(&mutex); - ib::error_or_warn(!srv_force_recovery) - << "Unable to apply log to corrupted page " << page_id; + (srv_force_recovery ? sql_print_warning : sql_print_error) + ("InnoDB: Unable to apply log to corrupted page " UINT32PF + " in file %s", page_id.page_no(), node.name); + return true; } -ATTRIBUTE_COLD void recv_sys_t::set_corrupt_log() +ATTRIBUTE_COLD void recv_sys_t::set_corrupt_log() noexcept { mysql_mutex_lock(&mutex); found_corrupt_log= true; mysql_mutex_unlock(&mutex); } -ATTRIBUTE_COLD void recv_sys_t::set_corrupt_fs() +ATTRIBUTE_COLD void recv_sys_t::set_corrupt_fs() noexcept { mysql_mutex_assert_owner(&mutex); if (!srv_force_recovery) @@ -4573,6 +4571,18 @@ inline void log_t::set_recovered() noexcept set_buf_free(offset); } +inline bool recv_sys_t::validate_checkpoint() const noexcept +{ + if (lsn >= file_checkpoint && lsn >= log_sys.next_checkpoint_lsn) + return false; + sql_print_error("InnoDB: The log was only scanned up to " + LSN_PF ", while the current LSN at the " + "time of the latest checkpoint " LSN_PF + " was " LSN_PF "!", + lsn, log_sys.next_checkpoint_lsn, file_checkpoint); + return true; +} + /** Start recovering from a redo log checkpoint. of first system tablespace page @return error code or DB_SUCCESS */ @@ -4600,9 +4610,7 @@ dberr_t recv_recovery_from_checkpoint_start() log_sys.latch.wr_lock(SRW_LOCK_CALL); log_sys.set_capacity(); - /* Start reading the log from the checkpoint lsn. The variable - contiguous_lsn contains an lsn up to which the log is known to - be contiguously written. */ + /* Start reading the log from the checkpoint lsn. */ ut_ad(recv_sys.pages.empty()); @@ -4634,12 +4642,12 @@ read_only_recovery: goto err_exit; } ut_ad(recv_sys.file_checkpoint); + ut_ad(log_sys.get_lsn() >= recv_sys.scanned_lsn); if (rewind) { recv_sys.lsn = log_sys.next_checkpoint_lsn; recv_sys.offset = 0; recv_sys.len = 0; } - ut_ad(!recv_max_page_lsn); rescan = recv_scan_log(false); if (srv_read_only_mode && recv_needed_recovery) { @@ -4652,7 +4660,7 @@ read_only_recovery: } } - log_sys.set_recovered_lsn(recv_sys.lsn); + log_sys.set_recovered_lsn(recv_sys.scanned_lsn); if (recv_needed_recovery) { bool missing_tablespace = false; @@ -4696,14 +4704,17 @@ read_only_recovery: tablespaces (not individual pages), while retaining the initial recv_sys.pages. */ mysql_mutex_lock(&recv_sys.mutex); + ut_ad(log_sys.get_lsn() >= recv_sys.lsn); recv_sys.clear(); recv_sys.lsn = log_sys.next_checkpoint_lsn; mysql_mutex_unlock(&recv_sys.mutex); } if (srv_operation <= SRV_OPERATION_EXPORT_RESTORED) { - deferred_spaces.deferred_dblwr(); + mysql_mutex_lock(&recv_sys.mutex); + deferred_spaces.deferred_dblwr(log_sys.get_lsn()); buf_dblwr.recover(); + mysql_mutex_unlock(&recv_sys.mutex); } ut_ad(srv_force_recovery <= SRV_FORCE_NO_UNDO_LOG_SCAN); @@ -4725,21 +4736,8 @@ read_only_recovery: ut_ad(recv_sys.pages.empty()); } - if (log_sys.is_latest() - && (recv_sys.lsn < log_sys.next_checkpoint_lsn - || recv_sys.lsn < recv_max_page_lsn)) { - - sql_print_error("InnoDB: We scanned the log up to " LSN_PF "." - " A checkpoint was at " LSN_PF - " and the maximum LSN on a database page was " - LSN_PF ". It is possible that the" - " database is now corrupt!", - recv_sys.lsn, - log_sys.next_checkpoint_lsn, - recv_max_page_lsn); - } - - if (recv_sys.lsn < log_sys.next_checkpoint_lsn) { + if (!log_sys.is_latest()) { + } else if (recv_sys.validate_checkpoint()) { err_exit: err = DB_ERROR; goto func_exit; @@ -4776,7 +4774,6 @@ err_exit: err = recv_rename_files(); } - recv_lsn_checks_on = true; mysql_mutex_unlock(&recv_sys.mutex); /* The database is now ready to start almost normal processing of user @@ -4790,14 +4787,17 @@ err_exit: goto func_exit; } -bool recv_dblwr_t::validate_page(const page_id_t page_id, - const byte *page, +bool recv_dblwr_t::validate_page(const page_id_t page_id, lsn_t max_lsn, const fil_space_t *space, - byte *tmp_buf) + const byte *page, byte *tmp_buf) + const noexcept { + mysql_mutex_assert_owner(&recv_sys.mutex); + uint32_t flags; + if (page_id.page_no() == 0) { - uint32_t flags= fsp_header_get_flags(page); + flags= fsp_header_get_flags(page); if (!fil_space_t::is_valid_flags(flags, page_id.space())) { uint32_t cflags= fsp_flags_convert_from_101(flags); @@ -4812,9 +4812,15 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, } /* Page 0 is never page_compressed or encrypted. */ - return !buf_page_is_corrupted(true, page, flags); + goto check_if_corrupted; } + flags= space->flags; + + if (space->full_crc32()) + check_if_corrupted: + return !buf_page_is_corrupted(max_lsn < LSN_MAX, page, flags); + ut_ad(tmp_buf); byte *tmp_frame= tmp_buf; byte *tmp_page= tmp_buf + srv_page_size; @@ -4822,9 +4828,6 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, const bool expect_encrypted= space->crypt_data && space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED; - if (space->full_crc32()) - return !buf_page_is_corrupted(true, page, space->flags); - if (expect_encrypted && mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)) { @@ -4851,126 +4854,45 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, return false; /* decompression failed */ if (decomp == srv_page_size) return false; /* the page was not compressed (invalid page type) */ - return !buf_page_is_corrupted(true, tmp_page, space->flags); + page= tmp_page; } - return !buf_page_is_corrupted(true, page, space->flags); + goto check_if_corrupted; } -byte *recv_dblwr_t::find_page(const page_id_t page_id, - const fil_space_t *space, byte *tmp_buf) +const byte *recv_dblwr_t::find_page(const page_id_t page_id, lsn_t max_lsn, + const fil_space_t *space, byte *tmp_buf) + const noexcept { - byte *result= NULL; - lsn_t max_lsn= 0; + mysql_mutex_assert_owner(&recv_sys.mutex); + ut_ad(recv_sys.scanned_lsn <= max_lsn); for (byte *page : pages) { if (page_get_page_no(page) != page_id.page_no() || page_get_space_id(page) != page_id.space()) continue; + const lsn_t lsn= mach_read_from_8(page + FIL_PAGE_LSN); if (page_id.page_no() == 0) { + if (!lsn) + continue; uint32_t flags= mach_read_from_4( FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); if (!fil_space_t::is_valid_flags(flags, page_id.space())) continue; } - const lsn_t lsn= mach_read_from_8(page + FIL_PAGE_LSN); - if (lsn <= max_lsn || - !validate_page(page_id, page, space, tmp_buf)) + if (lsn > max_lsn || lsn < log_sys.next_checkpoint_lsn || + !validate_page(page_id, max_lsn, space, page, tmp_buf)) { /* Mark processed for subsequent iterations in buf_dblwr_t::recover() */ - memset(page + FIL_PAGE_LSN, 0, 8); + memset_aligned<8>(page + FIL_PAGE_LSN, 0, 8); continue; } - ut_a(page_get_page_no(page) == page_id.page_no()); - max_lsn= lsn; - result= page; + return page; } - return result; -} - -bool recv_dblwr_t::restore_first_page(uint32_t space_id, const char *name, - pfs_os_file_t file) -{ - const page_id_t page_id(space_id, 0); - const byte* page= find_page(page_id); - if (!page) - { - /* If the first page of the given user tablespace is not there - in the doublewrite buffer, then the recovery is going to fail - now. Report error only when doublewrite buffer is not empty */ - if (pages.size()) - ib::error() << "Corrupted page " << page_id << " of datafile '" - << name << "' could not be found in the doublewrite buffer."; - return true; - } - - ulint physical_size= fil_space_t::physical_size( - mach_read_from_4(page + FSP_HEADER_OFFSET + FSP_SPACE_FLAGS)); - ib::info() << "Restoring page " << page_id << " of datafile '" - << name << "' from the doublewrite buffer. Writing " - << physical_size << " bytes into file '" << name << "'"; - - return os_file_write( - IORequestWrite, name, file, page, 0, physical_size) != - DB_SUCCESS; -} - -uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file) -{ - os_offset_t file_size= os_file_get_size(file); - if (file_size != (os_offset_t) -1) - { - for (const page_t *page : pages) - { - uint32_t space_id= page_get_space_id(page); - byte *read_page= nullptr; - if (page_get_page_no(page) > 0 || space_id == 0) - { -next_page: - aligned_free(read_page); - continue; - } - uint32_t flags= mach_read_from_4( - FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - page_id_t page_id(space_id, 0); - size_t page_size= fil_space_t::physical_size(flags); - if (file_size < 4 * page_size) - goto next_page; - read_page= - static_cast(aligned_malloc(3 * page_size, page_size)); - /* Read 3 pages from the file and match the space id - with the space id which is stored in - doublewrite buffer page. */ - if (os_file_read(IORequestRead, file, read_page, page_size, - 3 * page_size, nullptr) != DB_SUCCESS) - goto next_page; - for (ulint j= 0; j <= 2; j++) - { - byte *cur_page= read_page + j * page_size; - if (buf_is_zeroes(span(cur_page, page_size))) - { - space_id= 0; - goto early_exit; - } - if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 || - memcmp(cur_page + FIL_PAGE_SPACE_ID, - page + FIL_PAGE_SPACE_ID, 4) || - buf_page_is_corrupted(false, cur_page, flags)) - goto next_page; - } - if (!restore_first_page(space_id, name, file)) - { -early_exit: - aligned_free(read_page); - return space_id; - } - break; - } - } - return 0; + return nullptr; } diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index fa3ccfb6b80..9e86275dfbc 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -3960,9 +3960,10 @@ void fil_node_t::find_metadata(os_file_t file /** Read the first page of a data file. @return whether the page was found valid */ -bool fil_node_t::read_page0() +bool fil_node_t::read_page0(const byte *dpage, bool no_lsn) noexcept { mysql_mutex_assert_owner(&fil_system.mutex); + ut_ad(!dpage || no_lsn); const unsigned psize= space->physical_size(); #ifndef _WIN32 struct stat statbuf; @@ -3986,15 +3987,18 @@ bool fil_node_t::read_page0() if (!deferred) { - page_t *page= static_cast(aligned_malloc(psize, psize)); - if (os_file_read(IORequestRead, handle, page, 0, psize, nullptr) - != DB_SUCCESS) + page_t *apage= static_cast(aligned_malloc(psize, psize)); + if (os_file_read(IORequestRead, handle, apage, 0, psize, nullptr) != + DB_SUCCESS) { sql_print_error("InnoDB: Unable to read first page of file %s", name); - aligned_free(page); + err_exit: + aligned_free(apage); return false; } + const page_t *page= apage; + retry: const ulint space_id= memcmp_aligned<2> (FIL_PAGE_SPACE_ID + page, FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4) @@ -4002,8 +4006,16 @@ bool fil_node_t::read_page0() : mach_read_from_4(FIL_PAGE_SPACE_ID + page); uint32_t flags= fsp_header_get_flags(page); const uint32_t size= fsp_header_get_field(page, FSP_SIZE); + if (!space_id && !flags && !size && dpage) + { + retry_dpage: + page= dpage; + dpage= nullptr; + goto retry; + } const uint32_t free_limit= fsp_header_get_field(page, FSP_FREE_LIMIT); const uint32_t free_len= flst_get_len(FSP_HEADER_OFFSET + FSP_FREE + page); + if (!fil_space_t::is_valid_flags(flags, space->id)) { uint32_t cflags= fsp_flags_convert_from_101(flags); @@ -4020,7 +4032,6 @@ bool fil_node_t::read_page0() } } - aligned_free(page); goto invalid; } @@ -4029,28 +4040,40 @@ bool fil_node_t::read_page0() !fil_space_t::is_flags_equal((space->flags & ~FSP_FLAGS_MEM_MASK), (flags & ~FSP_FLAGS_MEM_MASK))) { -invalid: + invalid: + if (dpage) + goto retry_dpage; sql_print_error("InnoDB: Expected tablespace flags 0x%zx but found 0x%zx" " in the file %s", space->flags, flags, name); - return false; + goto err_exit; } flags_ok: ut_ad(!(flags & FSP_FLAGS_MEM_MASK)); + if (buf_page_is_corrupted(!no_lsn, page, flags) != NOT_CORRUPTED) + { + if (dpage) + goto retry_dpage; + sql_print_error("InnoDB: The first page of file %s is corrupted", name); + goto err_exit; + } + + if (UNIV_UNLIKELY(space_id != space->id)) + { + if (dpage) + goto retry_dpage; + sql_print_error("InnoDB: Expected tablespace id %zu but found %zu" + " in the file %s", ulint{space->id}, ulint{space_id}, + name); + goto err_exit; + } + /* Try to read crypt_data from page 0 if it is not yet read. */ if (!space->crypt_data) space->crypt_data= fil_space_read_crypt_data( fil_space_t::zip_size(flags), page); - aligned_free(page); - - if (UNIV_UNLIKELY(space_id != space->id)) - { - ib::error() << "Expected tablespace id " << space->id - << " but found " << space_id - << " in the file " << name; - return false; - } + aligned_free(apage); space->flags= (space->flags & FSP_FLAGS_MEM_MASK) | flags; ut_ad(space->free_limit == 0 || space->free_limit == free_limit); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 56b31c12063..b1173a30f8b 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -222,7 +222,12 @@ close_and_exit: if (!log_sys.attach(file, srv_log_file_size)) { goto close_and_exit; } - if (!fil_system.sys_space->open(create_new_db)) { + + mysql_mutex_lock(&recv_sys.mutex); + const bool all_opened = fil_system.sys_space->open(create_new_db); + mysql_mutex_unlock(&recv_sys.mutex); + + if (!all_opened) { goto err_exit; } @@ -644,13 +649,14 @@ static uint32_t srv_undo_tablespace_open(bool create, const char* name, ulint n_retries = 5; os_offset_t size= os_file_get_size(fh); ut_a(size != os_offset_t(-1)); + page_t *apage= nullptr; + const page_t *page= nullptr; if (!create) { - page_t *page= static_cast(aligned_malloc(srv_page_size, - srv_page_size)); + apage= static_cast(aligned_malloc(srv_page_size, srv_page_size)); undo_retry: - if (os_file_read(IORequestRead, fh, page, 0, srv_page_size, nullptr) != + if (os_file_read(IORequestRead, fh, apage, 0, srv_page_size, nullptr) != DB_SUCCESS) { err_exit: @@ -661,11 +667,12 @@ err_exit: n_retries--; goto undo_retry; } - ib::error() << "Unable to read first page of file " << name; - aligned_free(page); + sql_print_error("InnoDB: Unable to read first page of file %s", name); + aligned_free(apage); return ~0U; } + page= apage; DBUG_EXECUTE_IF("undo_space_read_fail", goto err_exit;); uint32_t id= mach_read_from_4(FIL_PAGE_SPACE_ID + page); @@ -673,18 +680,30 @@ err_exit: memcmp_aligned<2>(FIL_PAGE_SPACE_ID + page, FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4)) { - ib::error() << "Inconsistent tablespace ID in file " << name; + sql_print_error("InnoDB: Inconsistent tablespace ID in file %s", name); goto err_exit; } space_id= id; fsp_flags= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - if (buf_page_is_corrupted(false, page, fsp_flags) && - recv_sys.dblwr.restore_first_page(space_id, name, fh)) - goto err_exit; + if (buf_page_is_corrupted(false, page, fsp_flags)) + { + page= recv_sys.dblwr.find_page(page_id_t{space_id, 0}, LSN_MAX); + if (!page) + { + /* If the first page of the given user tablespace is not there + in the doublewrite buffer, then the recovery is going to fail + now. Report error only when doublewrite buffer is not empty */ + sql_print_error("InnoDB: Corrupted page " + "[page id: space=" UINT32PF ", page number=0]" + " of datafile '%s' could not be found" + " in the doublewrite buffer", space_id, name); + goto err_exit; + } - aligned_free(page); + fsp_flags= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + } } /* Load the tablespace into InnoDB's internal data structures. */ @@ -707,7 +726,7 @@ err_exit: space->set_sizes(SRV_UNDO_TABLESPACE_SIZE_IN_PAGES); space->size= file->size= uint32_t(size >> srv_page_size_shift); } - else if (!file->read_page0()) + else if (!file->read_page0(page, true)) { os_file_close(file->handle); file->handle= OS_FILE_CLOSED; @@ -716,6 +735,7 @@ err_exit: } mysql_mutex_unlock(&fil_system.mutex); + aligned_free(apage); return space_id; } @@ -874,11 +894,12 @@ dberr_t srv_undo_tablespaces_init(bool create_new_undo, mtr_t *mtr) srv_undo_tablespaces_active= srv_undo_tablespaces; uint32_t n_undo= (create_new_undo || srv_operation == SRV_OPERATION_BACKUP || - srv_operation == SRV_OPERATION_RESTORE_DELTA) + srv_operation == SRV_OPERATION_RESTORE_DELTA) ? srv_undo_tablespaces : TRX_SYS_N_RSEGS; - if (dberr_t err= srv_all_undo_tablespaces_open(create_new_undo, n_undo)) - return err; + mysql_mutex_lock(&recv_sys.mutex); + dberr_t err= srv_all_undo_tablespaces_open(create_new_undo, n_undo); + mysql_mutex_unlock(&recv_sys.mutex); /* Initialize srv_undo_space_id_start=0 when there are no dedicated undo tablespaces. */ @@ -886,16 +907,11 @@ dberr_t srv_undo_tablespaces_init(bool create_new_undo, mtr_t *mtr) srv_undo_space_id_start= 0; if (create_new_undo) - { - for (uint32_t i= 0; i < srv_undo_tablespaces; ++i) - { - dberr_t err= fsp_header_init(fil_space_get(srv_undo_space_id_start + i), - SRV_UNDO_TABLESPACE_SIZE_IN_PAGES, mtr); - if (err) return err; - } - } + for (uint32_t i= 0; err == DB_SUCCESS && i < srv_undo_tablespaces; ++i) + err= fsp_header_init(fil_space_get(srv_undo_space_id_start + i), + SRV_UNDO_TABLESPACE_SIZE_IN_PAGES, mtr); - return DB_SUCCESS; + return err; } /** Create the temporary file tablespace. @@ -1366,9 +1382,14 @@ dberr_t srv_start(bool create_new_db) /* Open data files in the system tablespace: we keep them open until database shutdown */ + mysql_mutex_lock(&recv_sys.mutex); ut_d(fil_system.sys_space->recv_size = srv_sys_space_size_debug); + err = fil_system.sys_space->open(create_new_db) + ? DB_SUCCESS : DB_ERROR; + mysql_mutex_unlock(&recv_sys.mutex); - if (fil_system.sys_space->open(create_new_db)) { + + if (err == DB_SUCCESS) { mtr_t mtr; mtr.start(); err= srv_undo_tablespaces_init(create_new_db, &mtr);