From bae0844f6575767ee9ab730835a12779f13da685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 7 Jul 2017 00:55:01 +0300 Subject: [PATCH] Introduce a new InnoDB redo log format for MariaDB 10.3.1 The redo log format will be changed by MDEV-12288, and it could be changed further during MariaDB 10.3 development. We will allow startup from a clean redo log from any earlier InnoDB version (up to MySQL 5.7 or MariaDB 10.3), but we will refuse to do crash recovery from older-format redo logs. recv_log_format_0_recover(): Remove a reference to MySQL documentation, which may be misleading when it comes to MariaDB. recv_log_recover_10_2(): Check if a MariaDB 10.2.2/MySQL 5.7.9 redo log is clean. recv_find_max_checkpoint(): Invoke recv_log_recover_10_2() if the redo log is in the MariaDB 10.2.2 or MySQL 5.7.9 format. --- .../r/innodb_encrypt_log_corruption.result | 8 +- .../suite/innodb/r/log_corruption.result | 8 +- mysql-test/suite/innodb/t/log_corruption.test | 32 ++++-- storage/innobase/include/log0log.h | 6 +- storage/innobase/log/log0recv.cc | 100 ++++++++++++++---- 5 files changed, 121 insertions(+), 33 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result index 4a31f1ba454..6eb92aedeb7 100644 --- a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result +++ b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result @@ -28,7 +28,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/5.7/en/upgrading-downgrading.html/ in mysqld.1.err +FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\./ in mysqld.1.err # valid header, but old-format checkpoint blocks SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -40,6 +40,12 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: Upgrade after a crash is not supported\. The redo log was created with malicious intentions, or perhaps, and it appears corrupted\./ in mysqld.1.err +# same, but with current-version header +SELECT * FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err FOUND 1 /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err # --innodb-force-recovery=6 (skip the entire redo log) diff --git a/mysql-test/suite/innodb/r/log_corruption.result b/mysql-test/suite/innodb/r/log_corruption.result index 3a20a11cd8f..35b7b00ead6 100644 --- a/mysql-test/suite/innodb/r/log_corruption.result +++ b/mysql-test/suite/innodb/r/log_corruption.result @@ -28,7 +28,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/5.7/en/upgrading-downgrading.html/ in mysqld.1.err +FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\./ in mysqld.1.err # valid header, but old-format checkpoint blocks SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -40,6 +40,12 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: Upgrade after a crash is not supported\. The redo log was created with malicious intentions, or perhaps, and it appears corrupted\./ in mysqld.1.err +# same, but with current-version header +SELECT * FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err FOUND 1 /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err # --innodb-force-recovery=6 (skip the entire redo log) diff --git a/mysql-test/suite/innodb/t/log_corruption.test b/mysql-test/suite/innodb/t/log_corruption.test index 8013cc45830..3bf68cab4cc 100644 --- a/mysql-test/suite/innodb/t/log_corruption.test +++ b/mysql-test/suite/innodb/t/log_corruption.test @@ -165,7 +165,7 @@ EOF --source include/start_mysqld.inc eval $check_no_innodb; --source include/shutdown_mysqld.inc -let SEARCH_PATTERN=InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/5.7/en/upgrading-downgrading.html; +let SEARCH_PATTERN=InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\.; --source include/search_pattern_in_file.inc --echo # valid header, but old-format checkpoint blocks @@ -204,6 +204,21 @@ EOF --source include/start_mysqld.inc eval $check_no_innodb; --source include/shutdown_mysqld.inc +let SEARCH_PATTERN=InnoDB: Upgrade after a crash is not supported\. The redo log was created with malicious intentions, or perhaps, and it appears corrupted\.; +--source include/search_pattern_in_file.inc + +--echo # same, but with current-version header +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +print OUT pack("Nx[5]nx[5]", 103, 0x1286), "MariaDB 10.3.1"; +print OUT pack("x[478]N", 0x85021a0f); +close OUT or die; +EOF +--source include/start_mysqld.inc +eval $check_no_innodb; +--source include/shutdown_mysqld.inc + let SEARCH_PATTERN=InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122; --source include/search_pattern_in_file.inc let SEARCH_PATTERN=InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.; @@ -221,9 +236,8 @@ perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; binmode OUT; # header block -print OUT pack("Nx[5]nx[5]", 1, 0x1286); -print OUT "malicious intentions, or perhaps not"; -print OUT pack("x[456]N", 0xd42d53a2); +print OUT pack("Nx[5]nx[5]", 103, 0x1286), "MariaDB 10.3.1"; +print OUT pack("x[478]N", 0x85021a0f); # checkpoint page 1 and all-zero checkpoint 2 print OUT pack("x[13]nCNNx[264]", 0x1286, 12, 0, 0x80c); print OUT pack("H*x[212]Nx[1024]", "590DBAACFE922582", 0xc72d49c4); @@ -281,9 +295,8 @@ perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; binmode OUT; # header block -print OUT pack("Nx[5]nx[5]", 1, 0x1286); -print OUT "ibbackup was here!!!1!"; -print OUT pack("x[470]N", 0x52b54540); +print OUT pack("Nx[5]nx[5]", 103, 0x1286), "MariaDB 10.3.1"; +print OUT pack("x[478]N", 0x85021a0f); # invalid (all-zero) checkpoint page 1 and an empty log page print OUT chr(0) x 1024; # valid checkpoint block 2 @@ -326,9 +339,8 @@ perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; binmode OUT; # header block -print OUT pack("Nx[5]nx[5]", 1, 0x1286); -print OUT "ibbackup was here!!!1!"; -print OUT pack("x[470]N", 0x52b54540); +print OUT pack("Nx[5]nx[5]", 103, 0x1286), "MariaDB 10.3.1"; +print OUT pack("x[478]N", 0x85021a0f); # invalid (all-zero) checkpoint page 1 and an empty log page print OUT chr(0) x 1024; # valid checkpoint block 2 diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 543302c52f0..6a13d1d9640 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -510,9 +510,13 @@ or the MySQL version that created the redo log file. */ IB_TO_STR(MYSQL_VERSION_MINOR) "." \ IB_TO_STR(MYSQL_VERSION_PATCH) +/** The original (not version-tagged) InnoDB redo log format */ +#define LOG_HEADER_FORMAT_3_23 0 +/** The MySQL 5.7.9/MariaDB 10.2.2 log format */ +#define LOG_HEADER_FORMAT_10_2 1 /** The redo log format identifier corresponding to the current format version. Stored in LOG_HEADER_FORMAT. */ -#define LOG_HEADER_FORMAT_CURRENT 1 +#define LOG_HEADER_FORMAT_CURRENT 103 /** Encrypted MariaDB redo log */ #define LOG_HEADER_FORMAT_ENCRYPTED (1U<<31) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index ff7523a2954..84612fd2923 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -880,9 +880,6 @@ recv_log_format_0_recover(lsn_t lsn) static const char* NO_UPGRADE_RECOVERY_MSG = "Upgrade after a crash is not supported." " This redo log was created before MariaDB 10.2.2"; - static const char* NO_UPGRADE_RTFM_MSG = - ". Please follow the instructions at " - REFMAN "upgrading.html"; fil_io(IORequestLogRead, true, page_id_t(SRV_LOG_SPACE_FIRST_ID, page_no), @@ -895,15 +892,65 @@ recv_log_format_0_recover(lsn_t lsn) != log_block_get_checksum(buf) && !log_crypt_101_read_block(buf)) { ib::error() << NO_UPGRADE_RECOVERY_MSG - << ", and it appears corrupted" - << NO_UPGRADE_RTFM_MSG; + << ", and it appears corrupted."; return(DB_CORRUPTION); } if (log_block_get_data_len(buf) != (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) { - ib::error() << NO_UPGRADE_RECOVERY_MSG - << NO_UPGRADE_RTFM_MSG; + ib::error() << NO_UPGRADE_RECOVERY_MSG << "."; + return(DB_ERROR); + } + + /* Mark the redo log for upgrading. */ + srv_log_file_size = 0; + recv_sys->parse_start_lsn = recv_sys->recovered_lsn + = recv_sys->scanned_lsn + = recv_sys->mlog_checkpoint_lsn = lsn; + log_sys->last_checkpoint_lsn = log_sys->next_checkpoint_lsn + = log_sys->lsn = log_sys->write_lsn + = log_sys->current_flush_lsn = log_sys->flushed_to_disk_lsn + = lsn; + log_sys->next_checkpoint_no = 0; + return(DB_SUCCESS); +} + +/** Determine if a redo log from MySQL 5.7.9/MariaDB 10.2.2 is clean. +@return error code +@retval DB_SUCCESS if the redo log is clean +@retval DB_CORRUPTION if the redo log is corrupted +@retval DB_ERROR if the redo log is not empty */ +static +dberr_t +recv_log_recover_10_2() +{ + log_group_t* group = &log_sys->log; + const lsn_t lsn = group->lsn; + const lsn_t source_offset = log_group_calc_lsn_offset(lsn, group); + const ulint page_no + = (ulint) (source_offset / univ_page_size.physical()); + byte* buf = log_sys->buf; + + fil_io(IORequestLogRead, true, + page_id_t(SRV_LOG_SPACE_FIRST_ID, page_no), + univ_page_size, + (ulint) ((source_offset & ~(OS_FILE_LOG_BLOCK_SIZE - 1)) + % univ_page_size.physical()), + OS_FILE_LOG_BLOCK_SIZE, buf, NULL); + + if (log_block_calc_checksum(buf) != log_block_get_checksum(buf)) { + return(DB_CORRUPTION); + } + + if (group->is_encrypted()) { + log_crypt(buf, OS_FILE_LOG_BLOCK_SIZE, true); + } + + /* On a clean shutdown, the redo log will be logically empty + after the checkpoint lsn. */ + + if (log_block_get_data_len(buf) + != (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) { return(DB_ERROR); } @@ -945,31 +992,29 @@ recv_find_max_checkpoint(ulint* max_field) /* Check the header page checksum. There was no checksum in the first redo log format (version 0). */ group->format = mach_read_from_4(buf + LOG_HEADER_FORMAT); - if (group->format != 0 + if (group->format != LOG_HEADER_FORMAT_3_23 && !recv_check_log_header_checksum(buf)) { ib::error() << "Invalid redo log header checksum."; return(DB_CORRUPTION); } + char creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR + 1]; + + memcpy(creator, buf + LOG_HEADER_CREATOR, sizeof creator); + /* Ensure that the string is NUL-terminated. */ + creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR] = 0; + switch (group->format) { - case 0: + case LOG_HEADER_FORMAT_3_23: return(recv_find_max_checkpoint_0(&group, max_field)); + case LOG_HEADER_FORMAT_10_2: + case LOG_HEADER_FORMAT_10_2 | LOG_HEADER_FORMAT_ENCRYPTED: case LOG_HEADER_FORMAT_CURRENT: case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: break; default: - /* Ensure that the string is NUL-terminated. */ - buf[LOG_HEADER_CREATOR_END] = 0; ib::error() << "Unsupported redo log format." - " The redo log was created" - " with " << buf + LOG_HEADER_CREATOR << - ". Please follow the instructions at " - REFMAN "upgrading-downgrading.html"; - /* Do not issue a message about a possibility - to cleanly shut down the newer server version - and to remove the redo logs, because the - format of the system data structures may - radically change after MySQL 5.7. */ + " The redo log was created with " << creator << "."; return(DB_ERROR); } @@ -1030,6 +1075,21 @@ recv_find_max_checkpoint(ulint* max_field) return(DB_ERROR); } + switch (group->format) { + case LOG_HEADER_FORMAT_10_2: + case LOG_HEADER_FORMAT_10_2 | LOG_HEADER_FORMAT_ENCRYPTED: + dberr_t err = recv_log_recover_10_2(); + if (err != DB_SUCCESS) { + ib::error() + << "Upgrade after a crash is not supported." + " The redo log was created with " << creator + << (err == DB_ERROR + ? "." : ", and it appears corrupted."); + break; + } + return(err); + } + return(DB_SUCCESS); }