From 980d1bf1a921a270423ab36bd5d1ce2a1cd7590b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 7 Sep 2018 17:24:31 +0300 Subject: [PATCH] MDEV-14717: Prevent crash-downgrade to earlier MariaDB 10.2 A crash-downgrade of a RENAME (or TRUNCATE or table-rebuilding ALTER TABLE or OPTIMIZE TABLE) operation to an earlier 10.2 version would trigger a debug assertion failure during rollback, in trx_roll_pop_top_rec_of_trx(). In a non-debug build, the TRX_UNDO_RENAME_TABLE record would be misinterpreted as an update_undo log record, and typically the file name would be interpreted as DB_TRX_ID,DB_ROLL_PTR,PRIMARY KEY. If a matching record would be found, row_undo_mod() would hit ut_error in switch (node->rec_type). Typically, ut_a(table2 == NULL) would fail when opening the table from SQL. Because of this, we prevent a crash-downgrade to earlier MariaDB 10.2 versions by changing the InnoDB redo log format identifier to the 10.3 identifier, and by introducing a subformat identifier so that 10.2 can continue to refuse crash-downgrade from 10.3 or later. After a clean shutdown, a downgrade to MariaDB 10.2.13 or later would still be possible thanks to MDEV-14909. A downgrade to older 10.2 versions is only possible after removing the log files (not recommended). LOG_HEADER_FORMAT_CURRENT: Change to 103 (originally the 10.3 format). log_group_t: Add subformat. For 10.2, we will use subformat 1, and will refuse crash recovery from any other subformat of the 10.3 format, that is, a genuine 10.3 redo log. recv_find_max_checkpoint(): Allow startup after clean shutdown from a future LOG_HEADER_FORMAT_10_4 (unencrypted only). We cannot handle the encrypted 10.4 redo log block format, which was introduced in MDEV-12041. Allow crash recovery from the original 10.2 format as well as the new format. In Mariabackup --backup, do not allow any startup from 10.3 or 10.4 redo logs. recv_recovery_from_checkpoint_start(): Skip redo log apply for clean 10.3 redo log, but not for the new 10.2 redo log (10.3 format, subformat 1). srv_prepare_to_delete_redo_log_files(): On format or subformat mismatch, set srv_log_file_size = 0, so that we will display the correct message. innobase_start_or_create_for_mysql(): Check for format or subformat mismatch. xtrabackup_backup_func(): Remove debug assertions that were made redundant by the code changes in recv_find_max_checkpoint(). --- extra/mariabackup/xtrabackup.cc | 9 ++--- storage/innobase/include/log0log.h | 20 ++++++++--- storage/innobase/log/log0log.cc | 3 +- storage/innobase/log/log0recv.cc | 56 +++++++++++++++++++++++------- storage/innobase/srv/srv0start.cc | 18 ++++++++-- 5 files changed, 78 insertions(+), 28 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 1faa10f9c7a..5b3fe9fe435 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4139,9 +4139,6 @@ old_format: goto log_fail; } - ut_ad(!((log_sys->log.format ^ LOG_HEADER_FORMAT_CURRENT) - & ~LOG_HEADER_FORMAT_ENCRYPTED)); - const byte* buf = log_sys->checkpoint_buf; reread_log_header: @@ -4158,9 +4155,6 @@ reread_log_header: goto old_format; } - ut_ad(!((log_sys->log.format ^ LOG_HEADER_FORMAT_CURRENT) - & ~LOG_HEADER_FORMAT_ENCRYPTED)); - log_group_header_read(&log_sys->log, max_cp_field); if (checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) { @@ -4188,7 +4182,8 @@ reread_log_header: byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr[OS_FILE_LOG_BLOCK_SIZE]; memset(log_hdr, 0, sizeof log_hdr); mach_write_to_4(LOG_HEADER_FORMAT + log_hdr, log_sys->log.format); - mach_write_to_4(LOG_HEADER_SUBFORMAT + log_hdr, 1); + mach_write_to_4(LOG_HEADER_SUBFORMAT + log_hdr, + log_sys->log.subformat); mach_write_to_8(LOG_HEADER_START_LSN + log_hdr, checkpoint_lsn_start); strcpy(reinterpret_cast(LOG_HEADER_CREATOR + log_hdr), "Backup " MYSQL_SERVER_VERSION); diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index cd3f415a8d7..4759e5a85f4 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -513,10 +513,17 @@ or the MySQL version that created the redo log file. */ IB_TO_STR(MYSQL_VERSION_PATCH) /** The redo log format identifier corresponding to the current format version. -Stored in LOG_HEADER_FORMAT. */ -#define LOG_HEADER_FORMAT_CURRENT 1 -/** The MariaDB 10.3.2 log format */ -#define LOG_HEADER_FORMAT_10_3 103 +Stored in LOG_HEADER_FORMAT. +To prevent crash-downgrade to earlier 10.2 due to the inability to +roll back a retroactively introduced TRX_UNDO_RENAME_TABLE undo log record, +MariaDB 10.2.18 and later will use the 10.3 format, but LOG_HEADER_SUBFORMAT +1 instead of 0. MariaDB 10.3 will use subformat 0 (5.7-style TRUNCATE) or 2 +(MDEV-13564 backup-friendly TRUNCATE). */ +#define LOG_HEADER_FORMAT_CURRENT 103 +/** The old MariaDB 10.2.2..10.2.17 log format */ +#define LOG_HEADER_FORMAT_10_2 1 +/** Future MariaDB 10.4 log format */ +#define LOG_HEADER_FORMAT_10_4 104 /** Encrypted MariaDB redo log */ #define LOG_HEADER_FORMAT_ENCRYPTED (1U<<31) @@ -553,7 +560,10 @@ struct log_group_t{ /** number of files in the group */ ulint n_files; /** format of the redo log: e.g., LOG_HEADER_FORMAT_CURRENT */ - ulint format; + uint32_t format; + /** redo log subformat: 0 with separately logged TRUNCATE, + 1 with fully redo-logged TRUNCATE */ + uint32_t subformat; /** individual log file size in bytes, including the header */ lsn_t file_size; /** corruption status */ diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 209decae021..95c637bccfd 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -793,6 +793,7 @@ log_init(ulint n_files) group->format = srv_encrypt_log ? LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED : LOG_HEADER_FORMAT_CURRENT; + group->subformat = 1; group->file_size = srv_log_file_size; group->state = LOG_GROUP_OK; group->lsn = LOG_START_LSN; @@ -880,7 +881,7 @@ log_group_file_header_flush( memset(buf, 0, OS_FILE_LOG_BLOCK_SIZE); mach_write_to_4(buf + LOG_HEADER_FORMAT, group->format); - mach_write_to_4(buf + LOG_HEADER_SUBFORMAT, 1); + mach_write_to_4(buf + LOG_HEADER_SUBFORMAT, group->subformat); mach_write_to_8(buf + LOG_HEADER_START_LSN, start_lsn); strcpy(reinterpret_cast(buf) + LOG_HEADER_CREATOR, LOG_HEADER_CREATOR_CURRENT); diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index f28423ddde0..a69b425fdf4 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1147,6 +1147,9 @@ recv_find_max_checkpoint(ulint* max_field) /* Check the header page checksum. There was no checksum in the first redo log format (version 0). */ group->format = mach_read_from_4(buf + LOG_HEADER_FORMAT); + group->subformat = group->format + ? mach_read_from_4(buf + LOG_HEADER_SUBFORMAT) + : 0; if (group->format != 0 && !recv_check_log_header_checksum(buf)) { ib::error() << "Invalid redo log header checksum."; @@ -1164,8 +1167,11 @@ recv_find_max_checkpoint(ulint* max_field) return(recv_find_max_checkpoint_0(&group, max_field)); case LOG_HEADER_FORMAT_CURRENT: case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: - case LOG_HEADER_FORMAT_10_3: - case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED: + case LOG_HEADER_FORMAT_10_2: + case LOG_HEADER_FORMAT_10_2 | LOG_HEADER_FORMAT_ENCRYPTED: + case LOG_HEADER_FORMAT_10_4: + /* We can only parse the unencrypted LOG_HEADER_FORMAT_10_4. + The encrypted format uses a larger redo log block trailer. */ break; default: ib::error() << "Unsupported redo log format." @@ -1240,8 +1246,20 @@ recv_find_max_checkpoint(ulint* max_field) } switch (group->format) { - case LOG_HEADER_FORMAT_10_3: - case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED: + case LOG_HEADER_FORMAT_CURRENT: + case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: + if (group->subformat == 1) { + /* 10.2 with new crash-safe TRUNCATE */ + break; + } + /* fall through */ + case LOG_HEADER_FORMAT_10_4: + if (srv_operation == SRV_OPERATION_BACKUP) { + ib::error() + << "Incompatible redo log format." + " The redo log was created with " << creator; + return DB_ERROR; + } dberr_t err = recv_log_recover_10_3(); if (err != DB_SUCCESS) { ib::error() @@ -3370,7 +3388,6 @@ of first system tablespace page dberr_t recv_recovery_from_checkpoint_start(lsn_t flush_lsn) { - log_group_t* group; ulint max_cp_field; lsn_t checkpoint_lsn; bool rescan; @@ -3402,15 +3419,30 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) err = recv_find_max_checkpoint(&max_cp_field); - if (err != DB_SUCCESS - || (log_sys->log.format != 0 - && (log_sys->log.format & ~LOG_HEADER_FORMAT_ENCRYPTED) - != LOG_HEADER_FORMAT_CURRENT)) { - + if (err != DB_SUCCESS) { +skip_apply: log_mutex_exit(); return(err); } + switch (log_sys->log.format) { + case 0: + break; + case LOG_HEADER_FORMAT_10_2: + case LOG_HEADER_FORMAT_10_2 | LOG_HEADER_FORMAT_ENCRYPTED: + break; + case LOG_HEADER_FORMAT_CURRENT: + case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: + if (log_sys->log.subformat == 1) { + /* 10.2 with new crash-safe TRUNCATE */ + break; + } + /* fall through */ + default: + /* This must be a clean log from a newer version. */ + goto skip_apply; + } + log_group_header_read(&log_sys->log, max_cp_field); buf = log_sys->checkpoint_buf; @@ -3426,13 +3458,12 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) ut_ad(RECV_SCAN_SIZE <= log_sys->buf_size); - group = &log_sys->log; const lsn_t end_lsn = mach_read_from_8( buf + LOG_CHECKPOINT_END_LSN); ut_ad(recv_sys->n_addrs == 0); contiguous_lsn = checkpoint_lsn; - switch (group->format) { + switch (log_sys->log.format) { case 0: log_mutex_exit(); return recv_log_format_0_recover(checkpoint_lsn, @@ -3451,6 +3482,7 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) } /* Look for MLOG_CHECKPOINT. */ + log_group_t* group = &log_sys->log; recv_group_scan_log_recs(group, checkpoint_lsn, &contiguous_lsn, false); /* The first scan should not have stored or applied any records. */ diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index cda652a9855..ca0e1472aed 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1393,6 +1393,12 @@ srv_prepare_to_delete_redo_log_files( ulint pending_io = 0; ulint count = 0; + if ((log_sys->log.format & ~LOG_HEADER_FORMAT_ENCRYPTED) + != LOG_HEADER_FORMAT_CURRENT + || log_sys->log.subformat != 1) { + srv_log_file_size = 0; + } + do { /* Clean the buffer pool. */ buf_flush_sync_all_buf_pools(); @@ -1411,7 +1417,7 @@ srv_prepare_to_delete_redo_log_files( if (srv_log_file_size == 0) { info << ((log_sys->log.format & ~LOG_HEADER_FORMAT_ENCRYPTED) - != LOG_HEADER_FORMAT_10_3 + < LOG_HEADER_FORMAT_CURRENT ? "Upgrading redo log: " : "Downgrading redo log: "); } else if (n_files != srv_n_log_files @@ -2385,8 +2391,14 @@ files_checked: /* Leave the redo log alone. */ } else if (srv_log_file_size_requested == srv_log_file_size && srv_n_log_files_found == srv_n_log_files - && log_sys->is_encrypted() == srv_encrypt_log) { - /* No need to upgrade or resize the redo log. */ + && log_sys->log.format + == (srv_encrypt_log + ? LOG_HEADER_FORMAT_CURRENT + | LOG_HEADER_FORMAT_ENCRYPTED + : LOG_HEADER_FORMAT_CURRENT) + && log_sys->log.subformat == 1) { + /* No need to add or remove encryption, + upgrade, downgrade, or resize. */ } else { /* Prepare to delete the old redo log files */ flushed_lsn = srv_prepare_to_delete_redo_log_files(i);