mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-04-19 05:35:33 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions1

MDEV-11782: Redefine the innodb_encrypt_log format

Browse source 
Write only one encryption key to the checkpoint page.
Use 4 bytes of nonce. Encrypt more of each redo log block,
only skipping the 4-byte field LOG_BLOCK_HDR_NO which the
initialization vector is derived from.

Issue notes, not warning messages for rewriting the redo log files.

recv_recovery_from_checkpoint_finish(): Do not generate any redo log,
because we must avoid that before rewriting the redo log files, or
otherwise a crash during a redo log rewrite (removing or adding
encryption) may end up making the database unrecoverable.
Instead, do these tasks in innobase_start_or_create_for_mysql().

Issue a firm "Missing MLOG_CHECKPOINT" error message. Remove some
unreachable code and duplicated error messages for log corruption.

LOG_HEADER_FORMAT_ENCRYPTED: A flag for identifying an encrypted redo
log format.

log_group_t::is_encrypted(), log_t::is_encrypted(): Determine
if the redo log is in encrypted format.

recv_find_max_checkpoint(): Interpret LOG_HEADER_FORMAT_ENCRYPTED.

srv_prepare_to_delete_redo_log_files(): Display NOTE messages about
adding or removing encryption. Do not issue warnings for redo log
resizing any more.

innobase_start_or_create_for_mysql(): Rebuild the redo logs also when
the encryption changes.

innodb_log_checksums_func_update(): Always use the CRC-32C checksum
if innodb_encrypt_log. If needed, issue a warning
that innodb_encrypt_log implies innodb_log_checksums.

log_group_write_buf(): Compute the checksum on the encrypted
block contents, so that transmission errors or incomplete blocks can be
detected without decrypting.

Rewrite most of the redo log encryption code. Only remember one
encryption key at a time (but remember up to 5 when upgrading from the
MariaDB 10.1 format.)
This commit is contained in:
Marko Mäkelä 2017-02-10 12:11:42 +02:00
parent 743ac7c2d0
commit 2af28a363c
34 changed files with 749 additions and 1289 deletions

6
mysql-test/mysql-test-run.pl
View file

@ -4345,10 +4345,7 @@ sub extract_warning_lines ($$) {
qr/error .*connecting to master/,
qr/InnoDB: Error: in ALTER TABLE `test`.`t[12]`/,
qr/InnoDB: Error: table `test`.`t[12]` .*does not exist in the InnoDB internal/,
qr/InnoDB: Warning: Setting innodb_use_sys_malloc/,
qr/InnoDB: Warning: a long semaphore wait:/,
qr/InnoDB: Disabling redo log encryption/,
qr/InnoDB: Redo log crypto: Can't initialize to key version -1u/,
qr/InnoDB: Dumping buffer pool.*/,
qr/InnoDB: Buffer pool.*/,
qr/InnoDB: Warning: Writer thread is waiting this semaphore/,
@ -4422,9 +4419,6 @@ sub extract_warning_lines ($$) {
qr|InnoDB: TABLE to scan your table for corruption|,
qr/InnoDB: See also */,
qr/InnoDB: Cannot open .*ib_buffer_pool.* for reading: No such file or directory*/,
qr/InnoDB: Upgrading redo log:*/,
qr|InnoDB: Starting to delete and rewrite log files.|,
qr/InnoDB: New log files created, LSN=*/,
qr|InnoDB: Creating foreign key constraint system tables.|,
qr/InnoDB: Table .*mysql.*innodb_table_stats.* not found./,
qr/InnoDB: User stopword table .* does not exist./

19
mysql-test/suite/encryption/r/innodb-log-encrypt-crash.result
View file

@ -1,19 +0,0 @@
call mtr.add_suppression("InnoDB: New log files created, LSN=.*");
call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables.");
call mtr.add_suppression("InnoDB: Error: Table .*");
CREATE TABLE t1 (
pk bigint auto_increment,
col_int int,
col_int_key int,
col_char char(12),
col_char_key char(12),
primary key (pk),
key (`col_int_key` ),
key (`col_char_key` )
) ENGINE=InnoDB;
CREATE TABLE t2 LIKE t1;
INSERT INTO t1 VALUES (NULL,1,1,'foo','foo'),(NULL,2,2,'bar','bar'),(NULL,3,3,'baz','baz'),(NULL,4,4,'qux','qux');
INSERT INTO t2
SELECT NULL, a1.col_int, a1.col_int_key, a1.col_char, a1.col_char_key
FROM t1 a1, t1 a2, t1 a3, t1 a4, t1 a5, t1 a6, t1 a7, t1 a8, t1 a9, t1 a10;
DROP TABLE t1, t2;

53
mysql-test/suite/encryption/r/innodb-log-encrypt.result
View file

@ -1,53 +0,0 @@
create table t1(c1 bigint not null, b char(200), c varchar(200)) engine=innodb encrypted=yes encryption_key_id=1;
show warnings;
Level Code Message
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values(current_num, substring(MD5(RAND()), -64), REPEAT('privatejanprivate',10));
set current_num = current_num + 1;
end while;
end//
commit;
begin;
call innodb_insert_proc(2000);
commit;
update t1 set c1 = c1 +1;
select count(*) from t1;
count(*)
2000
# Kill the server
# ibdata1 yes on expecting NOT FOUND
NOT FOUND /privatejanprivate/ in ibdata1
# t1 yes on expecting NOT FOUND
NOT FOUND /privatejanprivate/ in t1.ibd
# log0 yes on expecting NOT FOUND
NOT FOUND /privatejanprivate/ in ib_logfile0
# log1 yes on expecting NOT FOUND
NOT FOUND /privatejanprivate/ in ib_logfile1
# Restart mysqld --innodb_encrypt_log=0
insert into t1 values(5000, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
insert into t1 values(5001, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
insert into t1 values(5002, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
insert into t1 values(5003, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
insert into t1 values(5004, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
# ibdata1 yes on expecting NOT FOUND
NOT FOUND /privatejanprivate/ in ibdata1
# t1 yes on expecting NOT FOUND
NOT FOUND /privatejanprivate/ in t1.ibd
# log0 yes on expecting NOT FOUND
NOT FOUND /privatejanprivate/ in ib_logfile0
# log1 yes on expecting NOT FOUND
NOT FOUND /privatejanprivate/ in ib_logfile1
# ibdata1 yes on expecting NOT FOUND
NOT FOUND /publicmessage/ in ibdata1
# t1 yes on expecting NOT FOUND
NOT FOUND /publicmessage/ in t1.ibd
# log0 no on expecting FOUND/NOTFOUND depending where insert goes
FOUND /publicmessage/ in ib_logfile0
# log1 no on expecting FOUND/NOTFOUND depending where insert goes
NOT FOUND /publicmessage/ in ib_logfile1
drop procedure innodb_insert_proc;
drop table t1;

65
mysql-test/suite/encryption/r/innodb_encrypt_log.result Normal file
View file

@ -0,0 +1,65 @@
#
# MDEV-9011: Redo log encryption does not work
#
#
# MDEV-9422 Encrypted redo log checksum errors
# on restart after killing busy server instance
#
SET GLOBAL innodb_log_checksums=0;
Warnings:
Warning 138 innodb_encrypt_log implies innodb_log_checksums
SELECT @@global.innodb_log_checksums;
@@global.innodb_log_checksums
1
CREATE TABLE t0 (
pk bigint auto_increment,
col_int int,
col_int_key int,
col_char char(12),
col_char_key char(12),
primary key (pk),
key (col_int_key),
key (col_char_key)
) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
CREATE TEMPORARY TABLE t LIKE t0;
INSERT INTO t VALUES
(NULL,1,1,'private','secret'),(NULL,2,2,'sacred','success'),
(NULL,3,3,'story','secure'),(NULL,4,4,'security','sacrament');
SET GLOBAL innodb_flush_log_at_trx_commit=1;
INSERT INTO t0
SELECT NULL, t1.col_int, t1.col_int_key, t1.col_char, t1.col_char_key
FROM t t1, t t2, t t3, t t4, t t5;
# Kill the server
# ibdata1 expecting NOT FOUND
NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in ibdata1
# t0.ibd expecting NOT FOUND
NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in t0.ibd
# ib_logfile0 expecting NOT FOUND
NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in ib_logfile0
# ib_logfile1 expecting NOT FOUND
NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in ib_logfile1
# Restart without redo log encryption
SELECT COUNT(*) FROM t0;
COUNT(*)
1024
CHECK TABLE t0;
Table Op Msg_type Msg_text
test.t0 check status OK
SET GLOBAL innodb_flush_log_at_trx_commit=1;
INSERT INTO t0 VALUES(NULL, 5, 5, 'public', 'gossip');
# Kill the server
# ib_logfile0 expecting NOT FOUND
NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in ib_logfile0
# ib_logfile0 expecting FOUND
FOUND /public|gossip/ in ib_logfile0
# ibdata1 expecting NOT FOUND
NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)|public|gossip/ in ibdata1
# t0.ibd expecting NOT FOUND
NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)|public|gossip/ in t0.ibd
SELECT COUNT(*) FROM t0;
COUNT(*)
1025
CHECK TABLE t0;
Table Op Msg_type Msg_text
test.t0 check status OK
DROP TABLE t0;

4
mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result
View file

@ -41,7 +41,7 @@ WHERE engine = 'innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS
FOUND /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
FOUND /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
# --innodb-force-recovery=6 (skip the entire redo log)
SELECT * FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
@ -54,7 +54,6 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS
FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
# --innodb-force-recovery=6 (skip the entire redo log)
SELECT * FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
@ -90,6 +89,7 @@ SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
1
1
FOUND /InnoDB: Encrypting redo log/ in mysqld.1.err
ib_buffer_pool
ib_logfile0
ib_logfile1

6
mysql-test/suite/encryption/t/innodb-log-encrypt-crash.opt
View file

@ -1,6 +0,0 @@
--innodb-encrypt-log=ON
--plugin-load-add=$FILE_KEY_MANAGEMENT_SO
--loose-file-key-management
--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt
--file-key-management-encryption-algorithm=aes_cbc
--innodb-buffer-pool-size=128M

41
mysql-test/suite/encryption/t/innodb-log-encrypt-crash.test
View file

@ -1,41 +0,0 @@
-- source include/have_innodb.inc
-- source include/not_embedded.inc
# test takes very long time on debug build
-- source include/not_debug.inc
-- source include/big_test.inc
-- source filekeys_plugin.inc
call mtr.add_suppression("InnoDB: New log files created, LSN=.*");
call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables.");
call mtr.add_suppression("InnoDB: Error: Table .*");
#
# MDEV-9422: Checksum errors on restart when killing busy instance that uses encrypted XtraDB tables
#
CREATE TABLE t1 (
pk bigint auto_increment,
col_int int,
col_int_key int,
col_char char(12),
col_char_key char(12),
primary key (pk),
key (`col_int_key` ),
key (`col_char_key` )
) ENGINE=InnoDB;
CREATE TABLE t2 LIKE t1;
INSERT INTO t1 VALUES (NULL,1,1,'foo','foo'),(NULL,2,2,'bar','bar'),(NULL,3,3,'baz','baz'),(NULL,4,4,'qux','qux');
INSERT INTO t2
SELECT NULL, a1.col_int, a1.col_int_key, a1.col_char, a1.col_char_key
FROM t1 a1, t1 a2, t1 a3, t1 a4, t1 a5, t1 a6, t1 a7, t1 a8, t1 a9, t1 a10;
--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--shutdown_server 0
--source include/wait_until_disconnected.inc
--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--enable_reconnect
--source include/wait_until_connected_again.inc
DROP TABLE t1, t2;

94
mysql-test/suite/encryption/t/innodb-log-encrypt.test
View file

@ -1,94 +0,0 @@
-- source include/have_innodb.inc
-- source include/not_embedded.inc
-- source filekeys_plugin.inc
#
# MDEV-9011: Redo log encryption does not work
#
create table t1(c1 bigint not null, b char(200), c varchar(200)) engine=innodb encrypted=yes encryption_key_id=1;
show warnings;
delimiter //;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values(current_num, substring(MD5(RAND()), -64), REPEAT('privatejanprivate',10));
set current_num = current_num + 1;
end while;
end//
delimiter ;//
commit;
begin;
call innodb_insert_proc(2000);
commit;
update t1 set c1 = c1 +1;
select count(*) from t1;
--let $MYSQLD_DATADIR=`select @@datadir`
--let ib1_IBD = $MYSQLD_DATADIR/ibdata1
--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd
--let log0 = $MYSQLD_DATADIR/ib_logfile0
--let log1 = $MYSQLD_DATADIR/ib_logfile1
--let SEARCH_RANGE = 10000000
--let SEARCH_PATTERN=privatejanprivate
-- source include/kill_mysqld.inc
--echo # ibdata1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$ib1_IBD
-- source include/search_pattern_in_file.inc
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--echo # log0 yes on expecting NOT FOUND
-- let SEARCH_FILE=$log0
-- source include/search_pattern_in_file.inc
--echo # log1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$log1
-- source include/search_pattern_in_file.inc
--echo # Restart mysqld --innodb_encrypt_log=0
-- let $restart_parameters=--innodb_encrypt_log=0
-- source include/start_mysqld.inc
insert into t1 values(5000, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
insert into t1 values(5001, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
insert into t1 values(5002, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
insert into t1 values(5003, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
insert into t1 values(5004, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
--let SEARCH_PATTERN=privatejanprivate
--echo # ibdata1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$ib1_IBD
-- source include/search_pattern_in_file.inc
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--echo # log0 yes on expecting NOT FOUND
-- let SEARCH_FILE=$log0
-- source include/search_pattern_in_file.inc
--echo # log1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$log1
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=publicmessage
--echo # ibdata1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$ib1_IBD
-- source include/search_pattern_in_file.inc
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--echo # log0 no on expecting FOUND/NOTFOUND depending where insert goes
-- let SEARCH_FILE=$log0
-- source include/search_pattern_in_file.inc
--echo # log1 no on expecting FOUND/NOTFOUND depending where insert goes
-- let SEARCH_FILE=$log1
-- source include/search_pattern_in_file.inc
drop procedure innodb_insert_proc;
drop table t1;

0
mysql-test/suite/encryption/t/innodb-log-encrypt.opt → mysql-test/suite/encryption/t/innodb_encrypt_log.opt
View file

94
mysql-test/suite/encryption/t/innodb_encrypt_log.test Normal file
View file

@ -0,0 +1,94 @@
-- source include/have_innodb.inc
-- source include/not_embedded.inc
-- source filekeys_plugin.inc
--echo #
--echo # MDEV-9011: Redo log encryption does not work
--echo #
--echo #
--echo # MDEV-9422 Encrypted redo log checksum errors
--echo # on restart after killing busy server instance
--echo #
--let $MYSQLD_DATADIR=`select @@datadir`
SET GLOBAL innodb_log_checksums=0;
SELECT @@global.innodb_log_checksums;
CREATE TABLE t0 (
pk bigint auto_increment,
col_int int,
col_int_key int,
col_char char(12),
col_char_key char(12),
primary key (pk),
key (col_int_key),
key (col_char_key)
) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
CREATE TEMPORARY TABLE t LIKE t0;
INSERT INTO t VALUES
(NULL,1,1,'private','secret'),(NULL,2,2,'sacred','success'),
(NULL,3,3,'story','secure'),(NULL,4,4,'security','sacrament');
# Force a redo log flush at the next commit.
SET GLOBAL innodb_flush_log_at_trx_commit=1;
INSERT INTO t0
SELECT NULL, t1.col_int, t1.col_int_key, t1.col_char, t1.col_char_key
FROM t t1, t t2, t t3, t t4, t t5;
--source include/kill_mysqld.inc
--let SEARCH_RANGE = 10000000
--let SEARCH_PATTERN=private|secret|sacr(ed|ament)|success|story|secur(e|ity)
--echo # ibdata1 expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/ibdata1
-- source include/search_pattern_in_file.inc
--echo # t0.ibd expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t0.ibd
-- source include/search_pattern_in_file.inc
--echo # ib_logfile0 expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/ib_logfile0
-- source include/search_pattern_in_file.inc
--echo # ib_logfile1 expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/ib_logfile1
-- source include/search_pattern_in_file.inc
--echo # Restart without redo log encryption
-- let $restart_parameters=--skip-innodb-encrypt-log --innodb-log-files-in-group=1
-- source include/start_mysqld.inc
SELECT COUNT(*) FROM t0;
CHECK TABLE t0;
# Force a redo log flush at the next commit.
SET GLOBAL innodb_flush_log_at_trx_commit=1;
# If we tested with UPDATE, we would get clear-text redo log for
# encrypted undo log written with the old secret values.
INSERT INTO t0 VALUES(NULL, 5, 5, 'public', 'gossip');
--source include/kill_mysqld.inc
--echo # ib_logfile0 expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/ib_logfile0
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=public|gossip
--echo # ib_logfile0 expecting FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/ib_logfile0
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=private|secret|sacr(ed|ament)|success|story|secur(e|ity)|public|gossip
--echo # ibdata1 expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/ibdata1
-- source include/search_pattern_in_file.inc
--echo # t0.ibd expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t0.ibd
-- source include/search_pattern_in_file.inc
--let $restart_parameters=
--source include/start_mysqld.inc
SELECT COUNT(*) FROM t0;
CHECK TABLE t0;
DROP TABLE t0;

5
mysql-test/suite/encryption/t/innodb_encrypt_log_corruption.test
View file

@ -3,9 +3,12 @@
SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
--source include/shutdown_mysqld.inc
--let SEARCH_PATTERN= InnoDB: Encrypting redo log
--source include/search_pattern_in_file.inc
--let $restart_parameters=
--source include/restart_mysqld.inc
--source include/start_mysqld.inc
--list_files $bugdir
--remove_files_wildcard $bugdir

4
mysql-test/suite/innodb/r/innodb-32k.result
View file

@ -1,7 +1,3 @@
call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value ");
call mtr.add_suppression("InnoDB: Resizing redo log from ");
call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
call mtr.add_suppression("InnoDB: New log files created, LSN=");
call mtr.add_suppression("Innodb: Cannot add field.*row size is");
# Test 1) Show the page size from Information Schema
SELECT variable_value FROM information_schema.global_status

4
mysql-test/suite/innodb/r/innodb-64k.result
View file

@ -1,7 +1,3 @@
call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *");
call mtr.add_suppression("InnoDB: Resizing redo log from *");
call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files.");
call mtr.add_suppression("InnoDB: New log files created, LSN=*");
# Test 1) Show the page size from Information Schema
SELECT variable_value FROM information_schema.global_status
WHERE LOWER(variable_name) = 'innodb_page_size';

4
mysql-test/suite/innodb/r/innodb-bigblob.result
View file

@ -1,7 +1,3 @@
call mtr.add_suppression("Resizing redo log from *");
call mtr.add_suppression("Starting to delete and rewrite log files.");
call mtr.add_suppression("New log files created, LSN=*");
call mtr.add_suppression("Writer thread is waiting this semaphore");
create table foo (id varchar(37) not null, content longblob) engine=INNODB;
insert into foo (id, content) values('xyz', '');
update foo set content=repeat('a', 43941888) where id='xyz';

5
mysql-test/suite/innodb/r/log_corruption.result
View file

@ -41,7 +41,7 @@ WHERE engine = 'innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS
FOUND /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
FOUND /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
# --innodb-force-recovery=6 (skip the entire redo log)
SELECT * FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
@ -54,7 +54,6 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS
FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
# --innodb-force-recovery=6 (skip the entire redo log)
SELECT * FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
@ -90,7 +89,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS
FOUND /InnoDB: Redo log crypto: getting mysqld crypto key from key version failed err = 4294967295/ in mysqld.1.err
FOUND /InnoDB: Obtaining redo log encryption key version 1 failed/ in mysqld.1.err
FOUND /InnoDB: Decrypting checkpoint failed/ in mysqld.1.err
ib_buffer_pool
ib_logfile0

2
mysql-test/suite/innodb/r/log_file.result
View file

@ -340,7 +340,7 @@ WHERE engine='innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
1
1
FOUND /Resizing redo log from 1\*\d+ to 3\*\d+ pages, LSN=\d+/ in mysqld.1.err
FOUND /Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+/ in mysqld.1.err
# Cleanup
bak_ib_logfile0
bak_ib_logfile1

4
mysql-test/suite/innodb/t/innodb-32k.test
View file

@ -3,10 +3,6 @@
--source include/have_innodb.inc
--source include/have_innodb_32k.inc
call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value ");
call mtr.add_suppression("InnoDB: Resizing redo log from ");
call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
call mtr.add_suppression("InnoDB: New log files created, LSN=");
call mtr.add_suppression("Innodb: Cannot add field.*row size is");
let $MYSQLD_DATADIR= `select @@datadir`;

5
mysql-test/suite/innodb/t/innodb-64k.test
View file

@ -3,11 +3,6 @@
--source include/have_innodb.inc
--source include/have_innodb_64k.inc
call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *");
call mtr.add_suppression("InnoDB: Resizing redo log from *");
call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files.");
call mtr.add_suppression("InnoDB: New log files created, LSN=*");
let $MYSQLD_DATADIR= `select @@datadir`;
--echo # Test 1) Show the page size from Information Schema

5
mysql-test/suite/innodb/t/innodb-bigblob.test
View file

@ -6,11 +6,6 @@
let $status_orig=`SELECT @@innodb_status_output`;
--enable_query_log
call mtr.add_suppression("Resizing redo log from *");
call mtr.add_suppression("Starting to delete and rewrite log files.");
call mtr.add_suppression("New log files created, LSN=*");
call mtr.add_suppression("Writer thread is waiting this semaphore");
create table foo (id varchar(37) not null, content longblob) engine=INNODB;
insert into foo (id, content) values('xyz', '');
update foo set content=repeat('a', 43941888) where id='xyz';

14
mysql-test/suite/innodb/t/log_corruption.test
View file

@ -9,11 +9,11 @@ call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE faile
call mtr.add_suppression("InnoDB: Unsupported redo log format");
call mtr.add_suppression("InnoDB: No valid checkpoint found");
call mtr.add_suppression("InnoDB: Invalid (log block|redo log header) checksum");
call mtr.add_suppression("InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT");
call mtr.add_suppression("InnoDB: Missing MLOG_CHECKPOINT");
call mtr.add_suppression("InnoDB: MLOG_FILE_NAME incorrect");
call mtr.add_suppression("InnoDB: ############### CORRUPT LOG RECORD FOUND");
call mtr.add_suppression("InnoDB: Found corrupted log");
call mtr.add_suppression("InnoDB: Redo log crypto: getting mysqld crypto key from key version failed");
call mtr.add_suppression("InnoDB: Log scan aborted at LSN");
call mtr.add_suppression("InnoDB: Obtaining redo log encryption key version 1 failed");
call mtr.add_suppression("InnoDB: Decrypting checkpoint failed");
--enable_query_log
@ -206,7 +206,7 @@ eval $check_no_innodb;
--source include/shutdown_mysqld.inc
let SEARCH_PATTERN=InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122;
--source include/search_pattern_in_file.inc
let SEARCH_PATTERN=InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
let SEARCH_PATTERN=InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
--source include/search_pattern_in_file.inc
--echo # --innodb-force-recovery=6 (skip the entire redo log)
--let $restart_parameters= $dirs --innodb-force-recovery=6
@ -232,14 +232,12 @@ print OUT pack("H*x[5]", "C0DEBA5E0022000c0000000138");
print OUT pack("H*x[475]H*", "12860cb7809781e80006626f677573", "089C0ADA");
EOF
--copy_file $bugdir/ib_logfile0 $bugdir/ib_logfile
# Anything below innodb_force_recovery=6 must find a valid redo log.
# Anything below innodb_force_recovery=6 must find an invalid redo log.
# Missing tablespace files are tolerated already with innodb_force_recovery=1.
--let $restart_parameters= $dirs --innodb-force-recovery=5
--source include/start_mysqld.inc
eval $check_no_innodb;
--source include/shutdown_mysqld.inc
let SEARCH_PATTERN=InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
--source include/search_pattern_in_file.inc
--echo # --innodb-force-recovery=6 (skip the entire redo log)
--let $restart_parameters= $dirs --innodb-force-recovery=6
--source include/start_mysqld.inc
@ -340,7 +338,7 @@ EOF
if (!$no_cleanup) {
eval $check_no_innodb;
--source include/shutdown_mysqld.inc
--let SEARCH_PATTERN= InnoDB: Redo log crypto: getting mysqld crypto key from key version failed err = 4294967295
--let SEARCH_PATTERN= InnoDB: Obtaining redo log encryption key version 1 failed
--source include/search_pattern_in_file.inc
--let SEARCH_PATTERN= InnoDB: Decrypting checkpoint failed
--source include/search_pattern_in_file.inc

2
mysql-test/suite/innodb/t/log_file.test
View file

@ -219,7 +219,7 @@ eval $check_no_innodb;
--source include/start_mysqld.inc
eval $check_yes_innodb;
--source include/shutdown_mysqld.inc
--let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ pages, LSN=\d+
--let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+
--source include/search_pattern_in_file.inc
--let $restart_parameters=

2
mysql-test/suite/innodb/t/log_file_name.test
View file

@ -160,7 +160,7 @@ call mtr.add_suppression("InnoDB: Cannot read first page in datafile: .*test.*ib
call mtr.add_suppression("InnoDB: Datafile '.*test.*ibd' is corrupted");
call mtr.add_suppression("InnoDB: Cannot replay file rename. Remove either file and try again");
call mtr.add_suppression("InnoDB: Cannot rename.*because the target file exists");
call mtr.add_suppression("InnoDB: Found corrupted log");
call mtr.add_suppression("InnoDB: Log scan aborted at LSN");
# The following are for the --innodb-force-recovery=1 with broken u* tables:
call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd");
call mtr.add_suppression("InnoDB: The error means the system cannot find the path specified");

9
mysql-test/suite/innodb/t/log_file_size.test
View file

@ -10,9 +10,6 @@ if (`SELECT @@innodb_log_file_size = 1048576`) {
}
--disable_query_log
call mtr.add_suppression("InnoDB: Resizing redo log");
call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
call mtr.add_suppression("InnoDB: New log files created");
call mtr.add_suppression("InnoDB: The log sequence numbers [0-9]+ and [0-9]+ in ibdata files do not match the log sequence number [0-9]+ in the ib_logfiles");
call mtr.add_suppression("syntax error in innodb_log_group_home_dir");
call mtr.add_suppression("Plugin 'InnoDB' init function returned error");
@ -100,7 +97,7 @@ let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery;
SELECT * FROM t1;
let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
--source include/search_pattern_in_file.inc
let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
--source include/search_pattern_in_file.inc
--let $restart_parameters= --debug=d,innodb_log_abort_5
@ -109,7 +106,7 @@ let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
SELECT * FROM t1;
let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
--source include/search_pattern_in_file.inc
let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
--source include/search_pattern_in_file.inc
--let $restart_parameters= --innodb-read-only
@ -126,7 +123,7 @@ SELECT * FROM t1;
let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
--source include/search_pattern_in_file.inc
let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
--source include/search_pattern_in_file.inc
--let $restart_parameters= --debug=d,innodb_log_abort_7

5
mysql-test/suite/innodb/t/log_file_size_checkpoint.test
View file

@ -8,11 +8,6 @@ let $n=250;
let $t=veryLongTableNameToCreateMLOG_FILE_NAMErecords;
--disable_query_log
call mtr.add_suppression("InnoDB: Resizing redo log");
call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
call mtr.add_suppression("InnoDB: New log files created");
FLUSH TABLES;
let $i=$n;
while ($i)
{

56
storage/innobase/handler/ha_innodb.cc
View file

@ -89,7 +89,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
#include "fts0types.h"
#include "ibuf0ibuf.h"
#include "lock0lock.h"
#include "log0log.h"
#include "log0crypt.h"
#include "mem0mem.h"
#include "mtr0mtr.h"
#include "os0file.h"
@ -3748,14 +3748,42 @@ static const char* deprecated_use_trim
/** Update log_checksum_algorithm_ptr with a pointer to the function
corresponding to whether checksums are enabled.
@param[in] check whether redo log block checksums are enabled */
static
void
innodb_log_checksums_func_update(bool check)
@param[in,out] thd client session, or NULL if at startup
@param[in] check whether redo log block checksums are enabled
@return whether redo log block checksums are enabled */
static inline
bool
innodb_log_checksums_func_update(THD* thd, bool check)
{
log_checksum_algorithm_ptr = check
? log_block_calc_checksum_crc32
: log_block_calc_checksum_none;
static const char msg[] = "innodb_encrypt_log implies"
" innodb_log_checksums";
ut_ad(!thd == !srv_was_started);
if (!check) {
check = srv_encrypt_log;
if (!check) {
} else if (thd) {
push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
HA_ERR_UNSUPPORTED, msg);
} else {
sql_print_warning(msg);
}
}
if (thd) {
log_mutex_enter();
log_checksum_algorithm_ptr = check
? log_block_calc_checksum_crc32
: log_block_calc_checksum_none;
log_mutex_exit();
} else {
log_checksum_algorithm_ptr = check
? log_block_calc_checksum_crc32
: log_block_calc_checksum_none;
}
return(check);
}
/****************************************************************//**
@ -4244,7 +4272,8 @@ innobase_change_buffering_inited_ok:
srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_NONE;
}
innodb_log_checksums_func_update(innodb_log_checksums);
innodb_log_checksums = innodb_log_checksums_func_update(
NULL, innodb_log_checksums);
#ifdef HAVE_LINUX_LARGE_PAGES
if ((os_use_large_pages = my_use_large_pages)) {
@ -20229,13 +20258,8 @@ innodb_log_checksums_update(
void* var_ptr,
const void* save)
{
my_bool check = *static_cast<my_bool*>(var_ptr)
= *static_cast<const my_bool*>(save);
/* Make sure we are the only log user */
mutex_enter(&log_sys->mutex);
innodb_log_checksums_func_update(check);
mutex_exit(&log_sys->mutex);
*static_cast<my_bool*>(var_ptr) = innodb_log_checksums_func_update(
thd, *static_cast<const my_bool*>(save));
}
static SHOW_VAR innodb_status_variables_export[]= {

1
storage/innobase/include/fil0crypt.h
View file

@ -27,6 +27,7 @@ Created 04/01/2015 Jan Lindström
#define fil0crypt_h
#include "os0event.h"
#include "my_crypt.h"
/**
* Magic pattern in start of crypt data on page 0

95
storage/innobase/include/log0crypt.h
View file

@ -22,27 +22,21 @@ Innodb log encrypt/decrypt
Created 11/25/2013 Minli Zhu
Modified Jan Lindström jan.lindstrom@mariadb.com
MDEV-11782: Rewritten for MariaDB 10.2 by Marko Mäkelä, MariaDB Corporation.
*******************************************************/
#ifndef log0crypt_h
#define log0crypt_h
#include "univ.i"
#include "ut0byte.h"
#include "my_crypt.h"
#include "log0log.h"
typedef int Crypt_result;
/* If true, enable redo log encryption. */
/** innodb_encrypt_log: whether to encrypt the redo log */
extern my_bool srv_encrypt_log;
/***********************************************************************
Set next checkpoint's key version to latest one, and generate new key */
/** Initialize the redo log encryption key.
@return whether the operation succeeded */
UNIV_INTERN
void
log_crypt_set_ver_and_key(
/*======================*/
ib_uint64_t next_checkpoint_no);/*!< in: next checkpoint no */
bool
log_crypt_init();
/*********************************************************************//**
Writes the crypto (version, msg and iv) info, which has been used for
@ -64,78 +58,23 @@ log_crypt_101_read_checkpoint(const byte* buf);
/** Decrypt a MariaDB 10.1 redo log block.
@param[in,out] buf log block
@return whether the decryption was successful */
UNIV_INTERN
bool
log_crypt_101_read_block(byte* buf);
/*********************************************************************//**
Read the crypto (version, msg and iv) info, which has been used for
log blocks with lsn <= this checkpoint's lsn, from a log header's
checkpoint buf. */
/** Read the checkpoint crypto (version, msg and iv) info.
@param[in] buf checkpoint buffer
@return whether the operation was successful */
UNIV_INTERN
bool
log_crypt_read_checkpoint_buf(
/*===========================*/
const byte* buf); /*!< in: checkpoint buffer */
log_crypt_read_checkpoint_buf(const byte* buf);
/********************************************************
Encrypt one or more log block before it is flushed to disk */
/** Encrypt or decrypt log blocks.
@param[in,out] buf log blocks to encrypt or decrypt
@param[in] size size of the buffer, in bytes
@param[in] decrypt whether to decrypt instead of encrypting */
UNIV_INTERN
void
log_encrypt_before_write(
/*=====================*/
ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */
byte* block, /*!< in/out: pointer to a log block */
const ulint size); /*!< in: size of log blocks */
/********************************************************
Decrypt a specified log segment after they are read from a log file to a buffer.
*/
UNIV_INTERN
void
log_decrypt_after_read(
/*===================*/
byte* frame, /*!< in/out: log segment */
const ulint size); /*!< in: log segment size */
/* Error codes for crypt info */
typedef enum {
LOG_UNENCRYPTED = 0,
LOG_CRYPT_KEY_NOT_FOUND = 1,
LOG_DECRYPT_MAYBE_FAILED = 2
} log_crypt_err_t;
/********************************************************
Check is the checkpoint information encrypted. This check
is based on fact has log group crypt info and based
on this crypt info was the key version different from
unencrypted key version. There is no realible way to
distinguish encrypted log block from corrupted log block,
but if log block corruption is found this function is
used to find out if log block is maybe encrypted but
encryption key, key management plugin or encryption
algorithm does not match.
@return TRUE, if log block may be encrypted */
UNIV_INTERN
ibool
log_crypt_block_maybe_encrypted(
/*============================*/
const byte* log_block, /*!< in: log block */
log_crypt_err_t* err_info); /*!< out: error info */
/********************************************************
Print crypt error message to error log */
UNIV_INTERN
void
log_crypt_print_error(
/*==================*/
log_crypt_err_t err_info); /*!< out: error info */
/*********************************************************************//**
Print checkpoint no from log block and all encryption keys from
checkpoints if they are present. Used for problem analysis. */
void
log_crypt_print_checkpoint_keys(
/*============================*/
const byte* log_block);
log_crypt(byte* buf, ulint size, bool decrypt = false);
#endif // log0crypt.h

49
storage/innobase/include/log0log.h
View file

@ -37,13 +37,9 @@ Created 12/9/1995 Heikki Tuuri
#include "univ.i"
#include "dyn0buf.h"
#include "sync0rw.h"
#include "log0crypt.h"
#include "log0types.h"
#include "os0event.h"
/** Redo log buffer */
struct log_t;
/** Redo log group */
struct log_group_t;
@ -275,15 +271,6 @@ objects! */
void
log_check_margins(void);
/******************************************************//**
Reads a specified log segment to a buffer. */
void
log_group_read_log_seg(
/*===================*/
byte* buf, /*!< in: buffer where to read */
log_group_t* group, /*!< in: log group */
lsn_t start_lsn, /*!< in: read area start */
lsn_t end_lsn); /*!< in: read area end */
/********************************************************//**
Sets the field values in group to correspond to a given lsn. For this function
to work, the values must already be correctly initialized to correspond to
@ -449,9 +436,6 @@ void
log_mem_free(void);
/*==============*/
/** Redo log system */
extern log_t* log_sys;
/** Whether to generate and require checksums on the redo log pages */
extern my_bool innodb_log_checksums;
@ -508,6 +492,12 @@ extern my_bool innodb_log_checksums;
#define LOG_CHECKPOINT_LSN 8
#define LOG_CHECKPOINT_OFFSET 16
#define LOG_CHECKPOINT_LOG_BUF_SIZE 24
/** MariaDB 10.2.5 encrypted redo log encryption key version (32 bits)*/
#define LOG_CHECKPOINT_CRYPT_KEY 32
/** MariaDB 10.2.5 encrypted redo log random nonce (32 bits) */
#define LOG_CHECKPOINT_CRYPT_NONCE 36
/** MariaDB 10.2.5 encrypted redo log random message (MY_AES_BLOCK_SIZE) */
#define LOG_CHECKPOINT_CRYPT_MESSAGE 40
/** Offsets of a log file header */
/* @{ */
@ -538,19 +528,11 @@ or the MySQL version that created the redo log file. */
/** The redo log format identifier corresponding to the current format version.
Stored in LOG_HEADER_FORMAT. */
#define LOG_HEADER_FORMAT_CURRENT 1
/** Encrypted MariaDB redo log */
#define LOG_HEADER_FORMAT_ENCRYPTED (1U<<31)
/* @} */
/** MariaDB Server 10.1 encrypted redo log offsets */
/* @{ */
#define LOG_CRYPT_VER (20 + 32 * 9)
#define LOG_CRYPT_MAX_ENTRIES (5)
#define LOG_CRYPT_ENTRY_SIZE (4 + 4 + 2 * MY_AES_BLOCK_SIZE)
#define LOG_CRYPT_SIZE (1 + 1 + \
(LOG_CRYPT_MAX_ENTRIES * \
LOG_CRYPT_ENTRY_SIZE))
/* @} */
#define LOG_CHECKPOINT_1 OS_FILE_LOG_BLOCK_SIZE
/* first checkpoint field in the log
header; we write alternately to the
@ -609,6 +591,12 @@ struct log_group_t{
byte* checkpoint_buf;
/** list of log groups */
UT_LIST_NODE_T(log_group_t) log_groups;
/** @return whether the redo log is encrypted */
bool is_encrypted() const
{
return((format & LOG_HEADER_FORMAT_ENCRYPTED) != 0);
}
};
/** Redo log buffer */
@ -750,8 +738,17 @@ struct log_t{
byte* checkpoint_buf; /*!< checkpoint header is read to this
buffer */
/* @} */
/** @return whether the redo log is encrypted */
bool is_encrypted() const
{
return(UT_LIST_GET_FIRST(log_groups)->is_encrypted());
}
};
/** Redo log system */
extern log_t* log_sys;
/** Test if flush order mutex is owned. */
#define log_flush_order_mutex_own() \
mutex_own(&log_sys->log_flush_order_mutex)

9
storage/innobase/include/log0recv.h
View file

@ -297,13 +297,4 @@ use these free frames to read in pages when we start applying the
log records to the database. */
extern ulint recv_n_pool_free_frames;
/******************************************************//**
Checks the 4-byte checksum to the trailer checksum field of a log
block. */
bool
log_block_checksum_is_ok(
/*===================================*/
const byte* block, /*!< in: pointer to a log block */
bool print_err); /*!< in print error ? */
#endif

713
storage/innobase/log/log0crypt.cc
View file

@ -22,55 +22,46 @@ Innodb log encrypt/decrypt
Created 11/25/2013 Minli Zhu Google
Modified Jan Lindström jan.lindstrom@mariadb.com
MDEV-11782: Rewritten for MariaDB 10.2 by Marko Mäkelä, MariaDB Corporation.
*******************************************************/
#include "m_string.h"
#include "log0crypt.h"
#include <my_crypt.h>
#include <my_crypt.h>
#include "my_crypt.h"
#include "log0log.h"
#include "log0crypt.h"
#include "srv0start.h" // for srv_start_lsn
#include "log0recv.h" // for recv_sys
#include "ha_prototypes.h" // IB_LOG_
#include "my_crypt.h"
/* Used for debugging */
// #define DEBUG_CRYPT 1
#define UNENCRYPTED_KEY_VER 0
/* If true, enable redo log encryption. */
extern my_bool srv_encrypt_log;
#include <algorithm> // std::sort
#include <deque>
/* If true, enable redo log encryption. */
UNIV_INTERN my_bool srv_encrypt_log = FALSE;
/*
Sub system type for InnoDB redo log crypto.
Set and used to validate crypto msg.
*/
static const byte redo_log_purpose_byte = 0x02;
/** innodb_encrypt_log: whether to encrypt the redo log */
my_bool srv_encrypt_log;
/** Redo log encryption key ID */
#define LOG_DEFAULT_ENCRYPTION_KEY 1
/*
Store this many keys into each checkpoint info
*/
static const size_t kMaxSavedKeys = LOG_CRYPT_MAX_ENTRIES;
typedef union {
uint32_t words[MY_AES_BLOCK_SIZE / sizeof(uint32_t)];
byte bytes[MY_AES_BLOCK_SIZE];
} aes_block_t;
struct crypt_info_t {
ib_uint64_t checkpoint_no; /*!< checkpoint no */
ulint checkpoint_no; /*!< checkpoint no; 32 bits */
uint key_version; /*!< mysqld key version */
byte crypt_msg[MY_AES_BLOCK_SIZE];
byte crypt_key[MY_AES_BLOCK_SIZE];
byte crypt_nonce[MY_AES_BLOCK_SIZE];
/** random string for encrypting the key */
aes_block_t crypt_msg;
/** the secret key */
aes_block_t crypt_key;
/** a random string for the per-block initialization vector */
union {
uint32_t word;
byte bytes[4];
} crypt_nonce;
};
static std::deque<crypt_info_t> crypt_info;
/** The crypt info */
static crypt_info_t info;
/** Crypt info when upgrading from 10.1 */
static crypt_info_t infos[5];
/*********************************************************************//**
Get a log block's start lsn.
@ -93,366 +84,167 @@ Get crypt info from checkpoint.
@return a crypt info or NULL if not present. */
static
const crypt_info_t*
get_crypt_info(
/*===========*/
ib_uint64_t checkpoint_no)
get_crypt_info(ulint checkpoint_no)
{
size_t items = crypt_info.size();
/* a log block only stores 4-bytes of checkpoint no */
checkpoint_no &= 0xFFFFFFFF;
for (size_t i = 0; i < items; i++) {
struct crypt_info_t* it = &crypt_info[i];
for (unsigned i = 0; i < 5; i++) {
const crypt_info_t* it = &infos[i];
if (it->checkpoint_no == checkpoint_no) {
if (it->key_version && it->checkpoint_no == checkpoint_no) {
return it;
}
}
/* If checkpoint contains more than one key and we did not
find the correct one use the first one. */
if (items) {
return (&crypt_info[0]);
}
return NULL;
return infos;
}
/*********************************************************************//**
Get crypt info from log block
@return a crypt info or NULL if not present. */
static
const crypt_info_t*
get_crypt_info(
/*===========*/
const byte* log_block)
{
ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block);
return get_crypt_info(checkpoint_no);
}
/*********************************************************************//**
Print checkpoint no from log block and all encryption keys from
checkpoints if they are present. Used for problem analysis. */
/** Encrypt or decrypt log blocks.
@param[in,out] buf log blocks to encrypt or decrypt
@param[in] size size of the buffer, in bytes
@param[in] decrypt whether to decrypt instead of encrypting */
UNIV_INTERN
void
log_crypt_print_checkpoint_keys(
/*============================*/
const byte* log_block)
log_crypt(byte* buf, ulint size, bool decrypt)
{
ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block);
ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0);
ut_a(info.key_version);
if (crypt_info.size()) {
ib::info() << "Redo log checkpoint encryption: " << checkpoint_no << " [ chk key ]: ";
for (size_t i = 0; i < crypt_info.size(); i++) {
struct crypt_info_t* it = &crypt_info[i];
ib::info() << "[" << it->checkpoint_no
<< "," << it->key_version
<< "]";
}
}
}
/*********************************************************************//**
Call AES CTR to encrypt/decrypt log blocks. */
static
Crypt_result
log_blocks_crypt(
/*=============*/
const byte* block, /*!< in: blocks before encrypt/decrypt*/
ulint size, /*!< in: size of block */
byte* dst_block, /*!< out: blocks after encrypt/decrypt */
int what, /*!< in: encrypt or decrypt*/
const crypt_info_t* crypt_info) /*!< in: crypt info or NULL */
{
byte *log_block = (byte*)block;
Crypt_result rc = MY_AES_OK;
uint dst_len;
byte aes_ctr_counter[MY_AES_BLOCK_SIZE];
byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT;
lsn_t lsn = is_encrypt ? log_sys->lsn : srv_start_lsn;
uint32_t aes_ctr_iv[MY_AES_BLOCK_SIZE / sizeof(uint32_t)];
compile_time_assert(sizeof(uint32_t) == 4);
const uint src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE;
for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) {
ulint log_block_no = log_block_get_hdr_no(log_block);
lsn_t log_block_start_lsn = log_block_get_start_lsn(
lsn, log_block_no);
#define LOG_CRYPT_HDR_SIZE 4
const crypt_info_t* info = crypt_info == NULL ? get_crypt_info(log_block) :
crypt_info;
#ifdef DEBUG_CRYPT
fprintf(stderr,
"%s %lu chkpt: %lu key: %u lsn: %lu\n",
is_encrypt ? "crypt" : "decrypt",
log_block_no,
log_block_get_checkpoint_no(log_block),
info ? info->key_version : 0,
log_block_start_lsn);
for (const byte* const end = buf + size; buf != end;
buf += OS_FILE_LOG_BLOCK_SIZE) {
byte dst[OS_FILE_LOG_BLOCK_SIZE - LOG_CRYPT_HDR_SIZE];
const ulint log_block_no = log_block_get_hdr_no(buf);
/* The log block number is not encrypted. */
*aes_ctr_iv =
#ifdef WORDS_BIGENDIAN
~LOG_BLOCK_FLUSH_BIT_MASK
#else
~(LOG_BLOCK_FLUSH_BIT_MASK >> 24)
#endif
/* If no key is found from checkpoint assume the log_block
to be unencrypted. If checkpoint contains the encryption key
compare log_block current checksum, if checksum matches,
block can't be encrypted. */
if (info == NULL ||
info->key_version == UNENCRYPTED_KEY_VER ||
(log_block_checksum_is_ok(log_block, false) &&
what == ENCRYPTION_FLAG_DECRYPT)) {
memcpy(dst_block, log_block, OS_FILE_LOG_BLOCK_SIZE);
goto next;
}
& (*reinterpret_cast<uint32_t*>(dst)
= *reinterpret_cast<const uint32_t*>(
buf + LOG_BLOCK_HDR_NO));
#if LOG_BLOCK_HDR_NO + 4 != LOG_CRYPT_HDR_SIZE
# error "LOG_BLOCK_HDR_NO has been moved; redo log format affected!"
#endif
aes_ctr_iv[1] = info.crypt_nonce.word;
mach_write_to_8(reinterpret_cast<byte*>(aes_ctr_iv + 2),
log_block_get_start_lsn(
decrypt ? srv_start_lsn : log_sys->lsn,
log_block_no));
ut_ad(what == ENCRYPTION_FLAG_DECRYPT ? !log_block_checksum_is_ok(log_block, false) :
log_block_checksum_is_ok(log_block, false));
// Assume log block header is not encrypted
memcpy(dst_block, log_block, LOG_BLOCK_HDR_SIZE);
// aes_ctr_counter = nonce(3-byte) + start lsn to a log block
// (8-byte) + lbn (4-byte) + abn
// (1-byte, only 5 bits are used). "+" means concatenate.
bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE);
memcpy(aes_ctr_counter, info->crypt_nonce, 3);
mach_write_to_8(aes_ctr_counter + 3, log_block_start_lsn);
mach_write_to_4(aes_ctr_counter + 11, log_block_no);
bzero(aes_ctr_counter + 15, 1);
int rc;
rc = encryption_crypt(log_block + LOG_BLOCK_HDR_SIZE, src_len,
dst_block + LOG_BLOCK_HDR_SIZE, &dst_len,
(unsigned char*)(info->crypt_key), 16,
aes_ctr_counter, MY_AES_BLOCK_SIZE,
what | ENCRYPTION_FLAG_NOPAD,
LOG_DEFAULT_ENCRYPTION_KEY,
info->key_version);
int rc = encryption_crypt(
buf + LOG_CRYPT_HDR_SIZE, sizeof dst, dst, &dst_len,
const_cast<byte*>(info.crypt_key.bytes),
sizeof info.crypt_key,
reinterpret_cast<byte*>(aes_ctr_iv), sizeof aes_ctr_iv,
decrypt
? ENCRYPTION_FLAG_DECRYPT | ENCRYPTION_FLAG_NOPAD
: ENCRYPTION_FLAG_ENCRYPT | ENCRYPTION_FLAG_NOPAD,
LOG_DEFAULT_ENCRYPTION_KEY,
info.key_version);
ut_a(rc == MY_AES_OK);
ut_a(dst_len == src_len);
next:
log_block += OS_FILE_LOG_BLOCK_SIZE;
dst_block += OS_FILE_LOG_BLOCK_SIZE;
}
ut_a(dst_len == sizeof dst);
if (decrypt) {
std::ostringstream s;
ut_print_buf_hex(s, buf + LOG_CRYPT_HDR_SIZE,
OS_FILE_LOG_BLOCK_SIZE
- LOG_CRYPT_HDR_SIZE);
ib::info() << "S: " << s.str();
std::ostringstream d;
ut_print_buf_hex(d, dst,
OS_FILE_LOG_BLOCK_SIZE
- LOG_CRYPT_HDR_SIZE);
ib::info() << "c: " << d.str();
}
return rc;
memcpy(buf + LOG_CRYPT_HDR_SIZE, dst, sizeof dst);
}
}
/*********************************************************************//**
Generate crypt key from crypt msg.
@return true if successfull, false if not. */
/** Generate crypt key from crypt msg.
@param[in,out] info encryption key
@param[in] upgrade whether to use the key in MariaDB 10.1 format
@return whether the operation was successful */
static
bool
init_crypt_key(
/*===========*/
crypt_info_t* info) /*< in/out: crypt info */
init_crypt_key(crypt_info_t* info, bool upgrade = false)
{
if (info->key_version == UNENCRYPTED_KEY_VER) {
memset(info->crypt_key, 0, sizeof(info->crypt_key));
memset(info->crypt_msg, 0, sizeof(info->crypt_msg));
memset(info->crypt_nonce, 0, sizeof(info->crypt_nonce));
return true;
byte mysqld_key[MY_AES_MAX_KEY_LENGTH];
uint keylen = sizeof mysqld_key;
compile_time_assert(16 == sizeof info->crypt_key);
if (uint rc = encryption_key_get(LOG_DEFAULT_ENCRYPTION_KEY,
info->key_version, mysqld_key,
&keylen)) {
ib::error()
<< "Obtaining redo log encryption key version "
<< info->key_version << " failed (" << rc
<< "). Maybe the key or the required encryption "
<< " key management plugin was not found.";
return false;
}
byte mysqld_key[MY_AES_MAX_KEY_LENGTH] = {0};
uint keylen= sizeof(mysqld_key);
uint rc;
rc = encryption_key_get(LOG_DEFAULT_ENCRYPTION_KEY, info->key_version, mysqld_key, &keylen);
if (rc) {
ib::error()
<< "Redo log crypto: getting mysqld crypto key "
<< "from key version failed err = " << rc
<< " Reason could be that requested key_version "
<< info->key_version
<< "is not found or required encryption "
<< " key management is not found.";
return false;
if (upgrade) {
while (keylen < sizeof mysqld_key) {
mysqld_key[keylen++] = 0;
}
}
uint dst_len;
int err= my_aes_crypt(MY_AES_ECB, ENCRYPTION_FLAG_NOPAD|ENCRYPTION_FLAG_ENCRYPT,
info->crypt_msg, sizeof(info->crypt_msg), //src, srclen
info->crypt_key, &dst_len, //dst, &dstlen
(unsigned char*)&mysqld_key, sizeof(mysqld_key),
NULL, 0);
int err= my_aes_crypt(MY_AES_ECB,
ENCRYPTION_FLAG_NOPAD | ENCRYPTION_FLAG_ENCRYPT,
info->crypt_msg.bytes, sizeof info->crypt_msg,
info->crypt_key.bytes, &dst_len,
mysqld_key, keylen, NULL, 0);
if (err != MY_AES_OK || dst_len != MY_AES_BLOCK_SIZE) {
fprintf(stderr,
"\nInnodb redo log crypto: getting redo log crypto key "
"failed err = %d len = %u.\n", err, dst_len);
ib::error() << "Getting redo log crypto key failed: err = "
<< err << ", len = " << dst_len;
return false;
}
return true;
}
/*********************************************************************//**
Compare function for checkpoint numbers
@return true if first checkpoint is larger than second one */
static
/** Initialize the redo log encryption key.
@return whether the operation succeeded */
UNIV_INTERN
bool
mysort(const crypt_info_t& i,
const crypt_info_t& j)
log_crypt_init()
{
return i.checkpoint_no > j.checkpoint_no;
}
ut_ad(log_mutex_own());
ut_ad(log_sys->is_encrypted());
/*********************************************************************//**
Add crypt info to set if it is not already present
@return true if successfull, false if not- */
static
bool
add_crypt_info(
/*===========*/
crypt_info_t* info, /*!< in: crypt info */
bool checkpoint_read)/*!< in: do we read checkpoint */
{
const crypt_info_t* found=NULL;
/* so that no one is searching array while we modify it */
ut_ad(mutex_own(&(log_sys->mutex)));
info.key_version = encryption_key_get_latest_version(
LOG_DEFAULT_ENCRYPTION_KEY);
found = get_crypt_info(info->checkpoint_no);
/* If one crypt info is found then we add a new one only if we
are reading checkpoint from the log. New checkpoints will always
use the first created crypt info. */
if (found != NULL &&
( found->checkpoint_no == info->checkpoint_no || !checkpoint_read)) {
// already present...
return true;
}
if (!init_crypt_key(info)) {
if (info.key_version == ENCRYPTION_KEY_VERSION_INVALID) {
ib::error() << "innodb_encrypt_log: cannot get key version";
info.key_version = 0;
return false;
}
crypt_info.push_back(*info);
/* a log block only stores 4-bytes of checkpoint no */
crypt_info.back().checkpoint_no &= 0xFFFFFFFF;
// keep keys sorted, assuming that last added key will be used most
std::sort(crypt_info.begin(), crypt_info.end(), mysort);
return true;
}
/*********************************************************************//**
Encrypt log blocks. */
UNIV_INTERN
Crypt_result
log_blocks_encrypt(
/*===============*/
const byte* block, /*!< in: blocks before encryption */
const ulint size, /*!< in: size of blocks, must be multiple of a log block */
byte* dst_block) /*!< out: blocks after encryption */
{
return log_blocks_crypt(block, size, dst_block, ENCRYPTION_FLAG_ENCRYPT, NULL);
}
/*********************************************************************//**
Set next checkpoint's key version to latest one, and generate current
key. Key version 0 means no encryption. */
UNIV_INTERN
void
log_crypt_set_ver_and_key(
/*======================*/
ib_uint64_t next_checkpoint_no)
{
crypt_info_t info;
info.checkpoint_no = next_checkpoint_no;
if (!srv_encrypt_log) {
info.key_version = UNENCRYPTED_KEY_VER;
} else {
info.key_version = encryption_key_get_latest_version(LOG_DEFAULT_ENCRYPTION_KEY);
if (my_random_bytes(info.crypt_msg.bytes, sizeof info.crypt_msg)
!= MY_AES_OK
|| my_random_bytes(info.crypt_nonce.bytes, sizeof info.crypt_nonce)
!= MY_AES_OK) {
ib::error() << "innodb_encrypt_log: my_random_bytes() failed";
return false;
}
if (info.key_version == UNENCRYPTED_KEY_VER) {
memset(info.crypt_msg, 0, sizeof(info.crypt_msg));
memset(info.crypt_nonce, 0, sizeof(info.crypt_nonce));
} else {
if (my_random_bytes(info.crypt_msg, MY_AES_BLOCK_SIZE) != MY_AES_OK) {
ib::error()
<< "Redo log crypto: generate "
<< MY_AES_BLOCK_SIZE
<< "-byte random number as crypto msg failed.";
ut_error;
}
if (my_random_bytes(info.crypt_nonce, MY_AES_BLOCK_SIZE) != MY_AES_OK) {
ib::error()
<< "Redo log crypto: generate "
<< MY_AES_BLOCK_SIZE
<< "-byte random number as AES_CTR nonce failed.";
ut_error;
}
}
add_crypt_info(&info, false);
}
/********************************************************
Encrypt one or more log block before it is flushed to disk */
UNIV_INTERN
void
log_encrypt_before_write(
/*=====================*/
ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */
byte* block, /*!< in/out: pointer to a log block */
const ulint size) /*!< in: size of log blocks */
{
ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0);
const crypt_info_t* info = get_crypt_info(next_checkpoint_no);
if (info == NULL) {
return;
}
/* If the key is not encrypted or user has requested not to
encrypt, do not change log block. */
if (info->key_version == UNENCRYPTED_KEY_VER || !srv_encrypt_log) {
return;
}
byte* dst_frame = (byte*)malloc(size);
//encrypt log blocks content
Crypt_result result = log_blocks_crypt(block, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL);
if (result == MY_AES_OK) {
ut_ad(block[0] == dst_frame[0]);
memcpy(block, dst_frame, size);
}
free(dst_frame);
if (unlikely(result != MY_AES_OK)) {
ut_error;
}
}
/********************************************************
Decrypt a specified log segment after they are read from a log file to a buffer.
*/
void
log_decrypt_after_read(
/*===================*/
byte* frame, /*!< in/out: log segment */
const ulint size) /*!< in: log segment size */
{
ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0);
byte* dst_frame = (byte*)malloc(size);
// decrypt log blocks content
Crypt_result result = log_blocks_crypt(frame, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL);
if (result == MY_AES_OK) {
memcpy(frame, dst_frame, size);
}
free(dst_frame);
if (unlikely(result != MY_AES_OK)) {
ut_error;
}
return init_crypt_key(&info);
}
/** Read the MariaDB 10.1 checkpoint crypto (version, msg and iv) info.
@ -467,13 +259,14 @@ log_crypt_101_read_checkpoint(const byte* buf)
const size_t n = *buf++ == 2 ? std::min(unsigned(*buf++), 5U) : 0;
for (size_t i = 0; i < n; i++) {
struct crypt_info_t info;
struct crypt_info_t& info = infos[i];
info.checkpoint_no = mach_read_from_4(buf);
info.key_version = mach_read_from_4(buf + 4);
memcpy(info.crypt_msg, buf + 8, MY_AES_BLOCK_SIZE);
memcpy(info.crypt_nonce, buf + 24, MY_AES_BLOCK_SIZE);
memcpy(info.crypt_msg.bytes, buf + 8, sizeof info.crypt_msg);
memcpy(info.crypt_nonce.bytes, buf + 24,
sizeof info.crypt_nonce);
if (!add_crypt_info(&info, true)) {
if (!init_crypt_key(&info, true)) {
return false;
}
buf += 4 + 4 + 2 * MY_AES_BLOCK_SIZE;
@ -491,7 +284,8 @@ log_crypt_101_read_block(byte* buf)
{
ut_ad(log_block_calc_checksum_format_0(buf)
!= log_block_get_checksum(buf));
const crypt_info_t* info = get_crypt_info(buf);
const crypt_info_t* info = get_crypt_info(
log_block_get_checkpoint_no(buf));
if (!info || info->key_version == 0) {
return false;
@ -499,27 +293,27 @@ log_crypt_101_read_block(byte* buf)
byte dst[OS_FILE_LOG_BLOCK_SIZE];
uint dst_len;
byte aes_ctr_counter[MY_AES_BLOCK_SIZE];
byte aes_ctr_iv[MY_AES_BLOCK_SIZE];
const uint src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE;
ulint log_block_no = log_block_get_hdr_no(buf);
lsn_t log_block_start_lsn = log_block_get_start_lsn(
srv_start_lsn, log_block_no);
/* The log block header is not encrypted. */
memcpy(dst, buf, LOG_BLOCK_HDR_SIZE);
memcpy(aes_ctr_counter, info->crypt_nonce, 3);
mach_write_to_8(aes_ctr_counter + 3, log_block_start_lsn);
mach_write_to_4(aes_ctr_counter + 11, log_block_no);
aes_ctr_counter[15] = 0;
memcpy(aes_ctr_iv, info->crypt_nonce.bytes, 3);
mach_write_to_8(aes_ctr_iv + 3,
log_block_get_start_lsn(srv_start_lsn, log_block_no));
memcpy(aes_ctr_iv + 11, buf, 4);
aes_ctr_iv[11] &= ~(LOG_BLOCK_FLUSH_BIT_MASK >> 24);
aes_ctr_iv[15] = 0;
int rc = encryption_crypt(buf + LOG_BLOCK_HDR_SIZE, src_len,
dst + LOG_BLOCK_HDR_SIZE, &dst_len,
const_cast<byte*>(info->crypt_key),
const_cast<byte*>(info->crypt_key.bytes),
MY_AES_BLOCK_SIZE,
aes_ctr_counter, MY_AES_BLOCK_SIZE,
aes_ctr_iv, MY_AES_BLOCK_SIZE,
ENCRYPTION_FLAG_DECRYPT
| ENCRYPTION_FLAG_NOPAD,
LOG_DEFAULT_ENCRYPTION_KEY,
@ -535,184 +329,47 @@ log_crypt_101_read_block(byte* buf)
return true;
}
/*********************************************************************//**
Writes the crypto (version, msg and iv) info, which has been used for
log blocks with lsn <= this checkpoint's lsn, to a log header's
checkpoint buf. */
/** Add the encryption information to a redo log checkpoint buffer.
@param[in,out] buf checkpoint buffer */
UNIV_INTERN
void
log_crypt_write_checkpoint_buf(
/*===========================*/
byte* buf) /*!< in/out: checkpoint buffer */
log_crypt_write_checkpoint_buf(byte* buf)
{
byte *save = buf;
ut_ad(info.key_version);
compile_time_assert(16 == sizeof info.crypt_msg);
compile_time_assert(LOG_CHECKPOINT_CRYPT_MESSAGE
- LOG_CHECKPOINT_CRYPT_NONCE
== sizeof info.crypt_nonce);
// Only write kMaxSavedKeys (sort keys to remove oldest)
std::sort(crypt_info.begin(), crypt_info.end(), mysort);
while (crypt_info.size() > kMaxSavedKeys) {
crypt_info.pop_back();
}
bool encrypted = false;
for (size_t i = 0; i < crypt_info.size(); i++) {
const crypt_info_t & it = crypt_info[i];
if (it.key_version != UNENCRYPTED_KEY_VER) {
encrypted = true;
break;
}
}
if (encrypted == false) {
// if no encryption is inuse then zero out
// crypt data for upward/downward compability
memset(buf + LOG_CRYPT_VER, 0, LOG_CRYPT_SIZE);
return;
}
ib_uint64_t checkpoint_no = mach_read_from_8(buf + LOG_CHECKPOINT_NO);
buf += LOG_CRYPT_VER;
mach_write_to_1(buf + 0, redo_log_purpose_byte);
mach_write_to_1(buf + 1, crypt_info.size());
buf += 2;
for (size_t i = 0; i < crypt_info.size(); i++) {
struct crypt_info_t* it = &crypt_info[i];
mach_write_to_4(buf + 0, static_cast<ulint>(it->checkpoint_no));
mach_write_to_4(buf + 4, it->key_version);
memcpy(buf + 8, it->crypt_msg, MY_AES_BLOCK_SIZE);
memcpy(buf + 24, it->crypt_nonce, MY_AES_BLOCK_SIZE);
buf += LOG_CRYPT_ENTRY_SIZE;
}
#ifdef DEBUG_CRYPT
fprintf(stderr, "write chk: %lu [ chk key ]: ", checkpoint_no);
for (size_t i = 0; i < crypt_info.size(); i++) {
struct crypt_info_t* it = &crypt_info[i];
fprintf(stderr, "[ %lu %u ] ",
it->checkpoint_no,
it->key_version);
}
fprintf(stderr, "\n");
#else
(void)checkpoint_no; // unused variable
#endif
ut_a((ulint)(buf - save) <= OS_FILE_LOG_BLOCK_SIZE);
memcpy(buf + LOG_CHECKPOINT_CRYPT_MESSAGE, info.crypt_msg.bytes,
sizeof info.crypt_msg);
memcpy(buf + LOG_CHECKPOINT_CRYPT_NONCE, info.crypt_nonce.bytes,
sizeof info.crypt_nonce);
mach_write_to_4(buf + LOG_CHECKPOINT_CRYPT_KEY, info.key_version);
}
/*********************************************************************//**
Read the crypto (version, msg and iv) info, which has been used for
log blocks with lsn <= this checkpoint's lsn, from a log header's
checkpoint buf. */
/** Read the checkpoint crypto (version, msg and iv) info.
@param[in] buf checkpoint buffer
@return whether the operation was successful */
UNIV_INTERN
bool
log_crypt_read_checkpoint_buf(
/*===========================*/
const byte* buf) { /*!< in: checkpoint buffer */
log_crypt_read_checkpoint_buf(const byte* buf)
{
info.checkpoint_no = mach_read_from_4(buf + (LOG_CHECKPOINT_NO + 4));
info.key_version = mach_read_from_4(buf + LOG_CHECKPOINT_CRYPT_KEY);
buf += LOG_CRYPT_VER;
byte scheme = buf[0];
if (scheme != redo_log_purpose_byte) {
return true;
}
buf++;
size_t n = buf[0];
buf++;
for (size_t i = 0; i < n; i++) {
struct crypt_info_t info;
info.checkpoint_no = mach_read_from_4(buf + 0);
info.key_version = mach_read_from_4(buf + 4);
memcpy(info.crypt_msg, buf + 8, MY_AES_BLOCK_SIZE);
memcpy(info.crypt_nonce, buf + 24, MY_AES_BLOCK_SIZE);
if (!add_crypt_info(&info, true)) {
return false;
}
buf += LOG_CRYPT_ENTRY_SIZE;
}
#ifdef DEBUG_CRYPT
fprintf(stderr, "read [ chk key ]: ");
for (size_t i = 0; i < crypt_info.size(); i++) {
struct crypt_info_t* it = &crypt_info[i];
fprintf(stderr, "[ %lu %u ] ",
it->checkpoint_no,
it->key_version);
}
fprintf(stderr, "\n");
#if MY_AES_BLOCK_SIZE != 16
# error "MY_AES_BLOCK_SIZE != 16; redo log checkpoint format affected"
#endif
return true;
}
/********************************************************
Check is the checkpoint information encrypted. This check
is based on fact has log group crypt info and based
on this crypt info was the key version different from
unencrypted key version. There is no realible way to
distinguish encrypted log block from corrupted log block,
but if log block corruption is found this function is
used to find out if log block is maybe encrypted but
encryption key, key management plugin or encryption
algorithm does not match.
@return TRUE, if log block may be encrypted */
UNIV_INTERN
ibool
log_crypt_block_maybe_encrypted(
/*============================*/
const byte* log_block, /*!< in: log block */
log_crypt_err_t* err_info) /*!< out: error info */
{
ibool maybe_encrypted = FALSE;
const crypt_info_t* crypt_info;
*err_info = LOG_UNENCRYPTED;
crypt_info = get_crypt_info(log_block);
if (crypt_info &&
crypt_info->key_version != UNENCRYPTED_KEY_VER) {
byte mysqld_key[MY_AES_BLOCK_SIZE] = {0};
uint keylen= sizeof(mysqld_key);
/* Log block contains crypt info and based on key
version block could be encrypted. */
*err_info = LOG_DECRYPT_MAYBE_FAILED;
maybe_encrypted = TRUE;
if (encryption_key_get(LOG_DEFAULT_ENCRYPTION_KEY,
crypt_info->key_version, mysqld_key, &keylen)) {
*err_info = LOG_CRYPT_KEY_NOT_FOUND;
}
}
return (maybe_encrypted);
}
/********************************************************
Print crypt error message to error log */
UNIV_INTERN
void
log_crypt_print_error(
/*==================*/
log_crypt_err_t err_info) /*!< out: error info */
{
switch(err_info) {
case LOG_CRYPT_KEY_NOT_FOUND:
ib::error()
<< "Redo log crypto: getting mysqld crypto key "
<< "from key version failed. Reason could be that "
<< "requested key version is not found or required "
<< "encryption key management plugin is not found.";
break;
case LOG_DECRYPT_MAYBE_FAILED:
ib::error()
<< "Redo log crypto: failed to decrypt log block. "
<< "Reason could be that requested key version is "
<< "not found, required encryption key management "
<< "plugin is not found or configured encryption "
<< "algorithm and/or method does not match.";
break;
default:
ut_error; /* Real bug */
}
compile_time_assert(16 == sizeof info.crypt_msg);
compile_time_assert(LOG_CHECKPOINT_CRYPT_MESSAGE
- LOG_CHECKPOINT_CRYPT_NONCE
== sizeof info.crypt_nonce);
memcpy(info.crypt_msg.bytes, buf + LOG_CHECKPOINT_CRYPT_MESSAGE,
sizeof info.crypt_msg);
memcpy(info.crypt_nonce.bytes, buf + LOG_CHECKPOINT_CRYPT_NONCE,
sizeof info.crypt_nonce);
return init_crypt_key(&info);
}

120
storage/innobase/log/log0log.cc
View file

@ -40,6 +40,7 @@ Created 12/9/1995 Heikki Tuuri
#include "log0log.ic"
#endif
#include "log0crypt.h"
#include "mem0mem.h"
#include "buf0buf.h"
#include "buf0flu.h"
@ -56,9 +57,6 @@ Created 12/9/1995 Heikki Tuuri
#include "srv0mon.h"
#include "sync0sync.h"
/* Used for debugging */
// #define DEBUG_CRYPT 1
/*
General philosophy of InnoDB redo-logs:
@ -898,7 +896,9 @@ log_group_init(
group->id = id;
group->n_files = n_files;
group->format = LOG_HEADER_FORMAT_CURRENT;
group->format = srv_encrypt_log
? LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED
: LOG_HEADER_FORMAT_CURRENT;
group->file_size = file_size;
group->space_id = space_id;
group->state = LOG_GROUP_OK;
@ -987,11 +987,13 @@ log_group_file_header_flush(
ut_ad(!recv_no_log_write);
ut_ad(group->id == 0);
ut_a(nth_file < group->n_files);
ut_ad((group->format & ~LOG_HEADER_FORMAT_ENCRYPTED)
== LOG_HEADER_FORMAT_CURRENT);
buf = *(group->file_header_bufs + nth_file);
memset(buf, 0, OS_FILE_LOG_BLOCK_SIZE);
mach_write_to_4(buf + LOG_HEADER_FORMAT, LOG_HEADER_FORMAT_CURRENT);
mach_write_to_4(buf + LOG_HEADER_FORMAT, group->format);
mach_write_to_8(buf + LOG_HEADER_START_LSN, start_lsn);
strcpy(reinterpret_cast<char*>(buf) + LOG_HEADER_CREATOR,
LOG_HEADER_CREATOR_CURRENT);
@ -1115,6 +1117,10 @@ loop:
|| log_block_get_hdr_no(buf)
== log_block_convert_lsn_to_no(start_lsn));
if (log_sys->is_encrypted()) {
log_crypt(buf, write_len);
}
/* Calculate the checksums for each log block and write them to
the trailer fields of the log blocks */
@ -1135,8 +1141,6 @@ loop:
ut_a(next_offset / UNIV_PAGE_SIZE <= ULINT_MAX);
log_encrypt_before_write(log_sys->next_checkpoint_no,
buf, write_len);
const ulint page_no
= (ulint) (next_offset / univ_page_size.physical());
@ -1625,7 +1629,9 @@ log_group_checkpoint(
mach_write_to_8(buf + LOG_CHECKPOINT_NO, log_sys->next_checkpoint_no);
mach_write_to_8(buf + LOG_CHECKPOINT_LSN, log_sys->next_checkpoint_lsn);
log_crypt_write_checkpoint_buf(buf);
if (log_sys->is_encrypted()) {
log_crypt_write_checkpoint_buf(buf);
}
lsn_offset = log_group_calc_lsn_offset(log_sys->next_checkpoint_lsn,
group);
@ -1689,30 +1695,16 @@ log_group_header_read(
/** Write checkpoint info to the log header and invoke log_mutex_exit().
@param[in] sync whether to wait for the write to complete */
void
log_write_checkpoint_info(
bool sync)
log_write_checkpoint_info(bool sync)
{
log_group_t* group;
ut_ad(log_mutex_own());
ut_ad(!srv_read_only_mode);
if (!srv_read_only_mode) {
for (group = UT_LIST_GET_FIRST(log_sys->log_groups);
group;
group = UT_LIST_GET_NEXT(log_groups, group)) {
for (log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups);
group;
group = UT_LIST_GET_NEXT(log_groups, group)) {
log_group_checkpoint(group);
}
}
/* generate key version and key used to encrypt future blocks,
*
* NOTE: the +1 is as the next_checkpoint_no will be updated once
* the checkpoint info has been written and THEN blocks will be encrypted
* with new key
*/
if (srv_encrypt_log) {
log_crypt_set_ver_and_key(log_sys->next_checkpoint_no + 1);
log_group_checkpoint(group);
}
log_mutex_exit();
@ -1991,78 +1983,6 @@ loop:
}
}
/******************************************************//**
Reads a specified log segment to a buffer. */
void
log_group_read_log_seg(
/*===================*/
byte* buf, /*!< in: buffer where to read */
log_group_t* group, /*!< in: log group */
lsn_t start_lsn, /*!< in: read area start */
lsn_t end_lsn) /*!< in: read area end */
{
ulint len;
lsn_t source_offset;
ut_ad(log_mutex_own());
loop:
source_offset = log_group_calc_lsn_offset(start_lsn, group);
ut_a(end_lsn - start_lsn <= ULINT_MAX);
len = (ulint) (end_lsn - start_lsn);
ut_ad(len != 0);
if ((source_offset % group->file_size) + len > group->file_size) {
/* If the above condition is true then len (which is ulint)
is > the expression below, so the typecast is ok */
len = (ulint) (group->file_size -
(source_offset % group->file_size));
}
log_sys->n_log_ios++;
MONITOR_INC(MONITOR_LOG_IO);
ut_a(source_offset / UNIV_PAGE_SIZE <= ULINT_MAX);
const ulint page_no
= (ulint) (source_offset / univ_page_size.physical());
fil_io(IORequestLogRead, true,
page_id_t(group->space_id, page_no),
univ_page_size,
(ulint) (source_offset % univ_page_size.physical()),
len, buf, NULL);
#ifdef DEBUG_CRYPT
fprintf(stderr, "BEFORE DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx offset %lu\n",
log_block_get_hdr_no(buf),
log_block_get_checkpoint_no(buf),
log_block_calc_checksum(buf),
log_block_get_checksum(buf), source_offset);
#endif
log_decrypt_after_read(buf, len);
#ifdef DEBUG_CRYPT
fprintf(stderr, "AFTER DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx\n",
log_block_get_hdr_no(buf),
log_block_get_checkpoint_no(buf),
log_block_calc_checksum(buf),
log_block_get_checksum(buf));
#endif
start_lsn += len;
buf += len;
if (start_lsn != end_lsn) {
goto loop;
}
}
/**
Checks that there is enough free space in the log to start a new query step.
Flushes the log buffer or makes a new checkpoint if necessary. NOTE: this

316
storage/innobase/log/log0recv.cc
View file

@ -688,6 +688,99 @@ recv_sys_debug_free(void)
mutex_exit(&(recv_sys->mutex));
}
/** Read a log segment to a buffer.
@param[out] buf buffer
@param[in] group redo log files
@param[in] start_lsn read area start
@param[in] end_lsn read area end
@return valid end_lsn */
static
lsn_t
log_group_read_log_seg(
byte* buf,
const log_group_t* group,
lsn_t start_lsn,
lsn_t end_lsn)
{
ulint len;
lsn_t source_offset;
ut_ad(log_mutex_own());
loop:
source_offset = log_group_calc_lsn_offset(start_lsn, group);
ut_a(end_lsn - start_lsn <= ULINT_MAX);
len = (ulint) (end_lsn - start_lsn);
ut_ad(len != 0);
const bool at_eof = (source_offset % group->file_size) + len
> group->file_size;
if (at_eof) {
/* If the above condition is true then len (which is ulint)
is > the expression below, so the typecast is ok */
len = (ulint) (group->file_size -
(source_offset % group->file_size));
}
log_sys->n_log_ios++;
MONITOR_INC(MONITOR_LOG_IO);
ut_a(source_offset / UNIV_PAGE_SIZE <= ULINT_MAX);
const ulint page_no
= (ulint) (source_offset / univ_page_size.physical());
fil_io(IORequestLogRead, true,
page_id_t(group->space_id, page_no),
univ_page_size,
(ulint) (source_offset % univ_page_size.physical()),
len, buf, NULL);
for (ulint l = 0; l < len; l += OS_FILE_LOG_BLOCK_SIZE,
buf += OS_FILE_LOG_BLOCK_SIZE,
start_lsn += OS_FILE_LOG_BLOCK_SIZE) {
const ulint block_number = log_block_get_hdr_no(buf);
if (block_number != log_block_convert_lsn_to_no(start_lsn)) {
/* Garbage or an incompletely written log block.
We will not report any error, because this can
happen when InnoDB was killed while it was
writing redo log. We simply treat this as an
abrupt end of the redo log. */
return(start_lsn);
}
if (innodb_log_checksums || group->is_encrypted()) {
ulint crc = log_block_calc_checksum_crc32(buf);
ulint cksum = log_block_get_checksum(buf);
if (crc != cksum) {
ib::error() << "Invalid log block checksum."
<< " block: " << block_number
<< " checkpoint no: "
<< log_block_get_checkpoint_no(buf)
<< " expected: " << crc
<< " found: " << cksum;
return(start_lsn);
}
if (group->is_encrypted()) {
log_crypt(buf, OS_FILE_LOG_BLOCK_SIZE, true);
}
}
}
if (start_lsn != end_lsn) {
goto loop;
}
return(start_lsn);
}
/********************************************************//**
Copies a log segment from the most up-to-date log group to the other log
groups, so that they all contain the latest log data. Also writes the info
@ -695,8 +788,7 @@ about the latest checkpoint to the groups, and inits the fields in the group
memory structs to up-to-date values. */
static
void
recv_synchronize_groups(void)
/*=========================*/
recv_synchronize_groups()
{
const lsn_t recovered_lsn = recv_sys->recovered_lsn;
@ -705,14 +797,9 @@ recv_synchronize_groups(void)
const lsn_t start_lsn = ut_uint64_align_down(recovered_lsn,
OS_FILE_LOG_BLOCK_SIZE);
const lsn_t end_lsn = ut_uint64_align_up(recovered_lsn,
OS_FILE_LOG_BLOCK_SIZE);
ut_ad(start_lsn != end_lsn);
log_group_read_log_seg(log_sys->buf,
UT_LIST_GET_FIRST(log_sys->log_groups),
start_lsn, end_lsn);
start_lsn, start_lsn + OS_FILE_LOG_BLOCK_SIZE);
ut_ad(UT_LIST_GET_LEN(log_sys->log_groups) == 1);
@ -730,8 +817,10 @@ recv_synchronize_groups(void)
over the max checkpoint info, thus making the preservation of max
checkpoint info on disk certain */
log_write_checkpoint_info(true);
log_mutex_enter();
if (!srv_read_only_mode) {
log_write_checkpoint_info(true);
log_mutex_enter();
}
}
/** Check the consistency of a log header block.
@ -799,7 +888,7 @@ recv_find_max_checkpoint_0(
buf + LOG_CHECKPOINT_NO);
if (!log_crypt_101_read_checkpoint(buf)) {
ib::warn() << "Decrypting checkpoint failed";
ib::error() << "Decrypting checkpoint failed";
continue;
}
@ -930,6 +1019,7 @@ recv_find_max_checkpoint(
return(recv_find_max_checkpoint_0(
max_group, max_field));
case LOG_HEADER_FORMAT_CURRENT:
case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED:
break;
default:
/* Ensure that the string is NUL-terminated. */
@ -963,6 +1053,13 @@ recv_find_max_checkpoint(
continue;
}
if (group->is_encrypted()
&& !log_crypt_read_checkpoint_buf(buf)) {
ib::error() << "Reading checkpoint"
" encryption info failed.";
continue;
}
group->state = LOG_GROUP_OK;
group->lsn = mach_read_from_8(
@ -972,11 +1069,6 @@ recv_find_max_checkpoint(
checkpoint_no = mach_read_from_8(
buf + LOG_CHECKPOINT_NO);
if (!log_crypt_read_checkpoint_buf(buf)) {
ib::error() << "Reading checkpoint encryption info failed.";
return DB_ERROR;
}
DBUG_PRINT("ib_log",
("checkpoint " UINT64PF " at " LSN_PF
" found in group " ULINTPF,
@ -1008,28 +1100,6 @@ recv_find_max_checkpoint(
return(DB_SUCCESS);
}
/** Check the 4-byte checksum to the trailer checksum field of a log
block.
@param[in] block log block
@param[in] print_err whether to report checksum mismatch
@return whether the checksum matches */
bool
log_block_checksum_is_ok(const byte* block, bool print_err)
{
bool valid
= log_block_get_checksum(block) == log_block_calc_checksum(block);
if (!valid && print_err) {
ib::error() << "Invalid log block checksum."
<< " block: " << log_block_get_hdr_no(block)
<< " checkpoint no: " << log_block_get_checkpoint_no(block)
<< " expected: " << log_block_calc_checksum(block)
<< " found: " << log_block_get_checksum(block);
}
return(valid || !innodb_log_checksums);
}
/** Try to parse a single log record body and also applies it if
specified.
@param[in] type redo log entry type
@ -2300,8 +2370,7 @@ bool
recv_parse_log_recs(
lsn_t checkpoint_lsn,
store_t store,
bool apply,
dberr_t* err)
bool apply)
{
byte* ptr;
byte* end_ptr;
@ -2716,62 +2785,22 @@ recv_scan_log_recs(
lsn_t* contiguous_lsn, /*!< in/out: it is known that all log
groups contain contiguous log data up
to this lsn */
lsn_t* group_scanned_lsn,/*!< out: scanning succeeded up to
lsn_t* group_scanned_lsn)/*!< out: scanning succeeded up to
this lsn */
dberr_t* err) /*!< out: error code or DB_SUCCESS */
{
const byte* log_block = buf;
ulint no;
lsn_t scanned_lsn = start_lsn;
bool finished = false;
ulint data_len;
bool more_data = false;
bool apply = recv_sys->mlog_checkpoint_lsn != 0;
bool maybe_encrypted = false;
ut_ad(start_lsn % OS_FILE_LOG_BLOCK_SIZE == 0);
ut_ad(len % OS_FILE_LOG_BLOCK_SIZE == 0);
ut_ad(len >= OS_FILE_LOG_BLOCK_SIZE);
*err = DB_SUCCESS;
do {
ut_ad(!finished);
no = log_block_get_hdr_no(log_block);
ulint expected_no = log_block_convert_lsn_to_no(scanned_lsn);
if (no != expected_no) {
/* Garbage or an incompletely written log block.
We will not report any error, because this can
happen when InnoDB was killed while it was
writing redo log. We simply treat this as an
abrupt end of the redo log. */
finished = true;
break;
}
if (!log_block_checksum_is_ok(log_block, true)) {
log_crypt_err_t log_crypt_err;
maybe_encrypted = log_crypt_block_maybe_encrypted(log_block,
&log_crypt_err);
/* Print checkpoint encryption keys if present */
log_crypt_print_checkpoint_keys(log_block);
if (maybe_encrypted) {
/* Log block maybe encrypted finish processing*/
log_crypt_print_error(log_crypt_err);
*err = DB_ERROR;
return (TRUE);
}
/* Garbage or an incompletely written log block.
This could be the result of killing the server
while it was writing this log block. We treat
this as an abrupt end of the redo log. */
finished = true;
break;
}
if (log_block_get_flush_bit(log_block)) {
/* This block was a start of a log flush operation:
@ -2852,7 +2881,6 @@ recv_scan_log_recs(
ib::error()
<< "Set innodb_force_recovery"
" to ignore this error.";
*err = DB_ERROR;
return(true);
}
} else if (!recv_sys->found_corrupt_log) {
@ -2888,25 +2916,13 @@ recv_scan_log_recs(
if (more_data && !recv_sys->found_corrupt_log) {
/* Try to parse more log records */
bool parse_finished = false;
dberr_t parse_err = DB_SUCCESS;
parse_finished = recv_parse_log_recs(checkpoint_lsn,
*store_to_hash, apply,
&parse_err);
if (parse_err != DB_SUCCESS) {
ib::info() << "Parsing more log records failed checkpoint_lsn "
<< checkpoint_lsn << " error " << parse_err;
parse_finished = true;
}
if (parse_finished) {
if (recv_parse_log_recs(checkpoint_lsn,
*store_to_hash, apply)) {
ut_ad(recv_sys->found_corrupt_log
|| recv_sys->found_corrupt_fs
|| recv_sys->mlog_checkpoint_lsn
== recv_sys->recovered_lsn);
*err = parse_err;
return(true);
}
@ -2932,15 +2948,13 @@ Parses and hashes the log records if new data found.
until which all redo log has been scanned
@param[in] last_phase whether changes
can be applied to the tablespaces
@param[out] err DB_SUCCESS or error code
@return whether rescan is needed (not everything was stored) */
static
bool
recv_group_scan_log_recs(
log_group_t* group,
lsn_t* contiguous_lsn,
bool last_phase,
dberr_t* err)
bool last_phase)
{
DBUG_ENTER("recv_group_scan_log_recs");
DBUG_ASSERT(!last_phase || recv_sys->mlog_checkpoint_lsn > 0);
@ -2971,38 +2985,33 @@ recv_group_scan_log_recs(
* (buf_pool_get_n_pages()
- (recv_n_pool_free_frames * srv_buf_pool_instances));
*err = DB_SUCCESS;
end_lsn = *contiguous_lsn = ut_uint64_align_down(
group->scanned_lsn = end_lsn = *contiguous_lsn = ut_uint64_align_down(
*contiguous_lsn, OS_FILE_LOG_BLOCK_SIZE);
do {
if (*err != DB_SUCCESS) {
DBUG_RETURN(false);
}
if (last_phase && store_to_hash == STORE_NO) {
store_to_hash = STORE_IF_EXISTS;
/* We must not allow change buffer
merge here, because it would generate
redo log records before we have
finished the redo log scan. */
*err = recv_apply_hashed_log_recs(FALSE);
if (recv_apply_hashed_log_recs(FALSE) != DB_SUCCESS) {
DBUG_RETURN(false);
}
}
start_lsn = end_lsn;
end_lsn += RECV_SCAN_SIZE;
log_group_read_log_seg(
log_sys->buf, group, start_lsn, end_lsn);
} while (!recv_scan_log_recs(
end_lsn = log_group_read_log_seg(
log_sys->buf, group, start_lsn,
start_lsn + RECV_SCAN_SIZE);
} while (end_lsn != start_lsn
&& !recv_scan_log_recs(
available_mem, &store_to_hash, log_sys->buf,
RECV_SCAN_SIZE,
end_lsn - start_lsn,
checkpoint_lsn,
start_lsn, contiguous_lsn, &group->scanned_lsn, err));
start_lsn, contiguous_lsn, &group->scanned_lsn));
if (recv_sys->found_corrupt_log || recv_sys->found_corrupt_fs) {
ib::error() << "Found corrupted log when looking checkpoint lsn: "
<< contiguous_lsn << " error = " << *err;
DBUG_RETURN(false);
}
@ -3185,8 +3194,7 @@ recv_recovery_from_checkpoint_start(
if (srv_force_recovery >= SRV_FORCE_NO_LOG_REDO) {
ib::info() << "The user has set SRV_FORCE_NO_LOG_REDO on,"
" skipping log redo";
ib::info() << "innodb_force_recovery=6 skips redo log apply";
return(DB_SUCCESS);
}
@ -3231,6 +3239,7 @@ recv_recovery_from_checkpoint_start(
log_mutex_exit();
return(recv_log_format_0_recover(checkpoint_lsn));
case LOG_HEADER_FORMAT_CURRENT:
case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED:
break;
default:
ut_ad(0);
@ -3240,7 +3249,7 @@ recv_recovery_from_checkpoint_start(
}
/* Look for MLOG_CHECKPOINT. */
recv_group_scan_log_recs(group, &contiguous_lsn, false, &err);
recv_group_scan_log_recs(group, &contiguous_lsn, false);
/* The first scan should not have stored or applied any records. */
ut_ad(recv_sys->n_addrs == 0);
ut_ad(!recv_sys->found_corrupt_fs);
@ -3252,22 +3261,19 @@ recv_recovery_from_checkpoint_start(
if (recv_sys->found_corrupt_log && !srv_force_recovery) {
log_mutex_exit();
ib::error() << "Found corrupted log when looking checkpoint lsn: "
<< contiguous_lsn << " error = " << err;
ib::warn() << "Log scan aborted at LSN " << contiguous_lsn;
return(DB_ERROR);
}
if (recv_sys->mlog_checkpoint_lsn == 0) {
if (!srv_read_only_mode
&& group->scanned_lsn != checkpoint_lsn) {
ib::error() << "Ignoring the redo log due to missing"
ib::error() << "Missing"
" MLOG_CHECKPOINT between the checkpoint "
<< checkpoint_lsn << " and the end "
<< group->scanned_lsn << ".";
if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) {
log_mutex_exit();
return(DB_ERROR);
}
log_mutex_exit();
return(DB_ERROR);
}
group->scanned_lsn = checkpoint_lsn;
@ -3275,13 +3281,11 @@ recv_recovery_from_checkpoint_start(
} else {
contiguous_lsn = checkpoint_lsn;
rescan = recv_group_scan_log_recs(
group, &contiguous_lsn, false, &err);
group, &contiguous_lsn, false);
if ((recv_sys->found_corrupt_log && !srv_force_recovery)
|| recv_sys->found_corrupt_fs) {
log_mutex_exit();
ib::error() << "Found corrupted log in lsn:"
<< contiguous_lsn << " err = " << err;
return(DB_ERROR);
}
}
@ -3332,15 +3336,12 @@ recv_recovery_from_checkpoint_start(
if (rescan) {
contiguous_lsn = checkpoint_lsn;
recv_group_scan_log_recs(group, &contiguous_lsn, true, &err);
recv_group_scan_log_recs(group, &contiguous_lsn, true);
if ((recv_sys->found_corrupt_log
&& !srv_force_recovery)
|| recv_sys->found_corrupt_fs) {
log_mutex_exit();
ib::error() << "Found corrupted log in lsn:"
<< contiguous_lsn << " err = " << err;
return(DB_ERROR);
}
}
@ -3380,10 +3381,6 @@ recv_recovery_from_checkpoint_start(
log_sys->next_checkpoint_lsn = checkpoint_lsn;
log_sys->next_checkpoint_no = checkpoint_no + 1;
/* here the checkpoint info is written without any redo logging ongoing
* and next_checkpoint_no is updated directly hence no +1 */
log_crypt_set_ver_and_key(log_sys->next_checkpoint_no);
recv_synchronize_groups();
if (!recv_needed_recovery) {
@ -3407,8 +3404,7 @@ recv_recovery_from_checkpoint_start(
MONITOR_SET(MONITOR_LSN_CHECKPOINT_AGE,
log_sys->lsn - log_sys->last_checkpoint_lsn);
log_sys->next_checkpoint_no = checkpoint_no + 1;
log_crypt_set_ver_and_key(log_sys->next_checkpoint_no);
log_sys->next_checkpoint_no = ++checkpoint_no;
mutex_enter(&recv_sys->mutex);
@ -3462,42 +3458,6 @@ recv_recovery_from_checkpoint_finish(void)
/* Free up the flush_rbt. */
buf_flush_free_flush_rbt();
/* Validate a few system page types that were left uninitialized
by older versions of MySQL. */
mtr_t mtr;
buf_block_t* block;
mtr.start();
mtr.set_sys_modified();
/* Bitmap page types will be reset in buf_dblwr_check_block()
without redo logging. */
block = buf_page_get(
page_id_t(IBUF_SPACE_ID, FSP_IBUF_HEADER_PAGE_NO),
univ_page_size, RW_X_LATCH, &mtr);
fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr);
/* Already MySQL 3.23.53 initialized FSP_IBUF_TREE_ROOT_PAGE_NO
to FIL_PAGE_INDEX. No need to reset that one. */
block = buf_page_get(
page_id_t(TRX_SYS_SPACE, TRX_SYS_PAGE_NO),
univ_page_size, RW_X_LATCH, &mtr);
fil_block_check_type(block, FIL_PAGE_TYPE_TRX_SYS, &mtr);
block = buf_page_get(
page_id_t(TRX_SYS_SPACE, FSP_FIRST_RSEG_PAGE_NO),
univ_page_size, RW_X_LATCH, &mtr);
fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr);
block = buf_page_get(
page_id_t(TRX_SYS_SPACE, FSP_DICT_HDR_PAGE_NO),
univ_page_size, RW_X_LATCH, &mtr);
fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr);
mtr.commit();
/* Roll back any recovered data dictionary transactions, so
that the data dictionary tables will be free of any locks.
The data dictionary latch should guarantee that there is at
most one data dictionary transaction active at a time. */
if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO) {
trx_rollback_or_clean_recovered(FALSE);
}
}
/********************************************************//**
@ -3546,18 +3506,14 @@ recv_reset_logs(
which we add
LOG_BLOCK_HDR_SIZE */
{
log_group_t* group;
ut_ad(log_mutex_own());
log_sys->lsn = ut_uint64_align_up(lsn, OS_FILE_LOG_BLOCK_SIZE);
group = UT_LIST_GET_FIRST(log_sys->log_groups);
while (group) {
for (log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups);
group; group = UT_LIST_GET_NEXT(log_groups, group)) {
group->lsn = log_sys->lsn;
group->lsn_offset = LOG_FILE_HDR_SIZE;
group = UT_LIST_GET_NEXT(log_groups, group);
}
log_sys->buf_next_to_write = 0;

4
storage/innobase/mysql-test/storage_engine/define_engine.inc
View file

@ -41,9 +41,5 @@ let $default_char_type = CHAR(8);
# e.g. creation of an additional schema or table, etc.
# The cleanup part should be defined in cleanup_engine.inc
CALL mtr.add_suppression("InnoDB: Resizing redo log from .* to .* pages, LSN=.*");
CALL mtr.add_suppression("InnoDB: Starting to delete and rewrite log files.");
CALL mtr.add_suppression("InnoDB: New log files created, LSN=.*");
--enable_query_log
--enable_result_log

223
storage/innobase/srv/srv0start.cc
View file

@ -63,7 +63,7 @@ Created 2/16/1996 Heikki Tuuri
#include "fsp0fsp.h"
#include "rem0rec.h"
#include "mtr0mtr.h"
#include "log0log.h"
#include "log0crypt.h"
#include "log0recv.h"
#include "page0page.h"
#include "page0cur.h"
@ -481,6 +481,9 @@ create_log_files(
/* Create a log checkpoint. */
log_mutex_enter();
if (log_sys->is_encrypted() && !log_crypt_init()) {
return(DB_ERROR);
}
ut_d(recv_no_log_write = false);
recv_reset_logs(lsn);
log_mutex_exit();
@ -536,7 +539,7 @@ create_log_files_rename(
fil_open_log_and_system_tablespace_files();
ib::warn() << "New log files created, LSN=" << lsn;
ib::info() << "New log files created, LSN=" << lsn;
return(err);
}
@ -1362,17 +1365,35 @@ srv_prepare_to_delete_redo_log_files(
flushed_lsn = log_sys->lsn;
{
ib::warn warning;
ib::info info;
if (srv_log_file_size == 0) {
warning << "Upgrading redo log: ";
info << "Upgrading redo log: ";
} else if (n_files != srv_n_log_files
|| srv_log_file_size
!= srv_log_file_size_requested) {
if (srv_encrypt_log
== log_sys->is_encrypted()) {
info << (srv_encrypt_log
? "Resizing encrypted"
: "Resizing");
} else if (srv_encrypt_log) {
info << "Encrypting and resizing";
} else {
info << "Removing encryption"
" and resizing";
}
info << " redo log from " << n_files
<< "*" << srv_log_file_size << " to ";
} else if (srv_encrypt_log) {
info << "Encrypting redo log: ";
} else {
warning << "Resizing redo log from "
<< n_files << "*"
<< srv_log_file_size << " to ";
info << "Removing redo log encryption: ";
}
warning << srv_n_log_files << "*"
<< srv_log_file_size_requested
<< " pages, LSN=" << flushed_lsn;
info << srv_n_log_files << "*"
<< srv_log_file_size_requested
<< " pages; LSN=" << flushed_lsn;
}
/* Flush the old log files. */
@ -2180,6 +2201,14 @@ files_checked:
recv_sys->dblwr.pages.clear();
if (err == DB_SUCCESS && !srv_read_only_mode) {
log_mutex_enter();
if (log_sys->is_encrypted() && !log_crypt_init()) {
err = DB_ERROR;
}
log_mutex_exit();
}
if (err == DB_SUCCESS) {
/* Initialize the change buffer. */
err = dict_boot();
@ -2291,6 +2320,115 @@ files_checked:
recv_recovery_from_checkpoint_finish();
/* Upgrade or resize or rebuild the redo logs before
generating any dirty pages, so that the old redo log
files will not be written to. */
if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) {
/* Completely ignore the redo log. */
} else if (srv_read_only_mode) {
/* Leave the redo log alone. */
} else if (srv_log_file_size_requested == srv_log_file_size
&& srv_n_log_files_found == srv_n_log_files
&& log_sys->is_encrypted() == srv_encrypt_log) {
/* No need to upgrade or resize the redo log. */
} else {
/* Prepare to delete the old redo log files */
flushed_lsn = srv_prepare_to_delete_redo_log_files(i);
DBUG_EXECUTE_IF("innodb_log_abort_1",
return(srv_init_abort(DB_ERROR)););
/* Prohibit redo log writes from any other
threads until creating a log checkpoint at the
end of create_log_files(). */
ut_d(recv_no_log_write = true);
ut_ad(!buf_pool_check_no_pending_io());
DBUG_EXECUTE_IF("innodb_log_abort_3",
return(srv_init_abort(DB_ERROR)););
/* Stamp the LSN to the data files. */
err = fil_write_flushed_lsn(flushed_lsn);
DBUG_EXECUTE_IF("innodb_log_abort_4", err = DB_ERROR;);
if (err != DB_SUCCESS) {
return(srv_init_abort(err));
}
/* Close and free the redo log files, so that
we can replace them. */
fil_close_log_files(true);
DBUG_EXECUTE_IF("innodb_log_abort_5",
return(srv_init_abort(DB_ERROR)););
/* Free the old log file space. */
log_group_close_all();
ib::info() << "Starting to delete and rewrite log"
" files.";
srv_log_file_size = srv_log_file_size_requested;
err = create_log_files(
logfilename, dirnamelen, flushed_lsn,
logfile0);
if (err == DB_SUCCESS) {
err = create_log_files_rename(
logfilename, dirnamelen, flushed_lsn,
logfile0);
}
if (err != DB_SUCCESS) {
return(srv_init_abort(err));
}
}
/* Validate a few system page types that were left
uninitialized by older versions of MySQL. */
if (!high_level_read_only) {
mtr_t mtr;
buf_block_t* block;
mtr.start();
mtr.set_sys_modified();
/* Bitmap page types will be reset in
buf_dblwr_check_block() without redo logging. */
block = buf_page_get(
page_id_t(IBUF_SPACE_ID,
FSP_IBUF_HEADER_PAGE_NO),
univ_page_size, RW_X_LATCH, &mtr);
fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr);
/* Already MySQL 3.23.53 initialized
FSP_IBUF_TREE_ROOT_PAGE_NO to
FIL_PAGE_INDEX. No need to reset that one. */
block = buf_page_get(
page_id_t(TRX_SYS_SPACE, TRX_SYS_PAGE_NO),
univ_page_size, RW_X_LATCH, &mtr);
fil_block_check_type(block, FIL_PAGE_TYPE_TRX_SYS,
&mtr);
block = buf_page_get(
page_id_t(TRX_SYS_SPACE,
FSP_FIRST_RSEG_PAGE_NO),
univ_page_size, RW_X_LATCH, &mtr);
fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr);
block = buf_page_get(
page_id_t(TRX_SYS_SPACE, FSP_DICT_HDR_PAGE_NO),
univ_page_size, RW_X_LATCH, &mtr);
fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr);
mtr.commit();
}
/* Roll back any recovered data dictionary transactions, so
that the data dictionary tables will be free of any locks.
The data dictionary latch should guarantee that there is at
most one data dictionary transaction active at a time. */
if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO) {
trx_rollback_or_clean_recovered(FALSE);
}
/* Fix-up truncate of tables in the system tablespace
if server crashed while truncate was active. The non-
system tables are done after tablespace discovery. Do
@ -2344,71 +2482,6 @@ files_checked:
return(srv_init_abort(err));
}
if (!srv_force_recovery
&& !recv_sys->found_corrupt_log
&& (srv_log_file_size_requested != srv_log_file_size
|| srv_n_log_files_found != srv_n_log_files)) {
/* Prepare to replace the redo log files. */
if (srv_read_only_mode) {
ib::error() << "Cannot resize log files"
" in read-only mode.";
return(srv_init_abort(DB_READ_ONLY));
}
/* Prepare to delete the old redo log files */
flushed_lsn = srv_prepare_to_delete_redo_log_files(i);
DBUG_EXECUTE_IF("innodb_log_abort_1",
return(srv_init_abort(DB_ERROR)););
/* Prohibit redo log writes from any other
threads until creating a log checkpoint at the
end of create_log_files(). */
ut_d(recv_no_log_write = true);
ut_ad(!buf_pool_check_no_pending_io());
DBUG_EXECUTE_IF("innodb_log_abort_3",
return(srv_init_abort(DB_ERROR)););
/* Stamp the LSN to the data files. */
err = fil_write_flushed_lsn(flushed_lsn);
DBUG_EXECUTE_IF("innodb_log_abort_4", err = DB_ERROR;);
if (err != DB_SUCCESS) {
return(srv_init_abort(err));
}
/* Close and free the redo log files, so that
we can replace them. */
fil_close_log_files(true);
DBUG_EXECUTE_IF("innodb_log_abort_5",
return(srv_init_abort(DB_ERROR)););
/* Free the old log file space. */
log_group_close_all();
ib::warn() << "Starting to delete and rewrite log"
" files.";
srv_log_file_size = srv_log_file_size_requested;
err = create_log_files(
logfilename, dirnamelen, flushed_lsn,
logfile0);
if (err == DB_SUCCESS) {
err = create_log_files_rename(
logfilename, dirnamelen, flushed_lsn,
logfile0);
}
if (err != DB_SUCCESS) {
return(srv_init_abort(err));
}
}
recv_recovery_rollback_active();
/* It is possible that file_format tag has never