mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2026-05-16 20:07:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

MDEV-8708: InnoDB temp file encryption

Browse source 
Added encryption support for online alter table where InnoDB temporary
files are used. Added similar support also for tables containing
full text-indexes.

Made sure that table remains encrypted during discard and import
tablespace.
This commit is contained in:
Jan Lindström 2015-09-04 15:54:20 +03:00
commit 509b836623
22 changed files with 1739 additions and 254 deletions
Download patch file Download diff file Expand all files Collapse all files

136
mysql-test/suite/encryption/r/innodb_encryption_discard_import.result Normal file
View file

@ -0,0 +1,136 @@
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB;
CREATE TABLE t3 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values (current_num,repeat('foobar',42));
insert into t2 values (current_num,repeat('temp', 42));
insert into t3 values (current_num,repeat('barfoo',42));
set current_num = current_num + 1;
end while;
end//
commit;
set autocommit=0;
call innodb_insert_proc(10000);
commit;
set autocommit=1;
# Wait max 10 min for key encryption threads to encrypt all spaces
# tablespaces should be now encrypted
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
FLUSH TABLE t1, t2, t3 FOR EXPORT;
# List before copying files
t1.cfg
t1.frm
t1.ibd
t2.cfg
t2.frm
t2.ibd
t3.cfg
t3.frm
t3.ibd
UNLOCK TABLES;
# Restarting server
# Done restarting server
# List before t1 DISCARD
t1.frm
t1.ibd
t2.frm
t2.ibd
t3.frm
t3.ibd
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
ALTER TABLE t1 DISCARD TABLESPACE;
ALTER TABLE t2 DISCARD TABLESPACE;
ALTER TABLE t3 DISCARD TABLESPACE;
# Discarded tablespaces should be encrypted
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# List after t1 DISCARD
t1.frm
t2.frm
t3.frm
# Tablespaces should be still encrypted
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
ALTER TABLE t1 IMPORT TABLESPACE;
ALTER TABLE t2 IMPORT TABLESPACE;
ALTER TABLE t3 IMPORT TABLESPACE;
# tablespaces should remain encrypted after import
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
ALTER TABLE t1 ENGINE InnoDB;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes
ALTER TABLE t2 ENGINE InnoDB;
SHOW CREATE TABLE t2;
Table Create Table
t2 CREATE TABLE `t2` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
ALTER TABLE t3 ENGINE InnoDB;
SHOW CREATE TABLE t3;
Table Create Table
t3 CREATE TABLE `t3` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `encrypted`=yes
# Wait max 10 min for key encryption threads to encrypt all spaces
# Tablespaces should be encrypted after alter table
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# Restarting server
# Done restarting server
# Verify that tables are still usable
SELECT COUNT(1) FROM t1;
COUNT(1)
10000
SELECT COUNT(1) FROM t2;
COUNT(1)
10000
SELECT COUNT(1) FROM t3;
COUNT(1)
10000
# Tablespaces should be encrypted after restart
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3;

176
mysql-test/suite/encryption/r/innodb_onlinealter_encryption.result Normal file
View file

@ -0,0 +1,176 @@
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB;
CREATE TABLE t3 (id INT, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t4 (id INT, a VARCHAR(255)) engine=InnoDB;
CREATE TABLE t5 (id INT NOT NULL PRIMARY KEY, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t6 (id INT, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB;
CREATE TABLE t7 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values (current_num,repeat('foobar',42));
insert into t2 values (current_num,repeat('temp', 42));
insert into t3 values (current_num,repeat('barfoo',42));
insert into t4 values (current_num,repeat('repeat',42));
insert into t5 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('author new',22));
insert into t6 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('mangled old',22));
insert into t7 values (current_num,repeat('mysql',42));
set current_num = current_num + 1;
end while;
end//
commit;
set autocommit=0;
call innodb_insert_proc(15000);
commit;
set autocommit=1;
# Wait max 10 min for key encryption threads to encrypt all spaces
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# t4 ... on expecting NOT FOUND
NOT FOUND /repeat/ in t4.ibd
# t5 ... on expecting NOT FOUND
NOT FOUND /author/ in t5.ibd
# t6 ... on expecting NOT FOUND
NOT FOUND /mangled/ in t6.ibd
# t7 ... on expecting NOT FOUND
NOT FOUND /mysql/ in t7.ibd
ALTER TABLE t1 ADD COLUMN b int default 2;
ALTER TABLE t2 ADD COLUMN b int default 2;
ALTER TABLE t7 ADD COLUMN b int default 2;
ALTER TABLE t1 ADD KEY a(a), ADD KEY b(b);
ALTER TABLE t2 ADD KEY a(a), ADD KEY b(b);
ALTER TABLE t3 ADD COLUMN c int default 5;
ALTER TABLE t4 ADD COLUMN c int default 5;
ALTER TABLE t3 ADD KEY (a), ADD KEY c(c);
ALTER TABLE t4 ADD KEY (a), ADD KEY c(c);
ALTER TABLE t5 ADD FULLTEXT(a);
ALTER TABLE t6 ADD FULLTEXT(a);
ALTER TABLE t7 ADD KEY a(a), ADD key b(b);
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
`b` int(11) DEFAULT '2',
PRIMARY KEY (`id`),
KEY `a` (`a`),
KEY `b` (`b`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes
SHOW CREATE TABLE t2;
Table Create Table
t2 CREATE TABLE `t2` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
`b` int(11) DEFAULT '2',
PRIMARY KEY (`id`),
KEY `a` (`a`),
KEY `b` (`b`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
SHOW CREATE TABLE t3;
Table Create Table
t3 CREATE TABLE `t3` (
`id` int(11) DEFAULT NULL,
`a` varchar(255) DEFAULT NULL,
`c` int(11) DEFAULT '5',
KEY `a` (`a`),
KEY `c` (`c`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes
SHOW CREATE TABLE t4;
Table Create Table
t4 CREATE TABLE `t4` (
`id` int(11) DEFAULT NULL,
`a` varchar(255) DEFAULT NULL,
`c` int(11) DEFAULT '5',
KEY `a` (`a`),
KEY `c` (`c`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
SHOW CREATE TABLE t5;
Table Create Table
t5 CREATE TABLE `t5` (
`id` int(11) NOT NULL,
`a` text,
`b` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
FULLTEXT KEY `b` (`b`),
FULLTEXT KEY `a` (`a`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes
SHOW CREATE TABLE t6;
Table Create Table
t6 CREATE TABLE `t6` (
`id` int(11) DEFAULT NULL,
`a` text,
`b` varchar(255) DEFAULT NULL,
FULLTEXT KEY `b` (`b`),
FULLTEXT KEY `a` (`a`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
SHOW CREATE TABLE t7;
Table Create Table
t7 CREATE TABLE `t7` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
`b` int(11) DEFAULT '2',
PRIMARY KEY (`id`),
KEY `a` (`a`),
KEY `b` (`b`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `encrypted`=yes
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# t4 ... on expecting NOT FOUND
NOT FOUND /repeat/ in t4.ibd
# t5 ... on expecting NOT FOUND
NOT FOUND /author/ in t5.ibd
# t6 ... on expecting NOT FOUND
NOT FOUND /mangled/ in t6.ibd
# t7 ... on expecting NOT FOUND
NOT FOUND /mysql/ in t7.ibd
# Restarting server
# Done restarting server
select count(1) from t1;
count(1)
15000
select count(1) from t2;
count(1)
15000
select count(1) from t3;
count(1)
15000
select count(1) from t4;
count(1)
15000
select count(1) from t5;
count(1)
15000
select count(1) from t6;
count(1)
15000
select count(1) from t7;
count(1)
15000
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# t4 ... on expecting NOT FOUND
NOT FOUND /repeat/ in t4.ibd
# t5 ... on expecting NOT FOUND
NOT FOUND /author/ in t5.ibd
# t6 ... on expecting NOT FOUND
NOT FOUND /mangled/ in t6.ibd
# t7 ... on expecting NOT FOUND
NOT FOUND /mysql/ in t7.ibd
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3, t4, t5, t6, t7;

8
mysql-test/suite/encryption/t/innodb_encryption_discard_import.opt Normal file
View file

@ -0,0 +1,8 @@
--innodb-encrypt-tables=ON
--innodb-encrypt-log=ON
--innodb-encryption-rotate-key-age=15
--innodb-encryption-threads=4
--innodb-tablespaces-encryption
--innodb-max-dirty-pages-pct=0.001

211
mysql-test/suite/encryption/t/innodb_encryption_discard_import.test Normal file
View file

@ -0,0 +1,211 @@
-- source include/have_innodb.inc
-- source include/have_example_key_management_plugin.inc
-- source include/not_valgrind.inc
-- source include/not_embedded.inc
-- source include/not_windows.inc
--let $MYSQLD_TMPDIR = `SELECT @@tmpdir`
--let $MYSQLD_DATADIR = `SELECT @@datadir`
--let SEARCH_RANGE = 10000000
--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd
--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd
--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd
--let t1_IBD_1 = $MYSQLD_TMPDIR/t1.ibd
--let t2_IBD_1 = $MYSQLD_TMPDIR/t2.ibd
--let t3_IBD_1 = $MYSQLD_TMPDIR/t3.ibd
--disable_query_log
let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
--enable_query_log
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB;
CREATE TABLE t3 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes;
delimiter //;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values (current_num,repeat('foobar',42));
insert into t2 values (current_num,repeat('temp', 42));
insert into t3 values (current_num,repeat('barfoo',42));
set current_num = current_num + 1;
end while;
end//
delimiter ;//
commit;
set autocommit=0;
call innodb_insert_proc(10000);
commit;
set autocommit=1;
--echo # Wait max 10 min for key encryption threads to encrypt all spaces
--let $wait_timeout= 600
--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0
--source include/wait_condition.inc
--sleep 5
--echo # tablespaces should be now encrypted
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
FLUSH TABLE t1, t2, t3 FOR EXPORT;
--echo # List before copying files
--list_files $MYSQLD_DATADIR/test
--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_TMPDIR/t1.cfg
--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_TMPDIR/t1.ibd
--copy_file $MYSQLD_DATADIR/test/t2.cfg $MYSQLD_TMPDIR/t2.cfg
--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_TMPDIR/t2.ibd
--copy_file $MYSQLD_DATADIR/test/t3.cfg $MYSQLD_TMPDIR/t3.cfg
--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_TMPDIR/t3.ibd
UNLOCK TABLES;
--echo # Restarting server
-- source include/restart_mysqld.inc
--echo # Done restarting server
--echo # List before t1 DISCARD
--list_files $MYSQLD_DATADIR/test
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
ALTER TABLE t1 DISCARD TABLESPACE;
ALTER TABLE t2 DISCARD TABLESPACE;
ALTER TABLE t3 DISCARD TABLESPACE;
--sleep 5
--echo # Discarded tablespaces should be encrypted
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD_1
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD_1
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD_1
-- source include/search_pattern_in_file.inc
--echo # List after t1 DISCARD
--list_files $MYSQLD_DATADIR/test
--copy_file $MYSQLD_TMPDIR/t1.cfg $MYSQLD_DATADIR/test/t1.cfg
--copy_file $MYSQLD_TMPDIR/t1.ibd $MYSQLD_DATADIR/test/t1.ibd
--copy_file $MYSQLD_TMPDIR/t2.cfg $MYSQLD_DATADIR/test/t2.cfg
--copy_file $MYSQLD_TMPDIR/t2.ibd $MYSQLD_DATADIR/test/t2.ibd
--copy_file $MYSQLD_TMPDIR/t3.cfg $MYSQLD_DATADIR/test/t3.cfg
--copy_file $MYSQLD_TMPDIR/t3.ibd $MYSQLD_DATADIR/test/t3.ibd
--sleep 5
--echo # Tablespaces should be still encrypted
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
ALTER TABLE t1 IMPORT TABLESPACE;
ALTER TABLE t2 IMPORT TABLESPACE;
ALTER TABLE t3 IMPORT TABLESPACE;
--sleep 5
--echo # tablespaces should remain encrypted after import
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
ALTER TABLE t1 ENGINE InnoDB;
SHOW CREATE TABLE t1;
ALTER TABLE t2 ENGINE InnoDB;
SHOW CREATE TABLE t2;
ALTER TABLE t3 ENGINE InnoDB;
SHOW CREATE TABLE t3;
--echo # Wait max 10 min for key encryption threads to encrypt all spaces
--let $wait_timeout= 600
--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0
--source include/wait_condition.inc
--sleep 5
--echo # Tablespaces should be encrypted after alter table
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--echo # Restarting server
-- source include/restart_mysqld.inc
--echo # Done restarting server
--echo # Verify that tables are still usable
SELECT COUNT(1) FROM t1;
SELECT COUNT(1) FROM t2;
SELECT COUNT(1) FROM t3;
--sleep 5
--echo # Tablespaces should be encrypted after restart
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3;
# reset system
--disable_query_log
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
--enable_query_log

8
mysql-test/suite/encryption/t/innodb_onlinealter_encryption.opt Normal file
View file

@ -0,0 +1,8 @@
--innodb-encrypt-tables=ON
--innodb-encrypt-log=ON
--innodb-encryption-rotate-key-age=15
--innodb-encryption-threads=4
--innodb-tablespaces-encryption
--innodb-max-dirty-pages-pct=0.001

191
mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test Normal file
View file

@ -0,0 +1,191 @@
-- source include/have_innodb.inc
-- source include/have_example_key_management_plugin.inc
-- source include/not_valgrind.inc
-- source include/not_embedded.inc
-- source include/not_windows.inc
--disable_query_log
let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
--enable_query_log
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
--let $MYSQLD_DATADIR=`select @@datadir`
--let SEARCH_RANGE = 10000000
--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd
--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd
--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd
--let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd
--let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd
--let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd
--let t7_IBD = $MYSQLD_DATADIR/test/t7.ibd
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB;
CREATE TABLE t3 (id INT, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t4 (id INT, a VARCHAR(255)) engine=InnoDB;
CREATE TABLE t5 (id INT NOT NULL PRIMARY KEY, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t6 (id INT, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB;
CREATE TABLE t7 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes;
delimiter //;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values (current_num,repeat('foobar',42));
insert into t2 values (current_num,repeat('temp', 42));
insert into t3 values (current_num,repeat('barfoo',42));
insert into t4 values (current_num,repeat('repeat',42));
insert into t5 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('author new',22));
insert into t6 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('mangled old',22));
insert into t7 values (current_num,repeat('mysql',42));
set current_num = current_num + 1;
end while;
end//
delimiter ;//
commit;
set autocommit=0;
call innodb_insert_proc(15000);
commit;
set autocommit=1;
--echo # Wait max 10 min for key encryption threads to encrypt all spaces
--let $wait_timeout= 600
--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0
--source include/wait_condition.inc
--sleep 5
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=repeat
--echo # t4 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=author
--echo # t5 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t5_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mangled
--echo # t6 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t6_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t7 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t7_IBD
-- source include/search_pattern_in_file.inc
ALTER TABLE t1 ADD COLUMN b int default 2;
ALTER TABLE t2 ADD COLUMN b int default 2;
ALTER TABLE t7 ADD COLUMN b int default 2;
ALTER TABLE t1 ADD KEY a(a), ADD KEY b(b);
ALTER TABLE t2 ADD KEY a(a), ADD KEY b(b);
ALTER TABLE t3 ADD COLUMN c int default 5;
ALTER TABLE t4 ADD COLUMN c int default 5;
ALTER TABLE t3 ADD KEY (a), ADD KEY c(c);
ALTER TABLE t4 ADD KEY (a), ADD KEY c(c);
ALTER TABLE t5 ADD FULLTEXT(a);
ALTER TABLE t6 ADD FULLTEXT(a);
ALTER TABLE t7 ADD KEY a(a), ADD key b(b);
SHOW CREATE TABLE t1;
SHOW CREATE TABLE t2;
SHOW CREATE TABLE t3;
SHOW CREATE TABLE t4;
SHOW CREATE TABLE t5;
SHOW CREATE TABLE t6;
SHOW CREATE TABLE t7;
--sleep 5
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=repeat
--echo # t4 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=author
--echo # t5 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t5_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mangled
--echo # t6 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t6_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t7 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t7_IBD
-- source include/search_pattern_in_file.inc
--echo # Restarting server
-- source include/restart_mysqld.inc
--echo # Done restarting server
select count(1) from t1;
select count(1) from t2;
select count(1) from t3;
select count(1) from t4;
select count(1) from t5;
select count(1) from t6;
select count(1) from t7;
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=repeat
--echo # t4 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=author
--echo # t5 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t5_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mangled
--echo # t6 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t6_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t7 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t7_IBD
-- source include/search_pattern_in_file.inc
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3, t4, t5, t6, t7;
# reset system
--disable_query_log
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
--enable_query_log

83
storage/innobase/fil/fil0crypt.cc
View file

@ -244,9 +244,6 @@ fil_space_merge_crypt_data(
ut_a(dst->type == CRYPT_SCHEME_UNENCRYPTED ||
dst->type == CRYPT_SCHEME_1);
/* no support for changing iv (yet?) */
ut_a(memcmp(src->iv, dst->iv, sizeof(src->iv)) == 0);
dst->encryption = src->encryption;
dst->type = src->type;
dst->min_key_version = src->min_key_version;
@ -287,7 +284,7 @@ fil_space_read_crypt_data(
page[offset + 4],
page[offset + 5]);
#endif
/* Create data is not stored. */
/* Crypt data is not stored. */
return NULL;
}
@ -370,7 +367,7 @@ fil_space_destroy_crypt_data(
mutex_enter(&(*crypt_data)->mutex);
(*crypt_data)->inited = false;
mutex_exit(&(*crypt_data)->mutex);
mutex_free(&(*crypt_data)->mutex);
mutex_free(& (*crypt_data)->mutex);
memset(*crypt_data, 0, sizeof(fil_space_crypt_t));
free(*crypt_data);
(*crypt_data) = NULL;
@ -555,42 +552,22 @@ fil_space_clear_crypt_data(
}
/******************************************************************
Encrypt a page */
Encrypt a buffer */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
row format compressed */
byte* dst_frame) /*!< in: outbut buffer */
{
fil_space_crypt_t* crypt_data = NULL;
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
uint key_version;
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR
|| orig_page_type==FIL_PAGE_TYPE_XDES) {
/* File space header or extent descriptor do not need to be
encrypted. */
return src_frame;
}
/* Get crypt data from file space */
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data == NULL) {
return src_frame;
}
ut_ad(crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF);
key_version = fil_crypt_get_latest_key_version(crypt_data);
uint key_version = fil_crypt_get_latest_key_version(crypt_data);
if (key_version == ENCRYPTION_KEY_VERSION_INVALID) {
ib_logf(IB_LOG_LEVEL_FATAL,
@ -599,6 +576,7 @@ fil_space_encrypt(
ut_error;
}
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
ibool page_compressed = (orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint header_len = FIL_PAGE_DATA;
@ -661,6 +639,45 @@ fil_space_encrypt(
return dst_frame;
}
/******************************************************************
Encrypt a page */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame) /*!< in: outbut buffer */
{
fil_space_crypt_t* crypt_data = NULL;
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR
|| orig_page_type==FIL_PAGE_TYPE_XDES) {
/* File space header or extent descriptor do not need to be
encrypted. */
return src_frame;
}
/* Get crypt data from file space */
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data == NULL) {
return src_frame;
}
ut_ad(crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF);
byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame);
return tmp;
}
/*********************************************************************
Check if extra buffer shall be allocated for decrypting after read
@return true if fil space has encryption data. */
@ -1075,7 +1092,7 @@ fil_crypt_start_encrypting_space(
do
{
if (fil_crypt_is_closing(space) ||
fil_space_found_by_id(space)) {
fil_space_found_by_id(space) == NULL) {
break;
}
@ -2326,11 +2343,11 @@ fil_crypt_threads_init()
fil_crypt_threads_event = os_event_create();
mutex_create(fil_crypt_threads_mutex_key,
&fil_crypt_threads_mutex, SYNC_NO_ORDER_CHECK);
fil_crypt_threads_inited = true;
uint cnt = srv_n_fil_crypt_threads;
srv_n_fil_crypt_threads = 0;
fil_crypt_set_thread_cnt(cnt);
fil_crypt_threads_inited = true;
}
/*********************************************************************

69
storage/innobase/fil/fil0fil.cc
View file

@ -6418,8 +6418,16 @@ fil_iterate(
ut_ad(!(n_bytes % iter.page_size));
byte* readptr = io_buffer;
if (iter.crypt_data != NULL) {
byte* writeptr = io_buffer;
bool encrypted = false;
/* Use additional crypt io buffer if tablespace is encrypted */
if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
encrypted = true;
readptr = iter.crypt_io_buffer;
writeptr = iter.crypt_io_buffer;
}
if (!os_file_read(iter.file, readptr, offset, (ulint) n_bytes)) {
@ -6432,12 +6440,15 @@ fil_iterate(
bool updated = false;
os_offset_t page_off = offset;
ulint n_pages_read = (ulint) n_bytes / iter.page_size;
bool decrypted = false;
for (ulint i = 0; i < n_pages_read; ++i) {
if (iter.crypt_data != NULL) {
ulint size = iter.page_size;
bool decrypted = fil_space_decrypt(
/* If tablespace is encrypted, we need to decrypt
the page. */
if (encrypted) {
decrypted = fil_space_decrypt(
iter.crypt_data,
io_buffer + i * size, //dst
iter.page_size,
@ -6468,6 +6479,32 @@ fil_iterate(
buf_block_set_state(block, BUF_BLOCK_NOT_USED);
buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
/* If tablespace is encrypted, encrypt page before we
write it back. Note that we should not encrypt the
buffer that is in buffer pool. */
if (decrypted && encrypted) {
unsigned char *src = io_buffer + (i * size);
unsigned char *dst = writeptr + (i * size);
ulint space = mach_read_from_4(
src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET);
ib_uint64_t lsn = mach_read_from_8(src + FIL_PAGE_LSN);
byte* tmp = fil_encrypt_buf(
iter.crypt_data,
space,
offset,
lsn,
src,
iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size,
dst);
if (tmp == src) {
/* TODO: remove unnecessary memcpy's */
memcpy(writeptr, src, size);
}
}
page_off += iter.page_size;
block->frame += iter.page_size;
}
@ -6475,7 +6512,7 @@ fil_iterate(
/* A page was updated in the set, write back to disk. */
if (updated
&& !os_file_write(
iter.filepath, iter.file, io_buffer,
iter.filepath, iter.file, writeptr,
offset, (ulint) n_bytes)) {
ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed");
@ -6637,28 +6674,6 @@ fil_tablespace_iterate(
mem_free(io_buffer);
if (iter.crypt_data != NULL) {
/* clear crypt data from page 0 and write it back */
os_file_read(file, page, 0, UNIV_PAGE_SIZE);
fil_space_clear_crypt_data(page, crypt_data_offset);
lsn_t lsn = mach_read_from_8(page + FIL_PAGE_LSN);
if (callback.get_zip_size() == 0) {
buf_flush_init_for_writing(
page, 0, lsn);
} else {
buf_flush_update_zip_checksum(
page, callback.get_zip_size(), lsn);
}
if (!os_file_write(
iter.filepath, iter.file, page,
0, iter.page_size)) {
ib_logf(IB_LOG_LEVEL_ERROR,
"os_file_write() failed");
return(DB_IO_ERROR);
}
mem_free(crypt_io_buffer);
iter.crypt_io_buffer = NULL;
fil_space_destroy_crypt_data(&iter.crypt_data);

2
storage/innobase/handler/ha_innodb.cc
View file

@ -11976,6 +11976,8 @@ ha_innobase::discard_or_import_tablespace(
| HA_STATUS_CONST
| HA_STATUS_VARIABLE
| HA_STATUS_AUTO);
fil_crypt_set_encrypt_tables(srv_encrypt_tables);
}
}

14
storage/innobase/include/fil0crypt.h
View file

@ -381,6 +381,20 @@ fil_crypt_set_encrypt_tables(
/*=========================*/
uint val); /*!< in: New srv_encrypt_tables setting */
/******************************************************************
Encrypt a buffer */
UNIV_INTERN
byte*
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame); /*!< in: outbut buffer */
/******************************************************************
Calculate post encryption checksum

6
storage/innobase/include/row0ftsort.h
View file

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2010, 2012, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -71,6 +72,7 @@ struct fts_psort_common_t {
store Doc ID during sort, if
Doc ID will not be big enough
to use 8 bytes value */
fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */
};
struct fts_psort_t {
@ -83,6 +85,10 @@ struct fts_psort_t {
/*!< buffer to write to file */
row_merge_block_t* block_alloc[FTS_NUM_AUX_INDEX];
/*!< buffer to allocated */
row_merge_block_t* crypt_block[FTS_NUM_AUX_INDEX];
/*!< buffer to crypt data */
row_merge_block_t* crypt_alloc[FTS_NUM_AUX_INDEX];
/*!< buffer to allocated */
ulint child_status; /*!< child thread status */
ulint state; /*!< parent thread state */
fts_doc_list_t fts_doc_list; /*!< doc list to process */

23
storage/innobase/include/row0merge.h
View file

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2005, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -351,7 +352,11 @@ row_merge_write(
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf); /*!< in: data */
const void* buf, /*!< in: data */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
void* crypt_buf, /*!< in: crypt buf or NULL */
ulint space); /*!< in: space id */
/********************************************************************//**
Empty a sort buffer.
@return sort buffer */
@ -386,7 +391,10 @@ row_merge_sort(
int* tmpfd, /*!< in/out: temporary file handle */
const bool update_progress, /*!< in: update progress status variable or not */
const float pct_progress, /*!< in: total progress percent until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
__attribute__((nonnull));
/*********************************************************************//**
Allocate a sort buffer.
@ -424,7 +432,11 @@ row_merge_read(
ulint offset, /*!< in: offset where to read
in number of row_merge_block_t
elements */
row_merge_block_t* buf); /*!< out: data */
row_merge_block_t* buf, /*!< out: data */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */
ulint space); /*!< in: space id */
/********************************************************************//**
Read a merge record.
@return pointer to next record, or NULL on I/O error or end of list */
@ -441,6 +453,9 @@ row_merge_read_rec(
const mrec_t** mrec, /*!< out: pointer to merge record,
or NULL on end of list
(non-NULL on I/O error) */
ulint* offsets)/*!< out: offsets of mrec */
ulint* offsets,/*!< out: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
__attribute__((nonnull, warn_unused_result));
#endif /* row0merge.h */

84
storage/innobase/row/row0ftsort.cc
View file

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2010, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -37,7 +38,8 @@ Created 10/13/2010 Jimmy Yang
do { \
b[N] = row_merge_read_rec( \
block[N], buf[N], b[N], index, \
fd[N], &foffs[N], &mrec[N], offsets[N]); \
fd[N], &foffs[N], &mrec[N], offsets[N], \
crypt_data, crypt_block[N], space); \
if (UNIV_UNLIKELY(!b[N])) { \
if (mrec[N]) { \
goto exit; \
@ -191,6 +193,8 @@ row_fts_psort_info_init(
fts_psort_t* merge_info = NULL;
ulint block_size;
ibool ret = TRUE;
fil_space_crypt_t* crypt_data = NULL;
bool encrypted = false;
block_size = 3 * srv_sort_buf_size;
@ -219,6 +223,19 @@ row_fts_psort_info_init(
common_info->sort_event = os_event_create();
common_info->merge_event = os_event_create();
common_info->opt_doc_id_size = opt_doc_id_size;
crypt_data = fil_space_get_crypt_data(new_table->space);
if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
common_info->crypt_data = crypt_data;
encrypted = true;
} else {
/* Not needed */
common_info->crypt_data = NULL;
crypt_data = NULL;
}
/* There will be FTS_NUM_AUX_INDEX number of "sort buckets" for
each parallel sort thread. Each "sort bucket" holds records for
@ -256,6 +273,29 @@ row_fts_psort_info_init(
ut_align(
psort_info[j].block_alloc[i], 1024));
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
if (encrypted) {
/* Need to align memory for O_DIRECT write */
psort_info[j].crypt_alloc[i] =
static_cast<row_merge_block_t*>(ut_malloc(
block_size + 1024));
psort_info[j].crypt_block[i] =
static_cast<row_merge_block_t*>(
ut_align(
psort_info[j].crypt_alloc[i], 1024));
if (!psort_info[j].crypt_block[i]) {
ret = FALSE;
goto func_exit;
}
} else {
psort_info[j].crypt_alloc[i] = NULL;
psort_info[j].crypt_block[i] = NULL;
}
if (!psort_info[j].merge_block[i]) {
ret = FALSE;
goto func_exit;
@ -313,6 +353,11 @@ row_fts_psort_info_destroy(
if (psort_info[j].block_alloc[i]) {
ut_free(psort_info[j].block_alloc[i]);
}
if (psort_info[j].crypt_alloc[i]) {
ut_free(psort_info[j].crypt_alloc[i]);
}
mem_free(psort_info[j].merge_file[i]);
}
@ -595,6 +640,7 @@ fts_parallel_tokenization(
ibool processed = FALSE;
merge_file_t** merge_file;
row_merge_block_t** block;
row_merge_block_t** crypt_block;
int tmpfd[FTS_NUM_AUX_INDEX];
ulint mycount[FTS_NUM_AUX_INDEX];
ib_uint64_t total_rec = 0;
@ -609,6 +655,7 @@ fts_parallel_tokenization(
fts_tokenize_ctx_t t_ctx;
ulint retried = 0;
dberr_t error = DB_SUCCESS;
fil_space_crypt_t* crypt_data = NULL;
ut_ad(psort_info);
@ -630,6 +677,8 @@ fts_parallel_tokenization(
? DATA_VARCHAR : DATA_VARMYSQL;
block = psort_info->merge_block;
crypt_block = psort_info->crypt_block;
crypt_data = psort_info->psort_common->crypt_data;
zip_size = dict_table_zip_size(table);
row_merge_fts_get_next_doc_item(psort_info, &doc_item);
@ -724,7 +773,10 @@ loop:
if (!row_merge_write(merge_file[t_ctx.buf_used]->fd,
merge_file[t_ctx.buf_used]->offset++,
block[t_ctx.buf_used])) {
block[t_ctx.buf_used],
crypt_data,
crypt_block[t_ctx.buf_used],
table->space)) {
error = DB_TEMP_FILE_WRITE_FAILURE;
goto func_exit;
}
@ -817,13 +869,21 @@ exit:
if (merge_file[i]->offset != 0) {
if (!row_merge_write(merge_file[i]->fd,
merge_file[i]->offset++,
block[i])) {
block[i],
crypt_data,
crypt_block[i],
table->space)) {
error = DB_TEMP_FILE_WRITE_FAILURE;
goto func_exit;
}
UNIV_MEM_INVALID(block[i][0],
srv_sort_buf_size);
if (crypt_block[i]) {
UNIV_MEM_INVALID(crypt_block[i][0],
srv_sort_buf_size);
}
}
buf[i] = row_merge_buf_empty(buf[i]);
@ -848,7 +908,10 @@ exit:
error = row_merge_sort(psort_info->psort_common->trx,
psort_info->psort_common->dup,
merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */);
merge_file[i], block[i], &tmpfd[i],
false, 0.0/* pct_progress */, 0.0/* pct_cost */,
crypt_data, crypt_block[i], table->space);
if (error != DB_SUCCESS) {
close(tmpfd[i]);
goto func_exit;
@ -1352,6 +1415,7 @@ row_fts_merge_insert(
mrec_buf_t** buf;
int* fd;
byte** block;
byte** crypt_block;
const mrec_t** mrec;
ulint count = 0;
int* sel_tree;
@ -1359,6 +1423,8 @@ row_fts_merge_insert(
ulint start;
fts_psort_insert_t ins_ctx;
ulint count_diag = 0;
fil_space_crypt_t* crypt_data = NULL;
ulint space;
ut_ad(index);
ut_ad(table);
@ -1371,6 +1437,7 @@ row_fts_merge_insert(
ins_ctx.trx->op_info = "inserting index entries";
ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size;
crypt_data = psort_info[0].psort_common->crypt_data;
heap = mem_heap_create(500 + sizeof(mrec_buf_t));
@ -1385,6 +1452,8 @@ row_fts_merge_insert(
fd = (int*) mem_heap_alloc(heap, sizeof(*fd) * fts_sort_pll_degree);
block = (byte**) mem_heap_alloc(
heap, sizeof(*block) * fts_sort_pll_degree);
crypt_block = (byte**) mem_heap_alloc(
heap, sizeof(*block) * fts_sort_pll_degree);
mrec = (const mrec_t**) mem_heap_alloc(
heap, sizeof(*mrec) * fts_sort_pll_degree);
sel_tree = (int*) mem_heap_alloc(
@ -1405,6 +1474,7 @@ row_fts_merge_insert(
offsets[i][0] = num;
offsets[i][1] = dict_index_get_n_fields(index);
block[i] = psort_info[i].merge_block[id];
crypt_block[i] = psort_info[i].crypt_block[id];
b[i] = psort_info[i].merge_block[id];
fd[i] = psort_info[i].merge_file[id]->fd;
foffs[i] = 0;
@ -1447,6 +1517,7 @@ row_fts_merge_insert(
ins_ctx.fts_table.table_id = table->id;
ins_ctx.fts_table.parent = index->table->name;
ins_ctx.fts_table.table = index->table;
space = table->space;
for (i = 0; i < fts_sort_pll_degree; i++) {
if (psort_info[i].merge_file[id]->n_rec == 0) {
@ -1459,7 +1530,10 @@ row_fts_merge_insert(
if (psort_info[i].merge_file[id]->offset > 0
&& (!row_merge_read(
fd[i], foffs[i],
(row_merge_block_t*) block[i]))) {
(row_merge_block_t*) block[i],
crypt_data,
(row_merge_block_t*) crypt_block[i],
space))) {
error = DB_CORRUPTION;
goto exit;
}

333
storage/innobase/row/row0merge.cc
View file

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2005, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -41,6 +42,7 @@ Completed by Sunny Bains and Marko Makela
#include "handler0alter.h"
#include "ha_prototypes.h"
#include "math.h" /* log() */
#include "fil0crypt.h"
float my_log2f(float n)
{
@ -76,6 +78,92 @@ UNIV_INTERN char srv_disable_sort_file_cache;
/* Maximum pending doc memory limit in bytes for a fts tokenization thread */
#define FTS_PENDING_DOC_MEMORY_LIMIT 1000000
/* Reserve free space from every block for key_version */
#define ROW_MERGE_RESERVE_SIZE 4
/******************************************************//**
Encrypt a merge block. */
static
void
row_merge_encrypt_buf(
/*==================*/
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
ulint offset, /*!< in: offset where to
write */
ulint space, /*!< in: tablespace id */
const byte* input_buf, /*!< in: input buffer */
byte* crypted_buf) /*!< out: crypted buffer */
{
uint key_version;
uint dstlen=0;
os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset;
key_version = encryption_key_get_latest_version(crypt_data->key_id);
/* Store key_version at the begining of the input buffer */
mach_write_to_4((byte *)crypted_buf, key_version);
int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE,
srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE,
crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen,
crypt_data, key_version,
space, ofs, 0);
if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt data-block "
" src: %p srclen: %lu buf: %p buflen: %d."
" return-code: %d. Can't continue!\n",
input_buf, (size_t)srv_sort_buf_size,
crypted_buf, dstlen, rc);
ut_error;
}
}
/******************************************************//**
Decrypt a merge block. */
static
bool
row_merge_decrypt_buf(
/*==================*/
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
ulint offset, /*!< in: offset where to
write */
ulint space, /*!< in: tablespace id */
const byte* input_buf, /*!< in: input buffer */
byte* crypted_buf) /*!< out: crypted buffer */
{
uint key_version;
uint dstlen=0;
os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset;
/* Read key_version from begining of the buffer */
key_version = mach_read_from_4((byte *)input_buf);
if (key_version == 0) {
/* block not encrypted */
return false;
}
int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE,
srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE,
crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen,
crypt_data, key_version,
space, ofs, 0);
if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt data-block "
" src: %p srclen: %lu buf: %p buflen: %d."
" return-code: %d. Can't continue!\n",
input_buf, (size_t)srv_sort_buf_size,
crypted_buf, dstlen, rc);
ut_error;
}
return true;
}
#ifdef UNIV_DEBUG
/******************************************************//**
Display a merge tuple. */
@ -193,7 +281,7 @@ row_merge_buf_create(
ulint buf_size;
mem_heap_t* heap;
max_tuples = srv_sort_buf_size
max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)
/ ut_max(1, dict_index_get_min_size(index));
buf_size = (sizeof *buf);
@ -608,8 +696,8 @@ row_merge_buf_add(
ut_ad(data_size < srv_sort_buf_size);
/* Reserve one byte for the end marker of row_merge_block_t. */
if (buf->total_size + data_size >= srv_sort_buf_size - 1) {
/* Reserve bytes for the end marker of row_merge_block_t. */
if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) {
DBUG_RETURN(0);
}
@ -781,7 +869,7 @@ row_merge_buf_write(
{
const dict_index_t* index = buf->index;
ulint n_fields= dict_index_get_n_fields(index);
byte* b = &block[0];
byte* b = &block[ROW_MERGE_RESERVE_SIZE];
for (ulint i = 0; i < buf->n_tuples; i++) {
const mtuple_t* entry = &buf->tuples[i];
@ -800,7 +888,7 @@ row_merge_buf_write(
/* Write an "end-of-chunk" marker. */
ut_a(b < &block[srv_sort_buf_size]);
ut_a(b == &block[0] + buf->total_size);
ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE);
*b++ = 0;
#ifdef UNIV_DEBUG_VALGRIND
/* The rest of the block is uninitialized. Initialize it
@ -857,7 +945,10 @@ row_merge_read(
ulint offset, /*!< in: offset where to read
in number of row_merge_block_t
elements */
row_merge_block_t* buf) /*!< out: data */
row_merge_block_t* buf, /*!< out: data */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size;
ibool success;
@ -881,6 +972,13 @@ row_merge_read(
success = os_file_read_no_error_handling(OS_FILE_FROM_FD(fd), buf,
ofs, srv_sort_buf_size);
/* For encrypted tables, decrypt data after reading and copy data */
if (crypt_data && crypt_buf) {
if (row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) {
memcpy(buf, crypt_buf, srv_sort_buf_size);
}
}
#ifdef POSIX_FADV_DONTNEED
/* Each block is read exactly once. Free up the file cache. */
posix_fadvise(fd, ofs, srv_sort_buf_size, POSIX_FADV_DONTNEED);
@ -906,15 +1004,29 @@ row_merge_write(
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf) /*!< in: data */
const void* buf, /*!< in: data */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
void* crypt_buf, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
size_t buf_len = srv_sort_buf_size;
os_offset_t ofs = buf_len * (os_offset_t) offset;
ibool ret;
void* out_buf = (void *)buf;
DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE););
ret = os_file_write("(merge)", OS_FILE_FROM_FD(fd), buf, ofs, buf_len);
/* For encrypted tables, encrypt data before writing */
if (crypt_data && crypt_buf) {
row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf);
out_buf = crypt_buf;
} else {
/* Mark block unencrypted */
mach_write_to_4((byte *)out_buf, 0);
}
ret = os_file_write("(merge)", OS_FILE_FROM_FD(fd), out_buf, ofs, buf_len);
#ifdef UNIV_DEBUG
if (row_merge_print_block_write) {
@ -948,7 +1060,10 @@ row_merge_read_rec(
const mrec_t** mrec, /*!< out: pointer to merge record,
or NULL on end of list
(non-NULL on I/O error) */
ulint* offsets)/*!< out: offsets of mrec */
ulint* offsets,/*!< out: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint extra_size;
ulint data_size;
@ -966,6 +1081,10 @@ row_merge_read_rec(
ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE
+ dict_index_get_n_fields(index));
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
extra_size = *b++;
if (UNIV_UNLIKELY(!extra_size)) {
@ -985,7 +1104,8 @@ row_merge_read_rec(
/* Read another byte of extra_size. */
if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) {
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
err_exit:
/* Signal I/O error. */
*mrec = b;
@ -993,7 +1113,7 @@ err_exit:
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
}
extra_size = (extra_size & 0x7f) << 8;
@ -1014,13 +1134,14 @@ err_exit:
ut_ad(avail_size < sizeof *buf);
memcpy(*buf, b, avail_size);
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
goto err_exit;
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
/* Copy the record. */
memcpy(*buf + avail_size, b, extra_size - avail_size);
@ -1075,13 +1196,14 @@ err_exit:
offsets[3] = (ulint) index;
#endif /* UNIV_DEBUG */
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
goto err_exit;
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
/* Copy the rest of the record. */
memcpy(*buf + avail_size, b, extra_size + data_size - avail_size);
@ -1157,7 +1279,10 @@ row_merge_write_rec(
int fd, /*!< in: file descriptor */
ulint* foffs, /*!< in/out: file offset */
const mrec_t* mrec, /*!< in: record to write */
const ulint* offsets)/*!< in: offsets of mrec */
const ulint* offsets,/*!< in: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint extra_size;
ulint size;
@ -1178,6 +1303,10 @@ row_merge_write_rec(
size = extra_size + (extra_size >= 0x80)
+ rec_offs_data_size(offsets);
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) {
/* The record spans two blocks.
Copy it to the temporary buffer first. */
@ -1192,14 +1321,15 @@ row_merge_write_rec(
record to the head of the new block. */
memcpy(b, buf[0], avail_size);
if (!row_merge_write(fd, (*foffs)++, block)) {
if (!row_merge_write(fd, (*foffs)++, block,
crypt_data, crypt_block, space)) {
return(NULL);
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
/* Copy the rest. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
memcpy(b, buf[0] + avail_size, size - avail_size);
b += size - avail_size;
} else {
@ -1221,7 +1351,10 @@ row_merge_write_eof(
row_merge_block_t* block, /*!< in/out: file buffer */
byte* b, /*!< in: pointer to end of block */
int fd, /*!< in: file descriptor */
ulint* foffs) /*!< in/out: file offset */
ulint* foffs, /*!< in/out: file offset */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ut_ad(block);
ut_ad(b >= &block[0]);
@ -1234,20 +1367,27 @@ row_merge_write_eof(
}
#endif /* UNIV_DEBUG */
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
*b++ = 0;
UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]);
UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size);
#ifdef UNIV_DEBUG_VALGRIND
/* The rest of the block is uninitialized. Initialize it
to avoid bogus warnings. */
memset(b, 0xff, &block[srv_sort_buf_size] - b);
#endif /* UNIV_DEBUG_VALGRIND */
if (!row_merge_write(fd, (*foffs)++, block)) {
if (!row_merge_write(fd, (*foffs)++, block,
crypt_data, crypt_block, space)) {
return(NULL);
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
return(&block[0]);
}
@ -1292,7 +1432,11 @@ row_merge_read_clustered_index(
ULINT_UNDEFINED if none is added */
ib_sequence_t& sequence,/*!< in/out: autoinc sequence */
row_merge_block_t* block, /*!< in/out: file buffer */
float pct_cost) /*!< in: percent of task weight out of total alter job */
float pct_cost, /*!< in: percent of task weight
out of total alter job */
fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */
row_merge_block_t* crypt_block)/*!< in: in/out: crypted file
buffer */
{
dict_index_t* clust_index; /* Clustered index */
mem_heap_t* row_heap; /* Heap memory to create
@ -1314,9 +1458,10 @@ row_merge_read_clustered_index(
ib_int64_t sig_count = 0;
mem_heap_t* conv_heap = NULL;
float curr_progress;
float curr_progress = 0.0;
ib_int64_t read_rows = 0;
ib_int64_t table_total_rows;
ib_int64_t table_total_rows = 0;
DBUG_ENTER("row_merge_read_clustered_index");
ut_ad((old_table == new_table) == !col_map);
@ -1807,14 +1952,15 @@ write_buffers:
row_merge_buf_write(buf, file, block);
if (!row_merge_write(file->fd, file->offset++,
block)) {
if (!row_merge_write(file->fd, file->offset++, block,
crypt_data, crypt_block, new_table->space)) {
err = DB_TEMP_FILE_WRITE_FAILURE;
trx->error_key_num = i;
break;
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
merge_buf[i] = row_merge_buf_empty(buf);
if (UNIV_LIKELY(row != NULL)) {
@ -1983,14 +2129,21 @@ wait_again:
b2 = row_merge_write_rec(&block[2 * srv_sort_buf_size], \
&buf[2], b2, \
of->fd, &of->offset, \
mrec##N, offsets##N); \
mrec##N, offsets##N, \
crypt_data, \
&crypt_block[2 * srv_sort_buf_size], \
space); \
if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \
goto corrupt; \
} \
b##N = row_merge_read_rec(&block[N * srv_sort_buf_size],\
&buf[N], b##N, INDEX, \
file->fd, foffs##N, \
&mrec##N, offsets##N); \
&mrec##N, offsets##N, \
crypt_data, \
&crypt_block[N * srv_sort_buf_size], \
space); \
\
if (UNIV_UNLIKELY(!b##N)) { \
if (mrec##N) { \
goto corrupt; \
@ -2015,7 +2168,11 @@ row_merge_blocks(
source list in the file */
ulint* foffs1, /*!< in/out: offset of second
source list in the file */
merge_file_t* of) /*!< in/out: output file */
merge_file_t* of, /*!< in/out: output file */
fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */
row_merge_block_t* crypt_block,/*!< in: in/out: crypted file
buffer */
ulint space) /*!< in: space id */
{
mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */
@ -2046,8 +2203,10 @@ row_merge_blocks(
/* Write a record and read the next record. Split the output
file in two halves, which can be merged on the following pass. */
if (!row_merge_read(file->fd, *foffs0, &block[0])
|| !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size])) {
if (!row_merge_read(file->fd, *foffs0, &block[0],
crypt_data, &crypt_block[0], space)
|| !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size],
crypt_data, &crypt_block[srv_sort_buf_size], space)) {
corrupt:
mem_heap_free(heap);
return(DB_CORRUPTION);
@ -2059,11 +2218,15 @@ corrupt:
b0 = row_merge_read_rec(
&block[0], &buf[0], b0, dup->index,
file->fd, foffs0, &mrec0, offsets0);
file->fd, foffs0, &mrec0, offsets0,
crypt_data, &crypt_block[0], space);
b1 = row_merge_read_rec(
&block[srv_sort_buf_size],
&buf[srv_sort_buf_size], b1, dup->index,
file->fd, foffs1, &mrec1, offsets1);
file->fd, foffs1, &mrec1, offsets1,
crypt_data, &crypt_block[srv_sort_buf_size], space);
if (UNIV_UNLIKELY(!b0 && mrec0)
|| UNIV_UNLIKELY(!b1 && mrec1)) {
@ -2105,8 +2268,11 @@ done0:
done1:
mem_heap_free(heap);
b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size],
b2, of->fd, &of->offset);
b2, of->fd, &of->offset,
crypt_data, &crypt_block[2 * srv_sort_buf_size], space);
return(b2 ? DB_SUCCESS : DB_CORRUPTION);
}
@ -2121,7 +2287,10 @@ row_merge_blocks_copy(
const merge_file_t* file, /*!< in: input file */
row_merge_block_t* block, /*!< in/out: 3 buffers */
ulint* foffs0, /*!< in/out: input file offset */
merge_file_t* of) /*!< in/out: output file */
merge_file_t* of, /*!< in/out: output file */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */
@ -2148,7 +2317,8 @@ row_merge_blocks_copy(
/* Write a record and read the next record. Split the output
file in two halves, which can be merged on the following pass. */
if (!row_merge_read(file->fd, *foffs0, &block[0])) {
if (!row_merge_read(file->fd, *foffs0, &block[0],
crypt_data, &crypt_block[0], space)) {
corrupt:
mem_heap_free(heap);
return(FALSE);
@ -2159,7 +2329,9 @@ corrupt:
b2 = &block[2 * srv_sort_buf_size];
b0 = row_merge_read_rec(&block[0], &buf[0], b0, index,
file->fd, foffs0, &mrec0, offsets0);
file->fd, foffs0, &mrec0, offsets0,
crypt_data, &crypt_block[0], space);
if (UNIV_UNLIKELY(!b0 && mrec0)) {
goto corrupt;
@ -2178,8 +2350,10 @@ done0:
(*foffs0)++;
mem_heap_free(heap);
return(row_merge_write_eof(&block[2 * srv_sort_buf_size],
b2, of->fd, &of->offset)
b2, of->fd, &of->offset,
crypt_data, &crypt_block[2 * srv_sort_buf_size], space)
!= NULL);
}
@ -2199,9 +2373,12 @@ row_merge(
int* tmpfd, /*!< in/out: temporary file handle */
ulint* num_run,/*!< in/out: Number of runs remain
to be merged */
ulint* run_offset) /*!< in/out: Array contains the
ulint* run_offset, /*!< in/out: Array contains the
first offset number for each merge
run */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint foffs0; /*!< first input offset */
ulint foffs1; /*!< second input offset */
@ -2214,6 +2391,10 @@ row_merge(
UNIV_MEM_ASSERT_W(&block[0], 3 * srv_sort_buf_size);
if (crypt_block) {
UNIV_MEM_ASSERT_W(&crypt_block[0], 3 * srv_sort_buf_size);
}
ut_ad(ihalf < file->offset);
of.fd = *tmpfd;
@ -2244,7 +2425,8 @@ row_merge(
run_offset[n_run++] = of.offset;
error = row_merge_blocks(dup, file, block,
&foffs0, &foffs1, &of);
&foffs0, &foffs1, &of,
crypt_data, crypt_block, space);
if (error != DB_SUCCESS) {
return(error);
@ -2264,7 +2446,8 @@ row_merge(
run_offset[n_run++] = of.offset;
if (!row_merge_blocks_copy(dup->index, file, block,
&foffs0, &of)) {
&foffs0, &of,
crypt_data, crypt_block, space)) {
return(DB_CORRUPTION);
}
}
@ -2281,7 +2464,8 @@ row_merge(
run_offset[n_run++] = of.offset;
if (!row_merge_blocks_copy(dup->index, file, block,
&foffs1, &of)) {
&foffs1, &of,
crypt_data, crypt_block, space)) {
return(DB_CORRUPTION);
}
}
@ -2336,7 +2520,10 @@ row_merge_sort(
const float pct_progress,
/*!< in: total progress percent
until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
const ulint half = file->offset / 2;
ulint num_runs;
@ -2388,7 +2575,8 @@ row_merge_sort(
sql_print_information("InnoDB: Online DDL : merge-sorting current run %lu estimated %lu runs", cur_run, num_runs);
error = row_merge(trx, dup, file, block, tmpfd,
&num_runs, run_offset);
&num_runs, run_offset,
crypt_data, crypt_block, space);
if(update_progress) {
merge_count++;
@ -2471,7 +2659,11 @@ row_merge_insert_index_tuples(
row_merge_block_t* block, /*!< in/out: file buffer */
const ib_int64_t table_total_rows, /*!< in: total rows of old table */
const float pct_progress, /*!< in: total progress percent until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent
*/
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
const byte* b;
mem_heap_t* heap;
@ -2502,9 +2694,10 @@ row_merge_insert_index_tuples(
offsets[1] = dict_index_get_n_fields(index);
}
b = block;
b = &block[0];
if (!row_merge_read(fd, foffs, block)) {
if (!row_merge_read(fd, foffs, block,
crypt_data, crypt_block, space)) {
error = DB_CORRUPTION;
} else {
buf = static_cast<mrec_buf_t*>(
@ -2520,7 +2713,8 @@ row_merge_insert_index_tuples(
mtr_t mtr;
b = row_merge_read_rec(block, buf, b, index,
fd, &foffs, &mrec, offsets);
fd, &foffs, &mrec, offsets,
crypt_data, crypt_block, space);
if (UNIV_UNLIKELY(!b)) {
/* End of list, or I/O error */
if (mrec) {
@ -3661,6 +3855,7 @@ row_merge_build_indexes(
{
merge_file_t* merge_files;
row_merge_block_t* block;
row_merge_block_t* crypt_block;
ulint block_size;
ulint i;
ulint j;
@ -3671,6 +3866,7 @@ row_merge_build_indexes(
fts_psort_t* merge_info = NULL;
ib_int64_t sig_count = 0;
bool fts_psort_initiated = false;
fil_space_crypt_t * crypt_data = NULL;
float total_static_cost = 0;
float total_dynamic_cost = 0;
@ -3695,6 +3891,27 @@ row_merge_build_indexes(
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
/* Get crypt data from tablespace if present. */
crypt_data = fil_space_get_crypt_data(new_table->space);
crypt_block = NULL;
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
crypt_block = static_cast<row_merge_block_t*>(
os_mem_alloc_large(&block_size));
if (crypt_block == NULL) {
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
} else {
/* Not needed */
crypt_data = NULL;
}
trx_start_if_not_started_xa(trx);
merge_files = static_cast<merge_file_t*>(
@ -3779,7 +3996,8 @@ row_merge_build_indexes(
trx, table, old_table, new_table, online, indexes,
fts_sort_idx, psort_info, merge_files, key_numbers,
n_indexes, add_cols, col_map,
add_autoinc, sequence, block, pct_cost);
add_autoinc, sequence, block, pct_cost,
crypt_data, crypt_block);
pct_progress += pct_cost;
@ -3799,6 +4017,10 @@ row_merge_build_indexes(
/* Now we have files containing index entries ready for
sorting and inserting. */
DBUG_EXECUTE_IF(
"ib_merge_wait_after_read",
os_thread_sleep(20000000);); /* 20 sec */
for (i = 0; i < n_indexes; i++) {
dict_index_t* sort_idx = indexes[i];
@ -3893,7 +4115,9 @@ wait_again:
error = row_merge_sort(
trx, &dup, &merge_files[i],
block, &tmpfd, true, pct_progress, pct_cost);
block, &tmpfd, true,
pct_progress, pct_cost,
crypt_data, crypt_block, new_table->space);
pct_progress += pct_cost;
@ -3901,6 +4125,10 @@ wait_again:
" merge-sorting index %s (%lu / %lu)",
buf, (i+1), n_indexes);
DBUG_EXECUTE_IF(
"ib_merge_wait_after_sort",
os_thread_sleep(20000000);); /* 20 sec */
if (error == DB_SUCCESS) {
pct_cost = (COST_BUILD_INDEX_STATIC +
(total_dynamic_cost * merge_files[i].offset /
@ -3916,7 +4144,8 @@ wait_again:
error = row_merge_insert_index_tuples(
trx->id, sort_idx, old_table,
merge_files[i].fd, block,
merge_files[i].n_rec, pct_progress, pct_cost);
merge_files[i].n_rec, pct_progress, pct_cost,
crypt_data, crypt_block, new_table->space);
pct_progress += pct_cost;
sql_print_information("InnoDB: Online DDL : "
@ -3990,6 +4219,10 @@ func_exit:
mem_free(merge_files);
os_mem_free_large(block, block_size);
if (crypt_block) {
os_mem_free_large(crypt_block, block_size);
}
DICT_TF2_FLAG_UNSET(new_table, DICT_TF2_FTS_ADD_DOC_ID);
if (online && old_table == new_table && error != DB_SUCCESS) {

76
storage/xtradb/fil/fil0crypt.cc
View file

@ -244,9 +244,6 @@ fil_space_merge_crypt_data(
ut_a(dst->type == CRYPT_SCHEME_UNENCRYPTED ||
dst->type == CRYPT_SCHEME_1);
/* no support for changing iv (yet?) */
ut_a(memcmp(src->iv, dst->iv, sizeof(src->iv)) == 0);
dst->encryption = src->encryption;
dst->type = src->type;
dst->min_key_version = src->min_key_version;
@ -555,42 +552,22 @@ fil_space_clear_crypt_data(
}
/******************************************************************
Encrypt a page */
Encrypt a buffer */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
row format compressed */
byte* dst_frame) /*!< in: outbut buffer */
{
fil_space_crypt_t* crypt_data = NULL;
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
uint key_version;
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR
|| orig_page_type==FIL_PAGE_TYPE_XDES) {
/* File space header or extent descriptor do not need to be
encrypted. */
return src_frame;
}
/* Get crypt data from file space */
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data == NULL) {
return src_frame;
}
ut_ad(crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF);
key_version = fil_crypt_get_latest_key_version(crypt_data);
uint key_version = fil_crypt_get_latest_key_version(crypt_data);
if (key_version == ENCRYPTION_KEY_VERSION_INVALID) {
ib_logf(IB_LOG_LEVEL_FATAL,
@ -599,6 +576,7 @@ fil_space_encrypt(
ut_error;
}
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
ibool page_compressed = (orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint header_len = FIL_PAGE_DATA;
@ -661,6 +639,45 @@ fil_space_encrypt(
return dst_frame;
}
/******************************************************************
Encrypt a page */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame) /*!< in: outbut buffer */
{
fil_space_crypt_t* crypt_data = NULL;
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR
|| orig_page_type==FIL_PAGE_TYPE_XDES) {
/* File space header or extent descriptor do not need to be
encrypted. */
return src_frame;
}
/* Get crypt data from file space */
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data == NULL) {
return src_frame;
}
ut_ad(crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF);
byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame);
return tmp;
}
/*********************************************************************
Check if extra buffer shall be allocated for decrypting after read
@return true if fil space has encryption data. */
@ -2330,6 +2347,7 @@ fil_crypt_threads_init()
uint cnt = srv_n_fil_crypt_threads;
srv_n_fil_crypt_threads = 0;
fil_crypt_set_thread_cnt(cnt);
fil_crypt_threads_inited = true;
}
/*********************************************************************

69
storage/xtradb/fil/fil0fil.cc
View file

@ -6475,8 +6475,16 @@ fil_iterate(
ut_ad(!(n_bytes % iter.page_size));
byte* readptr = io_buffer;
if (iter.crypt_data != NULL) {
byte* writeptr = io_buffer;
bool encrypted = false;
/* Use additional crypt io buffer if tablespace is encrypted */
if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
encrypted = true;
readptr = iter.crypt_io_buffer;
writeptr = iter.crypt_io_buffer;
}
if (!os_file_read(iter.file, readptr, offset, (ulint) n_bytes)) {
@ -6489,12 +6497,15 @@ fil_iterate(
bool updated = false;
os_offset_t page_off = offset;
ulint n_pages_read = (ulint) n_bytes / iter.page_size;
bool decrypted = false;
for (ulint i = 0; i < n_pages_read; ++i) {
if (iter.crypt_data != NULL) {
ulint size = iter.page_size;
bool decrypted = fil_space_decrypt(
/* If tablespace is encrypted, we need to decrypt
the page. */
if (encrypted) {
decrypted = fil_space_decrypt(
iter.crypt_data,
io_buffer + i * size, //dst
iter.page_size,
@ -6525,6 +6536,32 @@ fil_iterate(
buf_block_set_state(block, BUF_BLOCK_NOT_USED);
buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
/* If tablespace is encrypted, encrypt page before we
write it back. Note that we should not encrypt the
buffer that is in buffer pool. */
if (decrypted && encrypted) {
unsigned char *src = io_buffer + (i * size);
unsigned char *dst = writeptr + (i * size);
ulint space = mach_read_from_4(
src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET);
ib_uint64_t lsn = mach_read_from_8(src + FIL_PAGE_LSN);
byte* tmp = fil_encrypt_buf(
iter.crypt_data,
space,
offset,
lsn,
src,
iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size,
dst);
if (tmp == src) {
/* TODO: remove unnecessary memcpy's */
memcpy(writeptr, src, size);
}
}
page_off += iter.page_size;
block->frame += iter.page_size;
}
@ -6532,7 +6569,7 @@ fil_iterate(
/* A page was updated in the set, write back to disk. */
if (updated
&& !os_file_write(
iter.filepath, iter.file, io_buffer,
iter.filepath, iter.file, writeptr,
offset, (ulint) n_bytes)) {
ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed");
@ -6694,28 +6731,6 @@ fil_tablespace_iterate(
mem_free(io_buffer);
if (iter.crypt_data != NULL) {
/* clear crypt data from page 0 and write it back */
os_file_read(file, page, 0, UNIV_PAGE_SIZE);
fil_space_clear_crypt_data(page, crypt_data_offset);
lsn_t lsn = mach_read_from_8(page + FIL_PAGE_LSN);
if (callback.get_zip_size() == 0) {
buf_flush_init_for_writing(
page, 0, lsn);
} else {
buf_flush_update_zip_checksum(
page, callback.get_zip_size(), lsn);
}
if (!os_file_write(
iter.filepath, iter.file, page,
0, iter.page_size)) {
ib_logf(IB_LOG_LEVEL_ERROR,
"os_file_write() failed");
return(DB_IO_ERROR);
}
mem_free(crypt_io_buffer);
iter.crypt_io_buffer = NULL;
fil_space_destroy_crypt_data(&iter.crypt_data);

2
storage/xtradb/handler/ha_innodb.cc
View file

@ -12465,6 +12465,8 @@ ha_innobase::discard_or_import_tablespace(
| HA_STATUS_CONST
| HA_STATUS_VARIABLE
| HA_STATUS_AUTO);
fil_crypt_set_encrypt_tables(srv_encrypt_tables);
}
}

16
storage/xtradb/include/fil0crypt.h
View file

@ -381,6 +381,21 @@ fil_crypt_set_encrypt_tables(
/*=========================*/
uint val); /*!< in: New srv_encrypt_tables setting */
/******************************************************************
Encrypt a buffer */
UNIV_INTERN
byte*
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame); /*!< in: outbut buffer */
/******************************************************************
Calculate post encryption checksum
@return page checksum or BUF_NO_CHECKSUM_MAGIC
@ -392,6 +407,7 @@ fil_crypt_calculate_checksum(
ulint zip_size, /*!< in: zip_size or 0 */
byte* dst_frame); /*!< in: page where to calculate */
#ifndef UNIV_NONINL
#include "fil0crypt.ic"
#endif

6
storage/xtradb/include/row0ftsort.h
View file

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2010, 2012, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -71,6 +72,7 @@ struct fts_psort_common_t {
store Doc ID during sort, if
Doc ID will not be big enough
to use 8 bytes value */
fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */
};
struct fts_psort_t {
@ -83,6 +85,10 @@ struct fts_psort_t {
/*!< buffer to write to file */
row_merge_block_t* block_alloc[FTS_NUM_AUX_INDEX];
/*!< buffer to allocated */
row_merge_block_t* crypt_block[FTS_NUM_AUX_INDEX];
/*!< buffer to crypt data */
row_merge_block_t* crypt_alloc[FTS_NUM_AUX_INDEX];
/*!< buffer to allocated */
ulint child_status; /*!< child thread status */
ulint state; /*!< parent thread state */
fts_doc_list_t fts_doc_list; /*!< doc list to process */

23
storage/xtradb/include/row0merge.h
View file

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2005, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -351,7 +352,11 @@ row_merge_write(
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf); /*!< in: data */
const void* buf, /*!< in: data */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
void* crypt_buf, /*!< in: crypt buf or NULL */
ulint space); /*!< in: space id */
/********************************************************************//**
Empty a sort buffer.
@return sort buffer */
@ -386,7 +391,10 @@ row_merge_sort(
int* tmpfd, /*!< in/out: temporary file handle */
const bool update_progress, /*!< in: update progress status variable or not */
const float pct_progress, /*!< in: total progress percent until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
__attribute__((nonnull));
/*********************************************************************//**
Allocate a sort buffer.
@ -424,7 +432,11 @@ row_merge_read(
ulint offset, /*!< in: offset where to read
in number of row_merge_block_t
elements */
row_merge_block_t* buf); /*!< out: data */
row_merge_block_t* buf, /*!< out: data */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */
ulint space); /*!< in: space id */
/********************************************************************//**
Read a merge record.
@return pointer to next record, or NULL on I/O error or end of list */
@ -441,6 +453,9 @@ row_merge_read_rec(
const mrec_t** mrec, /*!< out: pointer to merge record,
or NULL on end of list
(non-NULL on I/O error) */
ulint* offsets)/*!< out: offsets of mrec */
ulint* offsets,/*!< out: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
__attribute__((nonnull, warn_unused_result));
#endif /* row0merge.h */

84
storage/xtradb/row/row0ftsort.cc
View file

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2010, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -38,7 +39,8 @@ Created 10/13/2010 Jimmy Yang
do { \
b[N] = row_merge_read_rec( \
block[N], buf[N], b[N], index, \
fd[N], &foffs[N], &mrec[N], offsets[N]); \
fd[N], &foffs[N], &mrec[N], offsets[N], \
crypt_data, crypt_block[N], space); \
if (UNIV_UNLIKELY(!b[N])) { \
if (mrec[N]) { \
goto exit; \
@ -194,6 +196,8 @@ row_fts_psort_info_init(
fts_psort_t* merge_info = NULL;
ulint block_size;
ibool ret = TRUE;
fil_space_crypt_t* crypt_data = NULL;
bool encrypted = false;
block_size = 3 * srv_sort_buf_size;
@ -222,6 +226,19 @@ row_fts_psort_info_init(
common_info->sort_event = os_event_create();
common_info->merge_event = os_event_create();
common_info->opt_doc_id_size = opt_doc_id_size;
crypt_data = fil_space_get_crypt_data(new_table->space);
if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
common_info->crypt_data = crypt_data;
encrypted = true;
} else {
/* Not needed */
common_info->crypt_data = NULL;
crypt_data = NULL;
}
/* There will be FTS_NUM_AUX_INDEX number of "sort buckets" for
each parallel sort thread. Each "sort bucket" holds records for
@ -259,6 +276,29 @@ row_fts_psort_info_init(
ut_align(
psort_info[j].block_alloc[i], 1024));
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
if (encrypted) {
/* Need to align memory for O_DIRECT write */
psort_info[j].crypt_alloc[i] =
static_cast<row_merge_block_t*>(ut_malloc(
block_size + 1024));
psort_info[j].crypt_block[i] =
static_cast<row_merge_block_t*>(
ut_align(
psort_info[j].crypt_alloc[i], 1024));
if (!psort_info[j].crypt_block[i]) {
ret = FALSE;
goto func_exit;
}
} else {
psort_info[j].crypt_alloc[i] = NULL;
psort_info[j].crypt_block[i] = NULL;
}
if (!psort_info[j].merge_block[i]) {
ret = FALSE;
goto func_exit;
@ -316,6 +356,11 @@ row_fts_psort_info_destroy(
if (psort_info[j].block_alloc[i]) {
ut_free(psort_info[j].block_alloc[i]);
}
if (psort_info[j].crypt_alloc[i]) {
ut_free(psort_info[j].crypt_alloc[i]);
}
mem_free(psort_info[j].merge_file[i]);
}
@ -598,6 +643,7 @@ fts_parallel_tokenization(
ibool processed = FALSE;
merge_file_t** merge_file;
row_merge_block_t** block;
row_merge_block_t** crypt_block;
int tmpfd[FTS_NUM_AUX_INDEX];
ulint mycount[FTS_NUM_AUX_INDEX];
ib_uint64_t total_rec = 0;
@ -612,6 +658,7 @@ fts_parallel_tokenization(
fts_tokenize_ctx_t t_ctx;
ulint retried = 0;
dberr_t error = DB_SUCCESS;
fil_space_crypt_t* crypt_data = NULL;
ut_ad(psort_info);
@ -633,6 +680,8 @@ fts_parallel_tokenization(
? DATA_VARCHAR : DATA_VARMYSQL;
block = psort_info->merge_block;
crypt_block = psort_info->crypt_block;
crypt_data = psort_info->psort_common->crypt_data;
zip_size = dict_table_zip_size(table);
row_merge_fts_get_next_doc_item(psort_info, &doc_item);
@ -727,7 +776,10 @@ loop:
if (!row_merge_write(merge_file[t_ctx.buf_used]->fd,
merge_file[t_ctx.buf_used]->offset++,
block[t_ctx.buf_used])) {
block[t_ctx.buf_used],
crypt_data,
crypt_block[t_ctx.buf_used],
table->space)) {
error = DB_TEMP_FILE_WRITE_FAILURE;
goto func_exit;
}
@ -820,13 +872,21 @@ exit:
if (merge_file[i]->offset != 0) {
if (!row_merge_write(merge_file[i]->fd,
merge_file[i]->offset++,
block[i])) {
block[i],
crypt_data,
crypt_block[i],
table->space)) {
error = DB_TEMP_FILE_WRITE_FAILURE;
goto func_exit;
}
UNIV_MEM_INVALID(block[i][0],
srv_sort_buf_size);
if (crypt_block[i]) {
UNIV_MEM_INVALID(crypt_block[i][0],
srv_sort_buf_size);
}
}
buf[i] = row_merge_buf_empty(buf[i]);
@ -851,7 +911,10 @@ exit:
error = row_merge_sort(psort_info->psort_common->trx,
psort_info->psort_common->dup,
merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */);
merge_file[i], block[i], &tmpfd[i],
false, 0.0/* pct_progress */, 0.0/* pct_cost */,
crypt_data, crypt_block[i], table->space);
if (error != DB_SUCCESS) {
close(tmpfd[i]);
goto func_exit;
@ -1355,6 +1418,7 @@ row_fts_merge_insert(
mrec_buf_t** buf;
int* fd;
byte** block;
byte** crypt_block;
const mrec_t** mrec;
ulint count = 0;
int* sel_tree;
@ -1362,6 +1426,8 @@ row_fts_merge_insert(
ulint start;
fts_psort_insert_t ins_ctx;
ulint count_diag = 0;
fil_space_crypt_t* crypt_data = NULL;
ulint space;
ut_ad(index);
ut_ad(table);
@ -1374,6 +1440,7 @@ row_fts_merge_insert(
ins_ctx.trx->op_info = "inserting index entries";
ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size;
crypt_data = psort_info[0].psort_common->crypt_data;
heap = mem_heap_create(500 + sizeof(mrec_buf_t));
@ -1388,6 +1455,8 @@ row_fts_merge_insert(
fd = (int*) mem_heap_alloc(heap, sizeof(*fd) * fts_sort_pll_degree);
block = (byte**) mem_heap_alloc(
heap, sizeof(*block) * fts_sort_pll_degree);
crypt_block = (byte**) mem_heap_alloc(
heap, sizeof(*block) * fts_sort_pll_degree);
mrec = (const mrec_t**) mem_heap_alloc(
heap, sizeof(*mrec) * fts_sort_pll_degree);
sel_tree = (int*) mem_heap_alloc(
@ -1408,6 +1477,7 @@ row_fts_merge_insert(
offsets[i][0] = num;
offsets[i][1] = dict_index_get_n_fields(index);
block[i] = psort_info[i].merge_block[id];
crypt_block[i] = psort_info[i].crypt_block[id];
b[i] = psort_info[i].merge_block[id];
fd[i] = psort_info[i].merge_file[id]->fd;
foffs[i] = 0;
@ -1450,6 +1520,7 @@ row_fts_merge_insert(
ins_ctx.fts_table.table_id = table->id;
ins_ctx.fts_table.parent = index->table->name;
ins_ctx.fts_table.table = index->table;
space = table->space;
for (i = 0; i < fts_sort_pll_degree; i++) {
if (psort_info[i].merge_file[id]->n_rec == 0) {
@ -1462,7 +1533,10 @@ row_fts_merge_insert(
if (psort_info[i].merge_file[id]->offset > 0
&& (!row_merge_read(
fd[i], foffs[i],
(row_merge_block_t*) block[i]))) {
(row_merge_block_t*) block[i],
crypt_data,
(row_merge_block_t*) crypt_block[i],
space))) {
error = DB_CORRUPTION;
goto exit;
}

333
storage/xtradb/row/row0merge.cc
View file

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2005, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -41,6 +42,7 @@ Completed by Sunny Bains and Marko Makela
#include "handler0alter.h"
#include "ha_prototypes.h"
#include "math.h" /* log2() */
#include "fil0crypt.h"
float my_log2f(float n)
{
@ -76,6 +78,92 @@ UNIV_INTERN char srv_disable_sort_file_cache;
/* Maximum pending doc memory limit in bytes for a fts tokenization thread */
#define FTS_PENDING_DOC_MEMORY_LIMIT 1000000
/* Reserve free space from every block for key_version */
#define ROW_MERGE_RESERVE_SIZE 4
/******************************************************//**
Encrypt a merge block. */
static
void
row_merge_encrypt_buf(
/*==================*/
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
ulint offset, /*!< in: offset where to
write */
ulint space, /*!< in: tablespace id */
const byte* input_buf, /*!< in: input buffer */
byte* crypted_buf) /*!< out: crypted buffer */
{
uint key_version;
uint dstlen=0;
os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset;
key_version = encryption_key_get_latest_version(crypt_data->key_id);
/* Store key_version at the begining of the input buffer */
mach_write_to_4((byte *)crypted_buf, key_version);
int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE,
srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE,
crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen,
crypt_data, key_version,
space, ofs, 0);
if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt data-block "
" src: %p srclen: %lu buf: %p buflen: %d."
" return-code: %d. Can't continue!\n",
input_buf, (size_t)srv_sort_buf_size,
crypted_buf, dstlen, rc);
ut_error;
}
}
/******************************************************//**
Decrypt a merge block. */
static
bool
row_merge_decrypt_buf(
/*==================*/
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
ulint offset, /*!< in: offset where to
write */
ulint space, /*!< in: tablespace id */
const byte* input_buf, /*!< in: input buffer */
byte* crypted_buf) /*!< out: crypted buffer */
{
uint key_version;
uint dstlen=0;
os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset;
/* Read key_version from begining of the buffer */
key_version = mach_read_from_4((byte *)input_buf);
if (key_version == 0) {
/* block not encrypted */
return false;
}
int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE,
srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE,
crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen,
crypt_data, key_version,
space, ofs, 0);
if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt data-block "
" src: %p srclen: %lu buf: %p buflen: %d."
" return-code: %d. Can't continue!\n",
input_buf, (size_t)srv_sort_buf_size,
crypted_buf, dstlen, rc);
ut_error;
}
return (true);
}
#ifdef UNIV_DEBUG
/******************************************************//**
Display a merge tuple. */
@ -193,7 +281,7 @@ row_merge_buf_create(
ulint buf_size;
mem_heap_t* heap;
max_tuples = srv_sort_buf_size
max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)
/ ut_max(1, dict_index_get_min_size(index));
buf_size = (sizeof *buf);
@ -610,8 +698,8 @@ row_merge_buf_add(
ut_ad(data_size < srv_sort_buf_size);
/* Reserve one byte for the end marker of row_merge_block_t. */
if (buf->total_size + data_size >= srv_sort_buf_size - 1) {
/* Reserve bytes for the end marker of row_merge_block_t. */
if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) {
DBUG_RETURN(0);
}
@ -783,7 +871,7 @@ row_merge_buf_write(
{
const dict_index_t* index = buf->index;
ulint n_fields= dict_index_get_n_fields(index);
byte* b = &block[0];
byte* b = &block[ROW_MERGE_RESERVE_SIZE];
for (ulint i = 0; i < buf->n_tuples; i++) {
const mtuple_t* entry = &buf->tuples[i];
@ -802,7 +890,7 @@ row_merge_buf_write(
/* Write an "end-of-chunk" marker. */
ut_a(b < &block[srv_sort_buf_size]);
ut_a(b == &block[0] + buf->total_size);
ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE);
*b++ = 0;
#ifdef UNIV_DEBUG_VALGRIND
/* The rest of the block is uninitialized. Initialize it
@ -859,7 +947,10 @@ row_merge_read(
ulint offset, /*!< in: offset where to read
in number of row_merge_block_t
elements */
row_merge_block_t* buf) /*!< out: data */
row_merge_block_t* buf, /*!< out: data */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size;
ibool success;
@ -883,6 +974,13 @@ row_merge_read(
success = os_file_read_no_error_handling(OS_FILE_FROM_FD(fd), buf,
ofs, srv_sort_buf_size);
/* For encrypted tables, decrypt data after reading and copy data */
if (crypt_data && crypt_buf) {
if( row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) {
memcpy(buf, crypt_buf, srv_sort_buf_size);
}
}
#ifdef POSIX_FADV_DONTNEED
/* Each block is read exactly once. Free up the file cache. */
posix_fadvise(fd, ofs, srv_sort_buf_size, POSIX_FADV_DONTNEED);
@ -908,15 +1006,29 @@ row_merge_write(
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf) /*!< in: data */
const void* buf, /*!< in: data */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
void* crypt_buf, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
size_t buf_len = srv_sort_buf_size;
os_offset_t ofs = buf_len * (os_offset_t) offset;
ibool ret;
void* out_buf = (void *)buf;
DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE););
ret = os_file_write("(merge)", OS_FILE_FROM_FD(fd), buf, ofs, buf_len);
/* For encrypted tables, encrypt data before writing */
if (crypt_data && crypt_buf) {
row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf);
out_buf = crypt_buf;
} else {
/* Mark block unencrypted */
mach_write_to_4((byte *)out_buf, 0);
}
ret = os_file_write("(merge)", OS_FILE_FROM_FD(fd), out_buf, ofs, buf_len);
#ifdef UNIV_DEBUG
if (row_merge_print_block_write) {
@ -950,7 +1062,10 @@ row_merge_read_rec(
const mrec_t** mrec, /*!< out: pointer to merge record,
or NULL on end of list
(non-NULL on I/O error) */
ulint* offsets)/*!< out: offsets of mrec */
ulint* offsets,/*!< out: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint extra_size;
ulint data_size;
@ -968,6 +1083,10 @@ row_merge_read_rec(
ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE
+ dict_index_get_n_fields(index));
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
extra_size = *b++;
if (UNIV_UNLIKELY(!extra_size)) {
@ -987,7 +1106,8 @@ row_merge_read_rec(
/* Read another byte of extra_size. */
if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) {
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
err_exit:
/* Signal I/O error. */
*mrec = b;
@ -995,7 +1115,7 @@ err_exit:
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
}
extra_size = (extra_size & 0x7f) << 8;
@ -1016,13 +1136,14 @@ err_exit:
ut_ad(avail_size < sizeof *buf);
memcpy(*buf, b, avail_size);
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
goto err_exit;
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
/* Copy the record. */
memcpy(*buf + avail_size, b, extra_size - avail_size);
@ -1077,13 +1198,14 @@ err_exit:
offsets[3] = (ulint) index;
#endif /* UNIV_DEBUG */
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
goto err_exit;
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
/* Copy the rest of the record. */
memcpy(*buf + avail_size, b, extra_size + data_size - avail_size);
@ -1159,7 +1281,10 @@ row_merge_write_rec(
int fd, /*!< in: file descriptor */
ulint* foffs, /*!< in/out: file offset */
const mrec_t* mrec, /*!< in: record to write */
const ulint* offsets)/*!< in: offsets of mrec */
const ulint* offsets,/*!< in: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint extra_size;
ulint size;
@ -1180,6 +1305,10 @@ row_merge_write_rec(
size = extra_size + (extra_size >= 0x80)
+ rec_offs_data_size(offsets);
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) {
/* The record spans two blocks.
Copy it to the temporary buffer first. */
@ -1194,14 +1323,15 @@ row_merge_write_rec(
record to the head of the new block. */
memcpy(b, buf[0], avail_size);
if (!row_merge_write(fd, (*foffs)++, block)) {
if (!row_merge_write(fd, (*foffs)++, block,
crypt_data, crypt_block, space)) {
return(NULL);
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
/* Copy the rest. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
memcpy(b, buf[0] + avail_size, size - avail_size);
b += size - avail_size;
} else {
@ -1223,7 +1353,10 @@ row_merge_write_eof(
row_merge_block_t* block, /*!< in/out: file buffer */
byte* b, /*!< in: pointer to end of block */
int fd, /*!< in: file descriptor */
ulint* foffs) /*!< in/out: file offset */
ulint* foffs, /*!< in/out: file offset */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ut_ad(block);
ut_ad(b >= &block[0]);
@ -1236,20 +1369,27 @@ row_merge_write_eof(
}
#endif /* UNIV_DEBUG */
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
*b++ = 0;
UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]);
UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size);
#ifdef UNIV_DEBUG_VALGRIND
/* The rest of the block is uninitialized. Initialize it
to avoid bogus warnings. */
memset(b, 0xff, &block[srv_sort_buf_size] - b);
#endif /* UNIV_DEBUG_VALGRIND */
if (!row_merge_write(fd, (*foffs)++, block)) {
if (!row_merge_write(fd, (*foffs)++, block,
crypt_data, crypt_block, space)) {
return(NULL);
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
return(&block[0]);
}
@ -1294,7 +1434,11 @@ row_merge_read_clustered_index(
ULINT_UNDEFINED if none is added */
ib_sequence_t& sequence,/*!< in/out: autoinc sequence */
row_merge_block_t* block, /*!< in/out: file buffer */
float pct_cost) /*!< in: percent of task weight out of total alter job */
float pct_cost, /*!< in: percent of task weight
out of total alter job */
fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */
row_merge_block_t* crypt_block)/*!< in: in/out: crypted file
buffer */
{
dict_index_t* clust_index; /* Clustered index */
mem_heap_t* row_heap; /* Heap memory to create
@ -1316,9 +1460,10 @@ row_merge_read_clustered_index(
ib_int64_t sig_count = 0;
mem_heap_t* conv_heap = NULL;
float curr_progress;
float curr_progress = 0.0;
ib_int64_t read_rows = 0;
ib_int64_t table_total_rows;
ib_int64_t table_total_rows = 0;
DBUG_ENTER("row_merge_read_clustered_index");
ut_ad((old_table == new_table) == !col_map);
@ -1815,14 +1960,15 @@ write_buffers:
row_merge_buf_write(buf, file, block);
if (!row_merge_write(file->fd, file->offset++,
block)) {
if (!row_merge_write(file->fd, file->offset++, block,
crypt_data, crypt_block, new_table->space)) {
err = DB_TEMP_FILE_WRITE_FAILURE;
trx->error_key_num = i;
break;
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
merge_buf[i] = row_merge_buf_empty(buf);
if (UNIV_LIKELY(row != NULL)) {
@ -1991,14 +2137,21 @@ wait_again:
b2 = row_merge_write_rec(&block[2 * srv_sort_buf_size], \
&buf[2], b2, \
of->fd, &of->offset, \
mrec##N, offsets##N); \
mrec##N, offsets##N, \
crypt_data, \
&crypt_block[2 * srv_sort_buf_size], \
space); \
if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \
goto corrupt; \
} \
b##N = row_merge_read_rec(&block[N * srv_sort_buf_size],\
&buf[N], b##N, INDEX, \
file->fd, foffs##N, \
&mrec##N, offsets##N); \
&mrec##N, offsets##N, \
crypt_data, \
&crypt_block[N * srv_sort_buf_size], \
space); \
\
if (UNIV_UNLIKELY(!b##N)) { \
if (mrec##N) { \
goto corrupt; \
@ -2023,7 +2176,11 @@ row_merge_blocks(
source list in the file */
ulint* foffs1, /*!< in/out: offset of second
source list in the file */
merge_file_t* of) /*!< in/out: output file */
merge_file_t* of, /*!< in/out: output file */
fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */
row_merge_block_t* crypt_block,/*!< in: in/out: crypted file
buffer */
ulint space) /*!< in: space id */
{
mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */
@ -2054,8 +2211,10 @@ row_merge_blocks(
/* Write a record and read the next record. Split the output
file in two halves, which can be merged on the following pass. */
if (!row_merge_read(file->fd, *foffs0, &block[0])
|| !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size])) {
if (!row_merge_read(file->fd, *foffs0, &block[0],
crypt_data, &crypt_block[0], space)
|| !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size],
crypt_data, &crypt_block[srv_sort_buf_size], space)) {
corrupt:
mem_heap_free(heap);
return(DB_CORRUPTION);
@ -2067,11 +2226,15 @@ corrupt:
b0 = row_merge_read_rec(
&block[0], &buf[0], b0, dup->index,
file->fd, foffs0, &mrec0, offsets0);
file->fd, foffs0, &mrec0, offsets0,
crypt_data, &crypt_block[0], space);
b1 = row_merge_read_rec(
&block[srv_sort_buf_size],
&buf[srv_sort_buf_size], b1, dup->index,
file->fd, foffs1, &mrec1, offsets1);
file->fd, foffs1, &mrec1, offsets1,
crypt_data, &crypt_block[srv_sort_buf_size], space);
if (UNIV_UNLIKELY(!b0 && mrec0)
|| UNIV_UNLIKELY(!b1 && mrec1)) {
@ -2113,8 +2276,11 @@ done0:
done1:
mem_heap_free(heap);
b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size],
b2, of->fd, &of->offset);
b2, of->fd, &of->offset,
crypt_data, &crypt_block[2 * srv_sort_buf_size], space);
return(b2 ? DB_SUCCESS : DB_CORRUPTION);
}
@ -2129,7 +2295,10 @@ row_merge_blocks_copy(
const merge_file_t* file, /*!< in: input file */
row_merge_block_t* block, /*!< in/out: 3 buffers */
ulint* foffs0, /*!< in/out: input file offset */
merge_file_t* of) /*!< in/out: output file */
merge_file_t* of, /*!< in/out: output file */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */
@ -2156,7 +2325,8 @@ row_merge_blocks_copy(
/* Write a record and read the next record. Split the output
file in two halves, which can be merged on the following pass. */
if (!row_merge_read(file->fd, *foffs0, &block[0])) {
if (!row_merge_read(file->fd, *foffs0, &block[0],
crypt_data, &crypt_block[0], space)) {
corrupt:
mem_heap_free(heap);
return(FALSE);
@ -2167,7 +2337,9 @@ corrupt:
b2 = &block[2 * srv_sort_buf_size];
b0 = row_merge_read_rec(&block[0], &buf[0], b0, index,
file->fd, foffs0, &mrec0, offsets0);
file->fd, foffs0, &mrec0, offsets0,
crypt_data, &crypt_block[0], space);
if (UNIV_UNLIKELY(!b0 && mrec0)) {
goto corrupt;
@ -2186,8 +2358,10 @@ done0:
(*foffs0)++;
mem_heap_free(heap);
return(row_merge_write_eof(&block[2 * srv_sort_buf_size],
b2, of->fd, &of->offset)
b2, of->fd, &of->offset,
crypt_data, &crypt_block[2 * srv_sort_buf_size], space)
!= NULL);
}
@ -2207,9 +2381,12 @@ row_merge(
int* tmpfd, /*!< in/out: temporary file handle */
ulint* num_run,/*!< in/out: Number of runs remain
to be merged */
ulint* run_offset) /*!< in/out: Array contains the
ulint* run_offset, /*!< in/out: Array contains the
first offset number for each merge
run */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint foffs0; /*!< first input offset */
ulint foffs1; /*!< second input offset */
@ -2222,6 +2399,10 @@ row_merge(
UNIV_MEM_ASSERT_W(&block[0], 3 * srv_sort_buf_size);
if (crypt_block) {
UNIV_MEM_ASSERT_W(&crypt_block[0], 3 * srv_sort_buf_size);
}
ut_ad(ihalf < file->offset);
of.fd = *tmpfd;
@ -2252,7 +2433,8 @@ row_merge(
run_offset[n_run++] = of.offset;
error = row_merge_blocks(dup, file, block,
&foffs0, &foffs1, &of);
&foffs0, &foffs1, &of,
crypt_data, crypt_block, space);
if (error != DB_SUCCESS) {
return(error);
@ -2272,7 +2454,8 @@ row_merge(
run_offset[n_run++] = of.offset;
if (!row_merge_blocks_copy(dup->index, file, block,
&foffs0, &of)) {
&foffs0, &of,
crypt_data, crypt_block, space)) {
return(DB_CORRUPTION);
}
}
@ -2289,7 +2472,8 @@ row_merge(
run_offset[n_run++] = of.offset;
if (!row_merge_blocks_copy(dup->index, file, block,
&foffs1, &of)) {
&foffs1, &of,
crypt_data, crypt_block, space)) {
return(DB_CORRUPTION);
}
}
@ -2344,7 +2528,10 @@ row_merge_sort(
const float pct_progress,
/*!< in: total progress percent
until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
const ulint half = file->offset / 2;
ulint num_runs;
@ -2396,7 +2583,8 @@ row_merge_sort(
sql_print_information("InnoDB: Online DDL : merge-sorting current run %lu estimated %lu runs", cur_run, num_runs);
error = row_merge(trx, dup, file, block, tmpfd,
&num_runs, run_offset);
&num_runs, run_offset,
crypt_data, crypt_block, space);
if(update_progress) {
merge_count++;
@ -2479,7 +2667,11 @@ row_merge_insert_index_tuples(
row_merge_block_t* block, /*!< in/out: file buffer */
const ib_int64_t table_total_rows, /*!< in: total rows of old table */
const float pct_progress, /*!< in: total progress percent until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent
*/
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
const byte* b;
mem_heap_t* heap;
@ -2510,9 +2702,10 @@ row_merge_insert_index_tuples(
offsets[1] = dict_index_get_n_fields(index);
}
b = block;
b = &block[0];
if (!row_merge_read(fd, foffs, block)) {
if (!row_merge_read(fd, foffs, block,
crypt_data, crypt_block, space)) {
error = DB_CORRUPTION;
} else {
buf = static_cast<mrec_buf_t*>(
@ -2528,7 +2721,8 @@ row_merge_insert_index_tuples(
mtr_t mtr;
b = row_merge_read_rec(block, buf, b, index,
fd, &foffs, &mrec, offsets);
fd, &foffs, &mrec, offsets,
crypt_data, crypt_block, space);
if (UNIV_UNLIKELY(!b)) {
/* End of list, or I/O error */
if (mrec) {
@ -3669,6 +3863,7 @@ row_merge_build_indexes(
{
merge_file_t* merge_files;
row_merge_block_t* block;
row_merge_block_t* crypt_block;
ulint block_size;
ulint i;
ulint j;
@ -3679,6 +3874,7 @@ row_merge_build_indexes(
fts_psort_t* merge_info = NULL;
ib_int64_t sig_count = 0;
bool fts_psort_initiated = false;
fil_space_crypt_t * crypt_data = NULL;
float total_static_cost = 0;
float total_dynamic_cost = 0;
@ -3703,6 +3899,27 @@ row_merge_build_indexes(
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
/* Get crypt data from tablespace if present. */
crypt_data = fil_space_get_crypt_data(new_table->space);
crypt_block = NULL;
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
crypt_block = static_cast<row_merge_block_t*>(
os_mem_alloc_large(&block_size, FALSE));
if (crypt_block == NULL) {
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
} else {
/* Not needed */
crypt_data = NULL;
}
trx_start_if_not_started_xa(trx);
merge_files = static_cast<merge_file_t*>(
@ -3787,7 +4004,8 @@ row_merge_build_indexes(
trx, table, old_table, new_table, online, indexes,
fts_sort_idx, psort_info, merge_files, key_numbers,
n_indexes, add_cols, col_map,
add_autoinc, sequence, block, pct_cost);
add_autoinc, sequence, block, pct_cost,
crypt_data, crypt_block);
pct_progress += pct_cost;
@ -3807,6 +4025,10 @@ row_merge_build_indexes(
/* Now we have files containing index entries ready for
sorting and inserting. */
DBUG_EXECUTE_IF(
"ib_merge_wait_after_read",
os_thread_sleep(20000000);); /* 20 sec */
for (i = 0; i < n_indexes; i++) {
dict_index_t* sort_idx = indexes[i];
@ -3901,7 +4123,9 @@ wait_again:
error = row_merge_sort(
trx, &dup, &merge_files[i],
block, &tmpfd, true, pct_progress, pct_cost);
block, &tmpfd, true,
pct_progress, pct_cost,
crypt_data, crypt_block, new_table->space);
pct_progress += pct_cost;
@ -3909,6 +4133,10 @@ wait_again:
" merge-sorting index %s (%lu / %lu)",
buf, (i+1), n_indexes);
DBUG_EXECUTE_IF(
"ib_merge_wait_after_sort",
os_thread_sleep(20000000);); /* 20 sec */
if (error == DB_SUCCESS) {
pct_cost = (COST_BUILD_INDEX_STATIC +
(total_dynamic_cost * merge_files[i].offset /
@ -3924,7 +4152,8 @@ wait_again:
error = row_merge_insert_index_tuples(
trx->id, sort_idx, old_table,
merge_files[i].fd, block,
merge_files[i].n_rec, pct_progress, pct_cost);
merge_files[i].n_rec, pct_progress, pct_cost,
crypt_data, crypt_block, new_table->space);
pct_progress += pct_cost;
sql_print_information("InnoDB: Online DDL : "
@ -3998,6 +4227,10 @@ func_exit:
mem_free(merge_files);
os_mem_free_large(block, block_size);
if (crypt_block) {
os_mem_free_large(crypt_block, block_size);
}
DICT_TF2_FLAG_UNSET(new_table, DICT_TF2_FTS_ADD_DOC_ID);
if (online && old_table == new_table && error != DB_SUCCESS) {