mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-01-29 02:05:57 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

MDEV-11738: Mariadb uses 100% of several of my 8 cpus doing nothing

Browse source 
MDEV-11581: Mariadb starts InnoDB encryption threads
when key has not changed or data scrubbing turned off

Background: Key rotation is based on background threads
(innodb-encryption-threads) periodically going through
all tablespaces on fil_system. For each tablespace
current used key version is compared to max key age
(innodb-encryption-rotate-key-age). This process
naturally takes CPU. Similarly, in same time need for
scrubbing is investigated. Currently, key rotation
is fully supported on Amazon AWS key management plugin
only but InnoDB does not have knowledge what key
management plugin is used.

This patch re-purposes innodb-encryption-rotate-key-age=0
to disable key rotation and background data scrubbing.
All new tables are added to special list for key rotation
and key rotation is based on sending a event to
background encryption threads instead of using periodic
checking (i.e. timeout).

fil0fil.cc: Added functions fil_space_acquire_low()
to acquire a tablespace when it could be dropped concurrently.
This function is used from fil_space_acquire() or
fil_space_acquire_silent() that will not print
any messages if we try to acquire space that does not exist.
fil_space_release() to release a acquired tablespace.
fil_space_next() to iterate tablespaces in fil_system
using fil_space_acquire() and fil_space_release().
Similarly, fil_space_keyrotation_next() to iterate new
list fil_system->rotation_list where new tables.
are added if key rotation is disabled.
Removed unnecessary functions fil_get_first_space_safe()
fil_get_next_space_safe()

fil_node_open_file(): After page 0 is read read also
crypt_info if it is not yet read.

btr_scrub_lock_dict_func()
buf_page_check_corrupt()
buf_page_encrypt_before_write()
buf_merge_or_delete_for_page()
lock_print_info_all_transactions()
row_fts_psort_info_init()
row_truncate_table_for_mysql()
row_drop_table_for_mysql()
    Use fil_space_acquire()/release() to access fil_space_t.

buf_page_decrypt_after_read():
    Use fil_space_get_crypt_data() because at this point
    we might not yet have read page 0.

fil0crypt.cc/fil0fil.h: Lot of changes. Pass fil_space_t* directly
to functions needing it and store fil_space_t* to rotation state.
Use fil_space_acquire()/release() when iterating tablespaces
and removed unnecessary is_closing from fil_crypt_t. Use
fil_space_t::is_stopping() to detect when access to
tablespace should be stopped. Removed unnecessary
fil_space_get_crypt_data().

fil_space_create(): Inform key rotation that there could
be something to do if key rotation is disabled and new
table with encryption enabled is created.
Remove unnecessary functions fil_get_first_space_safe()
and fil_get_next_space_safe(). fil_space_acquire()
and fil_space_release() are used instead. Moved
fil_space_get_crypt_data() and fil_space_set_crypt_data()
to fil0crypt.cc.

fsp_header_init(): Acquire fil_space_t*, write crypt_data
and release space.

check_table_options()
	Renamed FIL_SPACE_ENCRYPTION_* TO FIL_ENCRYPTION_*

i_s.cc: Added ROTATING_OR_FLUSHING field to
information_schema.innodb_tablespace_encryption
to show current status of key rotation.
This commit is contained in:
Jan Lindström 2017-03-14 12:56:01 +02:00
parent a2f34809e5
commit 50eb40a2a8
58 changed files with 3220 additions and 2857 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
mysql-test/suite/encryption/r/debug_key_management.result
View file

@ -8,13 +8,13 @@ innodb_encryption_rotation_iops 100
innodb_encryption_threads 4
select space,name,current_key_version from information_schema.innodb_tablespaces_encryption order by space;
space name current_key_version
0 NULL 1
0 ./ibdata1 1
1 mysql/innodb_table_stats 1
2 mysql/innodb_index_stats 1
set global debug_key_management_version=10;
select space,name,current_key_version from information_schema.innodb_tablespaces_encryption order by space;
space name current_key_version
0 NULL 10
0 ./ibdata1 10
1 mysql/innodb_table_stats 10
2 mysql/innodb_index_stats 10
set global innodb_encrypt_tables=OFF;

35
mysql-test/suite/encryption/r/encrypt_and_grep.result
View file

@ -6,6 +6,16 @@ insert t1 values (repeat('foobar', 42));
insert t2 values (repeat('temp', 42));
insert t3 values (repeat('dummy', 42));
# Wait max 10 min for key encryption threads to encrypt all spaces
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
NAME
test/t3
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
NAME
mysql/innodb_table_stats
mysql/innodb_index_stats
test/t1
test/t2
./ibdata1
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
@ -15,13 +25,23 @@ FOUND /dummy/ in t3.ibd
# ibdata1 expecting NOT FOUND
NOT FOUND /foobar/ in ibdata1
# Now turn off encryption and wait for threads to decrypt everything
SET GLOBAL innodb_encryption_threads = 4;
SET GLOBAL innodb_encryption_threads = 1;
SET GLOBAL innodb_encrypt_tables = off;
# Wait max 10 min for key encryption threads to decrypt all spaces
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
NAME
mysql/innodb_table_stats
mysql/innodb_index_stats
test/t2
test/t3
./ibdata1
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
NAME
test/t1
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting FOUND
FOUND /temp/ in t2.ibd
NOT FOUND /temp/ in t2.ibd
# t3 no on expecting FOUND
FOUND /dummy/ in t3.ibd
# ibdata1 expecting NOT FOUND
@ -30,6 +50,16 @@ NOT FOUND /foobar/ in ibdata1
SET GLOBAL innodb_encryption_threads = 4;
SET GLOBAL innodb_encrypt_tables = on;
# Wait max 10 min for key encryption threads to encrypt all spaces
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
NAME
test/t3
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
NAME
mysql/innodb_table_stats
mysql/innodb_index_stats
test/t1
test/t2
./ibdata1
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
@ -38,5 +68,4 @@ NOT FOUND /temp/ in t2.ibd
FOUND /dummy/ in t3.ibd
# ibdata1 expecting NOT FOUND
NOT FOUND /foobar/ in ibdata1
# TODO: add shutdown + grep tests
drop table t1, t2, t3;

66
mysql-test/suite/encryption/r/innodb-key-rotation-disable.result Normal file
View file

@ -0,0 +1,66 @@
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
NAME
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
NAME
mysql/innodb_table_stats
mysql/innodb_index_stats
./ibdata1
create database enctests;
use enctests;
create table t1(a int not null primary key, b char(200)) engine=innodb;
create table t2(a int not null primary key, b char(200)) engine=innodb row_format=compressed;
create table t3(a int not null primary key, b char(200)) engine=innodb page_compressed=yes;
create table t4(a int not null primary key, b char(200)) engine=innodb encrypted=yes;
create table t5(a int not null primary key, b char(200)) engine=innodb encrypted=yes row_format=compressed;
create table t6(a int not null primary key, b char(200)) engine=innodb encrypted=yes page_compressed=yes;
create table t7(a int not null primary key, b char(200)) engine=innodb encrypted=no;
create table t8(a int not null primary key, b char(200)) engine=innodb encrypted=no row_format=compressed;
create table t9(a int not null primary key, b char(200)) engine=innodb encrypted=no page_compressed=yes;
insert into t1 values (1, 'secredmessage');
insert into t2 values (1, 'secredmessage');
insert into t3 values (1, 'secredmessagecompressedaaaaaaaaabbbbbbbbbbbbbbccccccccccccccc');
insert into t4 values (1, 'secredmessage');
insert into t5 values (1, 'secredmessage');
insert into t6 values (1, 'secredmessagecompressedaaaaaaaaabbbbbbbbbbbbbbccccccccccccccc');
insert into t7 values (1, 'publicmessage');
insert into t8 values (1, 'publicmessage');
insert into t9 values (1, 'pugliccompressedaaaaaaaaabbbbbbbbbbbbbbccccccccccccccc');
# should list tables t1-t6
SELECT NAME,ENCRYPTION_SCHEME,CURRENT_KEY_ID FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'enctests%';
NAME ENCRYPTION_SCHEME CURRENT_KEY_ID
enctests/t1 1 1
enctests/t2 1 1
enctests/t3 1 1
enctests/t4 1 1
enctests/t5 1 1
enctests/t6 1 1
# should list tables t7-t9
SELECT NAME,ENCRYPTION_SCHEME,CURRENT_KEY_ID FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 and NAME LIKE 'enctests%';
NAME ENCRYPTION_SCHEME CURRENT_KEY_ID
enctests/t7 0 1
enctests/t8 0 1
enctests/t9 0 1
SET GLOBAL innodb_encrypt_tables=OFF;
ERROR 42000: Variable 'innodb_encrypt_tables' can't be set to the value of 'OFF'
SET GLOBAL innodb_encrypt_tables=ON;
ERROR 42000: Variable 'innodb_encrypt_tables' can't be set to the value of 'ON'
# t1 default on expecting NOT FOUND
NOT FOUND /secred/ in t1.ibd
# t2 default on expecting NOT FOUND
NOT FOUND /secred/ in t2.ibd
# t3 default on expecting NOT FOUND
NOT FOUND /secred/ in t3.ibd
# t4 on expecting NOT FOUND
NOT FOUND /secred/ in t4.ibd
# t5 on expecting NOT FOUND
NOT FOUND /secred/ in t5.ibd
# t6 on expecting NOT FOUND
NOT FOUND /secred/ in t6.ibd
# t7 off expecting FOUND
FOUND /public/ in t7.ibd
# t8 row compressed expecting NOT FOUND
FOUND /public/ in t8.ibd
# t9 page compressed expecting NOT FOUND
NOT FOUND /public/ in t9.ibd
use test;
drop database enctests;

1
mysql-test/suite/encryption/r/innodb_encryption.result
View file

@ -17,6 +17,7 @@ CURRENT_KEY_VERSION int(11) unsigned NO 0
KEY_ROTATION_PAGE_NUMBER bigint(21) unsigned YES NULL
KEY_ROTATION_MAX_PAGE_NUMBER bigint(21) unsigned YES NULL
CURRENT_KEY_ID int(11) unsigned NO 0
ROTATING_OR_FLUSHING int(1) unsigned NO 0
# Wait max 5 min for key encryption threads to encrypt one space
# Success!
# Wait max 10 min for key encryption threads to encrypt all space

41
mysql-test/suite/encryption/r/innodb_onlinealter_encryption.result
View file

@ -24,7 +24,7 @@ end while;
end//
commit;
set autocommit=0;
call innodb_insert_proc(15000);
call innodb_insert_proc(1500);
commit;
set autocommit=1;
# Wait max 10 min for key encryption threads to encrypt all spaces
@ -42,6 +42,8 @@ NOT FOUND /author/ in t5.ibd
NOT FOUND /mangled/ in t6.ibd
# t7 ... on expecting NOT FOUND
NOT FOUND /mysql/ in t7.ibd
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
ALTER TABLE t1 ADD COLUMN b int default 2;
ALTER TABLE t2 ADD COLUMN b int default 2;
ALTER TABLE t7 ADD COLUMN b int default 2;
@ -135,42 +137,5 @@ NOT FOUND /author/ in t5.ibd
NOT FOUND /mangled/ in t6.ibd
# t7 ... on expecting NOT FOUND
NOT FOUND /mysql/ in t7.ibd
# Restarting server
# Done restarting server
select count(1) from t1;
count(1)
15000
select count(1) from t2;
count(1)
15000
select count(1) from t3;
count(1)
15000
select count(1) from t4;
count(1)
15000
select count(1) from t5;
count(1)
15000
select count(1) from t6;
count(1)
15000
select count(1) from t7;
count(1)
15000
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# t4 ... on expecting NOT FOUND
NOT FOUND /repeat/ in t4.ibd
# t5 ... on expecting NOT FOUND
NOT FOUND /author/ in t5.ibd
# t6 ... on expecting NOT FOUND
NOT FOUND /mangled/ in t6.ibd
# t7 ... on expecting NOT FOUND
NOT FOUND /mysql/ in t7.ibd
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3, t4, t5, t6, t7;

3
mysql-test/suite/encryption/t/encrypt_and_grep.opt
View file

@ -1,8 +1,7 @@
--innodb-encrypt-tables=ON
--innodb-encrypt-log=ON
--innodb-encryption-rotate-key-age=15
--innodb-encryption-threads=4
--innodb-encryption-threads=1
--innodb-tablespaces-encryption
--innodb-max-dirty-pages-pct=0.001

29
mysql-test/suite/encryption/t/encrypt_and_grep.test
View file

@ -1,5 +1,5 @@
-- source include/have_innodb.inc
-- source include/have_example_key_management_plugin.inc
-- source include/have_file_key_management_plugin.inc
# embedded does not support restart
-- source include/not_embedded.inc
@ -30,7 +30,10 @@ insert t3 values (repeat('dummy', 42));
--let $wait_condition=SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0
--source include/wait_condition.inc
--sleep 5
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
--source include/shutdown_mysqld.inc
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
@ -49,15 +52,21 @@ insert t3 values (repeat('dummy', 42));
-- let SEARCH_FILE=$ib1_IBD
-- source include/search_pattern_in_file.inc
-- source include/start_mysqld.inc
--echo # Now turn off encryption and wait for threads to decrypt everything
SET GLOBAL innodb_encryption_threads = 4;
SET GLOBAL innodb_encryption_threads = 1;
SET GLOBAL innodb_encrypt_tables = off;
--echo # Wait max 10 min for key encryption threads to decrypt all spaces
--let $wait_timeout= 600
--let $wait_condition=SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
--let $wait_condition=SELECT COUNT(*) = 5 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND KEY_ROTATION_PAGE_NUMBER IS NULL;
--source include/wait_condition.inc
--sleep 5
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
--source include/shutdown_mysqld.inc
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
@ -76,6 +85,8 @@ SET GLOBAL innodb_encrypt_tables = off;
-- let SEARCH_FILE=$ib1_IBD
-- source include/search_pattern_in_file.inc
-- source include/start_mysqld.inc
--echo # Now turn on encryption and wait for threads to encrypt all spaces
SET GLOBAL innodb_encryption_threads = 4;
SET GLOBAL innodb_encrypt_tables = on;
@ -84,7 +95,11 @@ SET GLOBAL innodb_encrypt_tables = on;
--let $wait_timeout= 600
--let $wait_condition=SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
--source include/wait_condition.inc
--sleep 5
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
--source include/shutdown_mysqld.inc
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
@ -103,6 +118,6 @@ SET GLOBAL innodb_encrypt_tables = on;
-- let SEARCH_FILE=$ib1_IBD
-- source include/search_pattern_in_file.inc
--echo # TODO: add shutdown + grep tests
-- source include/start_mysqld.inc
drop table t1, t2, t3;

5
mysql-test/suite/encryption/t/innodb-key-rotation-disable.opt Normal file
View file

@ -0,0 +1,5 @@
--innodb-encrypt-tables
--innodb-encrypt-log
--innodb-encryption-rotate-key-age=0
--innodb-encryption-threads=4
--innodb-tablespaces-encryption

102
mysql-test/suite/encryption/t/innodb-key-rotation-disable.test Normal file
View file

@ -0,0 +1,102 @@
-- source include/have_innodb.inc
-- source include/have_file_key_management_plugin.inc
# not embedded because of restarts
-- source include/not_embedded.inc
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
--disable_query_log
--disable_warnings
let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`;
let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
let $encryption = `SELECT @@innodb_encrypt_tables`;
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
# zlib
set global innodb_compression_algorithm = 1;
--enable_warnings
--enable_query_log
create database enctests;
use enctests;
create table t1(a int not null primary key, b char(200)) engine=innodb;
create table t2(a int not null primary key, b char(200)) engine=innodb row_format=compressed;
create table t3(a int not null primary key, b char(200)) engine=innodb page_compressed=yes;
create table t4(a int not null primary key, b char(200)) engine=innodb encrypted=yes;
create table t5(a int not null primary key, b char(200)) engine=innodb encrypted=yes row_format=compressed;
create table t6(a int not null primary key, b char(200)) engine=innodb encrypted=yes page_compressed=yes;
create table t7(a int not null primary key, b char(200)) engine=innodb encrypted=no;
create table t8(a int not null primary key, b char(200)) engine=innodb encrypted=no row_format=compressed;
create table t9(a int not null primary key, b char(200)) engine=innodb encrypted=no page_compressed=yes;
insert into t1 values (1, 'secredmessage');
insert into t2 values (1, 'secredmessage');
insert into t3 values (1, 'secredmessagecompressedaaaaaaaaabbbbbbbbbbbbbbccccccccccccccc');
insert into t4 values (1, 'secredmessage');
insert into t5 values (1, 'secredmessage');
insert into t6 values (1, 'secredmessagecompressedaaaaaaaaabbbbbbbbbbbbbbccccccccccccccc');
insert into t7 values (1, 'publicmessage');
insert into t8 values (1, 'publicmessage');
insert into t9 values (1, 'pugliccompressedaaaaaaaaabbbbbbbbbbbbbbccccccccccccccc');
--echo # should list tables t1-t6
SELECT NAME,ENCRYPTION_SCHEME,CURRENT_KEY_ID FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'enctests%';
--echo # should list tables t7-t9
SELECT NAME,ENCRYPTION_SCHEME,CURRENT_KEY_ID FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 and NAME LIKE 'enctests%';
--error ER_WRONG_VALUE_FOR_VAR
SET GLOBAL innodb_encrypt_tables=OFF;
--error ER_WRONG_VALUE_FOR_VAR
SET GLOBAL innodb_encrypt_tables=ON;
--let $MYSQLD_DATADIR=`select @@datadir`
-- source include/shutdown_mysqld.inc
--let SEARCH_RANGE = 10000000
--let SEARCH_PATTERN=secred
--echo # t1 default on expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/enctests/t1.ibd
-- source include/search_pattern_in_file.inc
--echo # t2 default on expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/enctests/t2.ibd
-- source include/search_pattern_in_file.inc
--echo # t3 default on expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/enctests/t3.ibd
-- source include/search_pattern_in_file.inc
--echo # t4 on expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/enctests/t4.ibd
-- source include/search_pattern_in_file.inc
--echo # t5 on expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/enctests/t5.ibd
-- source include/search_pattern_in_file.inc
--echo # t6 on expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/enctests/t6.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=public
--echo # t7 off expecting FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/enctests/t7.ibd
-- source include/search_pattern_in_file.inc
--echo # t8 row compressed expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/enctests/t8.ibd
-- source include/search_pattern_in_file.inc
--echo # t9 page compressed expecting NOT FOUND
-- let SEARCH_FILE=$MYSQLD_DATADIR/enctests/t9.ibd
-- source include/search_pattern_in_file.inc
-- source include/start_mysqld.inc
use test;
drop database enctests;
# reset system
--disable_query_log
--disable_warnings
EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig;
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
set global innodb_compression_algorithm = DEFAULT;
--enable_warnings
--enable_query_log

116
mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test
View file

@ -1,26 +1,12 @@
-- source include/have_innodb.inc
-- source include/have_example_key_management_plugin.inc
-- source include/not_valgrind.inc
-- source include/have_file_key_management_plugin.inc
# test uses restart
-- source include/not_embedded.inc
-- source include/not_windows.inc
--disable_query_log
let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
--enable_query_log
--disable_warnings
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
--let $MYSQLD_DATADIR=`select @@datadir`
--let SEARCH_RANGE = 10000000
--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd
--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd
--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd
--let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd
--let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd
--let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd
--let t7_IBD = $MYSQLD_DATADIR/test/t7.ibd
--enable_warnings
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB;
@ -50,7 +36,7 @@ delimiter ;//
commit;
set autocommit=0;
call innodb_insert_proc(15000);
call innodb_insert_proc(1500);
commit;
set autocommit=1;
@ -58,37 +44,48 @@ set autocommit=1;
--let $wait_timeout= 600
--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0
--source include/wait_condition.inc
--let $MYSQLD_DATADIR=`select @@datadir`
--sleep 10
--source include/shutdown_mysqld.inc
--source include/wait_until_disconnected.inc
--let SEARCH_RANGE = 10000000
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t1.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t2.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t3.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=repeat
--echo # t4 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t4.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=author
--echo # t5 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t5_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t5.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mangled
--echo # t6 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t6_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t6.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t7 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t7_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t7.ibd
-- source include/search_pattern_in_file.inc
-- source include/start_mysqld.inc
--disable_warnings
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
--enable_warnings
ALTER TABLE t1 ADD COLUMN b int default 2;
ALTER TABLE t2 ADD COLUMN b int default 2;
ALTER TABLE t7 ADD COLUMN b int default 2;
@ -110,82 +107,39 @@ SHOW CREATE TABLE t5;
SHOW CREATE TABLE t6;
SHOW CREATE TABLE t7;
--sleep 10
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=repeat
--echo # t4 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=author
--echo # t5 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t5_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mangled
--echo # t6 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t6_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t7 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t7_IBD
-- source include/search_pattern_in_file.inc
--echo # Restarting server
-- source include/restart_mysqld.inc
--echo # Done restarting server
select count(1) from t1;
select count(1) from t2;
select count(1) from t3;
select count(1) from t4;
select count(1) from t5;
select count(1) from t6;
select count(1) from t7;
--source include/shutdown_mysqld.inc
--source include/wait_until_disconnected.inc
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t1.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t2.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t3.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=repeat
--echo # t4 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t4.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=author
--echo # t5 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t5_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t5.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mangled
--echo # t6 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t6_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t6.ibd
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t7 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t7_IBD
-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t7.ibd
-- source include/search_pattern_in_file.inc
-- source include/start_mysqld.inc
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3, t4, t5, t6, t7;
# reset system
--disable_query_log
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
--enable_query_log

2
mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result
View file

@ -377,7 +377,7 @@ space_id page_id start_lsn end_lsn
Warnings:
Warning 1012 InnoDB: SELECTing from INFORMATION_SCHEMA.innodb_changed_pages but the InnoDB storage engine is not installed
select * from information_schema.innodb_tablespaces_encryption;
SPACE NAME ENCRYPTION_SCHEME KEYSERVER_REQUESTS MIN_KEY_VERSION CURRENT_KEY_VERSION KEY_ROTATION_PAGE_NUMBER KEY_ROTATION_MAX_PAGE_NUMBER CURRENT_KEY_ID
SPACE NAME ENCRYPTION_SCHEME KEYSERVER_REQUESTS MIN_KEY_VERSION CURRENT_KEY_VERSION KEY_ROTATION_PAGE_NUMBER KEY_ROTATION_MAX_PAGE_NUMBER CURRENT_KEY_ID ROTATING_OR_FLUSHING
Warnings:
Warning 1012 InnoDB: SELECTing from INFORMATION_SCHEMA.innodb_tablespaces_encryption but the InnoDB storage engine is not installed
select * from information_schema.innodb_tablespaces_scrubbing;

2
mysql-test/suite/sys_vars/r/sysvars_innodb.result
View file

@ -782,7 +782,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME
DEFAULT_VALUE 1
VARIABLE_SCOPE GLOBAL
VARIABLE_TYPE INT UNSIGNED
VARIABLE_COMMENT Key rotation - re-encrypt in background all pages that were encrypted with a key that many (or more) versions behind
VARIABLE_COMMENT Key rotation - re-encrypt in background all pages that were encrypted with a key that many (or more) versions behind. Value 0 indicates that key rotation is disabled.
NUMERIC_MIN_VALUE 0
NUMERIC_MAX_VALUE 4294967295
NUMERIC_BLOCK_SIZE 0

48
storage/innobase/btr/btr0scrub.cc
View file

@ -111,11 +111,18 @@ log_scrub_failure(
Lock dict mutexes */
static
bool
btr_scrub_lock_dict_func(ulint space, bool lock_to_close_table,
btr_scrub_lock_dict_func(ulint space_id, bool lock_to_close_table,
const char * file, uint line)
{
uint start = time(0);
uint last = start;
time_t start = time(0);
time_t last = start;
/* FIXME: this is not the proper way of doing things. The
dict_sys->mutex should not be held by any thread for longer
than a few microseconds. It must not be held during I/O,
for example. So, what is the purpose for this busy-waiting?
This function should be rewritten as part of MDEV-8139:
Fix scrubbing tests. */
while (mutex_enter_nowait_func(&(dict_sys->mutex), file, line)) {
/* if we lock to close a table, we wait forever
@ -123,19 +130,24 @@ btr_scrub_lock_dict_func(ulint space, bool lock_to_close_table,
* is closing, and then instead give up
*/
if (lock_to_close_table == false) {
if (fil_crypt_is_closing(space)) {
fil_space_t* space = fil_space_acquire(space_id);
if (!space || space->stop_new_ops) {
if (space) {
fil_space_release(space);
}
return false;
}
fil_space_release(space);
}
os_thread_sleep(250000);
uint now = time(0);
time_t now = time(0);
if (now >= last + 30) {
fprintf(stderr,
"WARNING: %s:%u waited %u seconds for"
"WARNING: %s:%u waited %ld seconds for"
" dict_sys lock, space: %lu"
" lock_to_close_table: %u\n",
file, line, now - start, space,
" lock_to_close_table: %d\n",
file, line, now - start, space_id,
lock_to_close_table);
last = now;
@ -181,16 +193,24 @@ void
btr_scrub_table_close_for_thread(
btr_scrub_t *scrub_data)
{
if (scrub_data->current_table == NULL)
if (scrub_data->current_table == NULL) {
return;
}
bool lock_for_close = true;
btr_scrub_lock_dict(scrub_data->space, lock_for_close);
fil_space_t* space = fil_space_acquire(scrub_data->space);
/* perform the actual closing */
btr_scrub_table_close(scrub_data->current_table);
/* If tablespace is not marked as stopping perform
the actual close. */
if (space && !space->is_stopping()) {
mutex_enter(&dict_sys->mutex);
/* perform the actual closing */
btr_scrub_table_close(scrub_data->current_table);
mutex_exit(&dict_sys->mutex);
}
btr_scrub_unlock_dict();
if (space) {
fil_space_release(space);
}
scrub_data->current_table = NULL;
scrub_data->current_index = NULL;

78
storage/innobase/buf/buf0buf.cc
View file

@ -626,7 +626,7 @@ buf_page_is_corrupted(
ulint checksum_field2;
ulint space_id = mach_read_from_4(
read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint page_type = mach_read_from_4(
ulint page_type = mach_read_from_2(
read_buf + FIL_PAGE_TYPE);
/* We can trust page type if page compression is set on tablespace
@ -4480,11 +4480,14 @@ buf_page_check_corrupt(
byte* dst_frame = (zip_size) ? bpage->zip.data :
((buf_block_t*) bpage)->frame;
ulint space_id = bpage->space;
fil_space_t* space = fil_space_found_by_id(space_id);
fil_space_crypt_t* crypt_data = space->crypt_data;
fil_space_t* space = fil_space_acquire_silent(space_id);
bool still_encrypted = false;
bool corrupted = false;
ulint page_type = mach_read_from_2(dst_frame + FIL_PAGE_TYPE);
fil_space_crypt_t* crypt_data = NULL;
ut_ad(space);
crypt_data = space->crypt_data;
/* In buf_decrypt_after_read we have either decrypted the page if
page post encryption checksum matches and used key_id is found
@ -4513,7 +4516,7 @@ buf_page_check_corrupt(
ib_logf(IB_LOG_LEVEL_ERROR,
"%s: Block in space_id " ULINTPF " in file %s corrupted.",
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ? "Maybe corruption" : "Corruption",
space_id, space->name ? space->name : "NULL");
space_id, (space && space->name) ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"Based on page type %s (" ULINTPF ")",
fil_get_page_type_name(page_type), page_type);
@ -4523,7 +4526,7 @@ buf_page_check_corrupt(
ib_logf(IB_LOG_LEVEL_ERROR,
"Block in space_id " ULINTPF " in file %s encrypted.",
space_id, space->name ? space->name : "NULL");
space_id, (space && space->name) ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"However key management plugin or used key_version %u is not found or"
" used encryption algorithm or method does not match.",
@ -4535,6 +4538,10 @@ buf_page_check_corrupt(
}
}
if (space) {
fil_space_release(space);
}
return corrupted;
}
@ -6088,13 +6095,6 @@ buf_page_encrypt_before_write(
byte* src_frame,
ulint space_id)
{
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
ulint zip_size = buf_page_get_zip_size(bpage);
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
bool page_compressed = fil_space_is_page_compressed(bpage->space);
bool encrypted = true;
bpage->real_size = UNIV_PAGE_SIZE;
fil_page_type_validate(src_frame);
@ -6111,7 +6111,20 @@ buf_page_encrypt_before_write(
return src_frame;
}
if (crypt_data != NULL && crypt_data->not_encrypted()) {
fil_space_t* space = fil_space_acquire_silent(space_id);
/* Tablespace must exist during write operation */
if (!space) {
/* This could be true on discard if we have injected a error
case e.g. in innodb.innodb-wl5522-debug-zip so that space
is already marked as stop_new_ops = true. */
return src_frame;
}
fil_space_crypt_t* crypt_data = space->crypt_data;
bool encrypted = true;
if (space->crypt_data != NULL && space->crypt_data->not_encrypted()) {
/* Encryption is disabled */
encrypted = false;
}
@ -6128,11 +6141,17 @@ buf_page_encrypt_before_write(
encrypted = false;
}
bool page_compressed = fil_space_is_page_compressed(bpage->space);
if (!encrypted && !page_compressed) {
/* No need to encrypt or page compress the page */
fil_space_release(space);
return src_frame;
}
ulint zip_size = buf_page_get_zip_size(bpage);
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
/* Find free slot from temporary memory array */
buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
slot->out_buf = NULL;
@ -6142,11 +6161,10 @@ buf_page_encrypt_before_write(
if (!page_compressed) {
/* Encrypt page content */
byte* tmp = fil_space_encrypt(bpage->space,
byte* tmp = fil_space_encrypt(space,
bpage->offset,
bpage->newest_modification,
src_frame,
zip_size,
dst_frame);
ulint key_version = mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
@ -6184,11 +6202,10 @@ buf_page_encrypt_before_write(
if(encrypted) {
/* And then we encrypt the page content */
tmp = fil_space_encrypt(bpage->space,
tmp = fil_space_encrypt(space,
bpage->offset,
bpage->newest_modification,
tmp,
zip_size,
dst_frame);
}
@ -6199,6 +6216,7 @@ buf_page_encrypt_before_write(
fil_page_type_validate(dst_frame);
#endif
fil_space_release(space);
// return dst_frame which will be written
return dst_frame;
}
@ -6224,17 +6242,6 @@ buf_page_decrypt_after_read(
bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame);
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
bool success = true;
ulint space_id = mach_read_from_4(
dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
/* Page is encrypted if encryption information is found from
tablespace and page contains used key_version. This is true
also for pages first compressed and then encrypted. */
if (!crypt_data) {
key_version = 0;
}
bpage->key_version = key_version;
@ -6243,6 +6250,16 @@ buf_page_decrypt_after_read(
return (true);
}
fil_space_t* space = fil_space_acquire(bpage->space);
fil_space_crypt_t* crypt_data = space->crypt_data;
/* Page is encrypted if encryption information is found from
tablespace and page contains used key_version. This is true
also for pages first compressed and then encrypted. */
if (!crypt_data) {
key_version = 0;
}
if (page_compressed) {
/* the page we read is unencrypted */
/* Find free slot from temporary memory array */
@ -6284,15 +6301,15 @@ buf_page_decrypt_after_read(
#endif
/* decrypt using crypt_buf to dst_frame */
byte* res = fil_space_decrypt(bpage->space,
byte* res = fil_space_decrypt(space,
slot->crypt_buf,
size,
dst_frame,
&bpage->encrypted);
if (!res) {
success = false;
}
#ifdef UNIV_DEBUG
fil_page_type_validate(dst_frame);
#endif
@ -6323,5 +6340,6 @@ buf_page_decrypt_after_read(
}
}
fil_space_release(space);
return (success);
}

1141
storage/innobase/fil/fil0crypt.cc
View file

File diff suppressed because it is too large Load diff

432
storage/innobase/fil/fil0fil.cc
View file

@ -156,7 +156,11 @@ UNIV_INTERN mysql_pfs_key_t fil_space_latch_key;
/** The tablespace memory cache. This variable is NULL before the module is
initialized. */
fil_system_t* fil_system = NULL;
UNIV_INTERN fil_system_t* fil_system = NULL;
/** At this age or older a space/page will be rotated */
UNIV_INTERN extern uint srv_fil_crypt_rotate_key_age;
UNIV_INTERN extern ib_mutex_t fil_crypt_threads_mutex;
/** Determine if (i) is a user tablespace id or not. */
# define fil_is_user_tablespace_id(i) ((i) > srv_undo_tablespaces_open)
@ -613,6 +617,7 @@ fil_node_open_file(
node->handle = os_file_create_simple_no_error_handling(
innodb_file_data_key, node->name, OS_FILE_OPEN,
OS_FILE_READ_ONLY, &success, 0);
if (!success) {
/* The following call prints an error message */
os_file_get_last_error(true);
@ -664,6 +669,16 @@ fil_node_open_file(
const ulint space_id = fsp_header_get_space_id(page);
ulint flags = fsp_header_get_flags(page);
/* Try to read crypt_data from page 0 if it is not yet
read. */
if (!node->space->page_0_crypt_read) {
ulint offset = fsp_header_get_crypt_offset(
fsp_flags_get_zip_size(flags));
ut_ad(node->space->crypt_data == NULL);
node->space->crypt_data = fil_space_read_crypt_data(space_id, page, offset);
node->space->page_0_crypt_read = true;
}
ut_free(buf2);
os_file_close(node->handle);
@ -1452,17 +1467,24 @@ fil_space_truncate_start(
/*******************************************************************//**
Creates a space memory object and puts it to the 'fil system' hash table.
If there is an error, prints an error message to the .err log.
@param[in] name Space name
@param[in] id Space id
@param[in] flags Tablespace flags
@param[in] purpose FIL_TABLESPACE or FIL_LOG if log
@param[in] crypt_data Encryption information
@param[in] create_table True if this is create table
@param[in] mode Encryption mode
@return TRUE if success */
UNIV_INTERN
ibool
bool
fil_space_create(
/*=============*/
const char* name, /*!< in: space name */
ulint id, /*!< in: space id */
ulint flags, /*!< in: tablespace flags */
ulint purpose,/*!< in: FIL_TABLESPACE, or FIL_LOG if log */
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
bool create_table) /*!< in: true if create table */
const char* name,
ulint id,
ulint flags,
ulint purpose,
fil_space_crypt_t* crypt_data,
bool create_table,
fil_encryption_t mode)
{
fil_space_t* space;
@ -1486,7 +1508,7 @@ fil_space_create(
mutex_exit(&fil_system->mutex);
return(FALSE);
return(false);
}
ib_logf(IB_LOG_LEVEL_WARN,
@ -1513,7 +1535,7 @@ fil_space_create(
mutex_exit(&fil_system->mutex);
return(FALSE);
return(false);
}
space = static_cast<fil_space_t*>(mem_zalloc(sizeof(*space)));
@ -1544,7 +1566,6 @@ fil_space_create(
space->flags = flags;
space->magic_n = FIL_SPACE_MAGIC_N;
space->printed_compression_failure = false;
space->crypt_data = crypt_data;
/* In create table we write page 0 so we have already
@ -1569,14 +1590,28 @@ fil_space_create(
HASH_INSERT(fil_space_t, name_hash, fil_system->name_hash,
ut_fold_string(name), space);
space->is_in_unflushed_spaces = false;
UT_LIST_ADD_LAST(space_list, fil_system->space_list, space);
mutex_exit(&fil_system->mutex);
/* Inform key rotation that there could be something
to do */
if (purpose == FIL_TABLESPACE && !srv_fil_crypt_rotate_key_age && fil_crypt_threads_event &&
(mode == FIL_ENCRYPTION_ON || mode == FIL_ENCRYPTION_OFF ||
srv_encrypt_tables)) {
/* Key rotation is not enabled, need to inform background
encryption threads. */
UT_LIST_ADD_LAST(rotation_list, fil_system->rotation_list, space);
mutex_exit(&fil_system->mutex);
space->is_in_rotation_list = true;
mutex_enter(&fil_crypt_threads_mutex);
os_event_set(fil_crypt_threads_event);
mutex_exit(&fil_crypt_threads_mutex);
} else {
mutex_exit(&fil_system->mutex);
}
return(TRUE);
return(true);
}
/*******************************************************************//**
@ -1688,6 +1723,11 @@ fil_space_free(
space);
}
if (space->is_in_rotation_list) {
space->is_in_rotation_list = false;
UT_LIST_REMOVE(rotation_list, fil_system->rotation_list, space);
}
UT_LIST_REMOVE(space_list, fil_system->space_list, space);
ut_a(space->magic_n == FIL_SPACE_MAGIC_N);
@ -2344,7 +2384,6 @@ fil_read_first_page(
const char* check_msg = NULL;
fil_space_crypt_t* cdata;
buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
/* Align the memory for a possible read from a raw device */
@ -2383,7 +2422,7 @@ fil_read_first_page(
ulint space = fsp_header_get_space_id(page);
ulint offset = fsp_header_get_crypt_offset(
fsp_flags_get_zip_size(*flags), NULL);
fsp_flags_get_zip_size(*flags));
cdata = fil_space_read_crypt_data(space, page, offset);
@ -2768,7 +2807,7 @@ fil_op_log_parse_or_replay(
space_id, name, path, flags,
DICT_TF2_USE_TABLESPACE,
FIL_IBD_FILE_INITIAL_SIZE,
FIL_SPACE_ENCRYPTION_DEFAULT,
FIL_ENCRYPTION_DEFAULT,
FIL_DEFAULT_ENCRYPTION_KEY) != DB_SUCCESS) {
ut_error;
}
@ -2892,16 +2931,27 @@ fil_check_pending_operations(
*space = 0;
/* Wait for crypt threads to stop accessing space */
fil_space_crypt_close_tablespace(id);
mutex_enter(&fil_system->mutex);
fil_space_t* sp = fil_space_get_by_id(id);
if (sp) {
sp->stop_new_ops = TRUE;
/* space could be freed by other threads as soon
as n_pending_ops reaches 0, thus increment pending
ops here. */
sp->n_pending_ops++;
}
mutex_exit(&fil_system->mutex);
/* Wait for crypt threads to stop accessing space */
if (sp) {
fil_space_crypt_close_tablespace(sp);
/* We have "acquired" this space and must
free it now as below we compare n_pending_ops. */
fil_space_release(sp);
}
/* Check for pending change buffer merges. */
do {
@ -3944,13 +3994,13 @@ fil_create_new_single_table_tablespace(
/* Create crypt data if the tablespace is either encrypted or user has
requested it to remain unencrypted. */
if (mode == FIL_SPACE_ENCRYPTION_ON || mode == FIL_SPACE_ENCRYPTION_OFF ||
if (mode == FIL_ENCRYPTION_ON || mode == FIL_ENCRYPTION_OFF ||
srv_encrypt_tables) {
crypt_data = fil_space_create_crypt_data(mode, key_id);
}
success = fil_space_create(tablename, space_id, flags, FIL_TABLESPACE,
crypt_data, true);
crypt_data, true, mode);
if (!success || !fil_node_create(path, size, space_id, FALSE)) {
err = DB_ERROR;
@ -6633,7 +6683,8 @@ fil_iterate(
page_type == FIL_PAGE_PAGE_COMPRESSED);
/* If tablespace is encrypted, we need to decrypt
the page. */
the page. Note that tablespaces are not in
fil_system during import. */
if (encrypted) {
decrypted = fil_space_decrypt(
iter.crypt_data,
@ -6886,8 +6937,11 @@ fil_tablespace_iterate(
iter.n_io_buffers = n_io_buffers;
iter.page_size = callback.get_page_size();
/* In MariaDB/MySQL 5.6 tablespace does not exist
during import, therefore we can't use space directly
here. */
ulint crypt_data_offset = fsp_header_get_crypt_offset(
callback.get_zip_size(), 0);
callback.get_zip_size());
/* read (optional) crypt data */
iter.crypt_data = fil_space_read_crypt_data(
@ -6929,7 +6983,7 @@ fil_tablespace_iterate(
mem_free(io_buffer);
if (iter.crypt_data != NULL) {
if (crypt_io_buffer != NULL) {
mem_free(crypt_io_buffer);
iter.crypt_io_buffer = NULL;
fil_space_destroy_crypt_data(&iter.crypt_data);
@ -7175,36 +7229,6 @@ fil_get_first_space()
return out_id;
}
/******************************************************************
Get id of first tablespace that has node or ULINT_UNDEFINED if none */
UNIV_INTERN
ulint
fil_get_first_space_safe()
/*======================*/
{
ulint out_id = ULINT_UNDEFINED;
fil_space_t* space;
mutex_enter(&fil_system->mutex);
space = UT_LIST_GET_FIRST(fil_system->space_list);
if (space != NULL) {
do
{
if (!space->stop_new_ops && UT_LIST_GET_LEN(space->chain) > 0) {
out_id = space->id;
break;
}
space = UT_LIST_GET_NEXT(space_list, space);
} while (space != NULL);
}
mutex_exit(&fil_system->mutex);
return out_id;
}
/******************************************************************
Get id of next tablespace or ULINT_UNDEFINED if none */
UNIV_INTERN
@ -7246,165 +7270,205 @@ fil_get_next_space(
return out_id;
}
/******************************************************************
Get id of next tablespace that has node or ULINT_UNDEFINED if none */
UNIV_INTERN
ulint
fil_get_next_space_safe(
/*====================*/
ulint id) /*!< in: previous space id */
/** Acquire a tablespace when it could be dropped concurrently.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@param[in] silent whether to silently ignore missing tablespaces
@return the tablespace, or NULL if missing or being deleted */
inline
fil_space_t*
fil_space_acquire_low(
ulint id,
bool silent)
{
bool found;
fil_space_t* space;
ulint out_id = ULINT_UNDEFINED;
fil_space_t* space;
mutex_enter(&fil_system->mutex);
space = fil_space_get_by_id(id);
if (space == NULL) {
/* we didn't find it...search for space with space->id > id */
found = false;
if (!silent) {
ib_logf(IB_LOG_LEVEL_WARN, "Trying to access missing"
" tablespace " ULINTPF ".", id);
}
} else if (space->stop_new_ops) {
space = NULL;
} else {
space->n_pending_ops++;
}
mutex_exit(&fil_system->mutex);
return(space);
}
/** Acquire a tablespace when it could be dropped concurrently.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@return the tablespace, or NULL if missing or being deleted */
fil_space_t*
fil_space_acquire(
ulint id)
{
return(fil_space_acquire_low(id, false));
}
/** Acquire a tablespace that may not exist.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@return the tablespace, or NULL if missing or being deleted */
fil_space_t*
fil_space_acquire_silent(
ulint id)
{
return(fil_space_acquire_low(id, true));
}
/** Release a tablespace acquired with fil_space_acquire().
@param[in,out] space tablespace to release */
void
fil_space_release(
fil_space_t* space)
{
mutex_enter(&fil_system->mutex);
ut_ad(space->magic_n == FIL_SPACE_MAGIC_N);
ut_ad(space->n_pending_ops > 0);
space->n_pending_ops--;
mutex_exit(&fil_system->mutex);
}
/** Return the next fil_space_t.
Once started, the caller must keep calling this until it returns NULL.
fil_space_acquire() and fil_space_release() are invoked here which
blocks a concurrent operation from dropping the tablespace.
@param[in] prev_space Pointer to the previous fil_space_t.
If NULL, use the first fil_space_t on fil_system->space_list.
@return pointer to the next fil_space_t.
@retval NULL if this was the last*/
fil_space_t*
fil_space_next(
fil_space_t* prev_space)
{
fil_space_t* space=prev_space;
mutex_enter(&fil_system->mutex);
if (prev_space == NULL) {
space = UT_LIST_GET_FIRST(fil_system->space_list);
/* We can trust that space is not NULL because at least the
system tablespace is always present and loaded first. */
space->n_pending_ops++;
} else {
/* we found it, take next available space */
found = true;
}
ut_ad(space->n_pending_ops > 0);
while ((space = UT_LIST_GET_NEXT(space_list, space)) != NULL) {
/* Move on to the next fil_space_t */
space->n_pending_ops--;
space = UT_LIST_GET_NEXT(space_list, space);
if (!found && space->id <= id)
continue;
/* Skip spaces that are being created by
fil_ibd_create(), or dropped, or !tablespace. */
while (space != NULL
&& (UT_LIST_GET_LEN(space->chain) == 0
|| space->stop_new_ops
|| space->purpose != FIL_TABLESPACE)) {
space = UT_LIST_GET_NEXT(space_list, space);
}
if (!space->stop_new_ops) {
/* inc reference to prevent drop */
out_id = space->id;
break;
if (space != NULL) {
space->n_pending_ops++;
}
}
mutex_exit(&fil_system->mutex);
return out_id;
return(space);
}
/******************************************************************
Get crypt data for a tablespace */
UNIV_INTERN
fil_space_crypt_t*
fil_space_get_crypt_data(
/*=====================*/
ulint id) /*!< in: space id */
/**
Remove space from key rotation list if there are no more
pending operations.
@param[in] space Tablespace */
static
void
fil_space_remove_from_keyrotation(
fil_space_t* space)
{
fil_space_t* space;
fil_space_crypt_t* crypt_data = NULL;
ut_ad(mutex_own(&fil_system->mutex));
ut_ad(space);
ut_ad(fil_system);
mutex_enter(&fil_system->mutex);
space = fil_space_get_by_id(id);
mutex_exit(&fil_system->mutex);
if (space != NULL) {
/* If we have not yet read the page0
of this tablespace we will do it now. */
if (!space->crypt_data && !space->page_0_crypt_read) {
ulint space_id = space->id;
fil_node_t* node;
ut_a(space->crypt_data == NULL);
node = UT_LIST_GET_FIRST(space->chain);
byte *buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
byte *page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE));
fil_read(true, space_id, 0, 0, 0, UNIV_PAGE_SIZE, page,
NULL, NULL);
ulint offset = fsp_header_get_crypt_offset(
fsp_header_get_zip_size(page), NULL);
space->crypt_data = fil_space_read_crypt_data(space_id, page, offset);
ut_free(buf);
#ifdef UNIV_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
"Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d.",
space_id,
space->name,
space->crypt_data ? space->crypt_data->key_id : 0,
space->crypt_data ? space->crypt_data->encryption : 0,
node->handle);
#endif
ut_a(space->id == space_id);
space->page_0_crypt_read = true;
}
crypt_data = space->crypt_data;
if (!space->page_0_crypt_read) {
ib_logf(IB_LOG_LEVEL_WARN,
"Space %lu name %s contains encryption %d information for key_id %u but page0 is not read.",
space->id,
space->name,
space->crypt_data ? space->crypt_data->encryption : 0,
space->crypt_data ? space->crypt_data->key_id : 0);
}
if (space->n_pending_ops == 0) {
space->is_in_rotation_list = false;
UT_LIST_REMOVE(rotation_list, fil_system->rotation_list, space);
}
return(crypt_data);
}
/******************************************************************
Get crypt data for a tablespace */
UNIV_INTERN
fil_space_crypt_t*
fil_space_set_crypt_data(
/*=====================*/
ulint id, /*!< in: space id */
fil_space_crypt_t* crypt_data) /*!< in: crypt data */
{
fil_space_t* space;
fil_space_crypt_t* free_crypt_data = NULL;
fil_space_crypt_t* ret_crypt_data = NULL;
ut_ad(fil_system);
/** Return the next fil_space_t from key rotation list.
Once started, the caller must keep calling this until it returns NULL.
fil_space_acquire() and fil_space_release() are invoked here which
blocks a concurrent operation from dropping the tablespace.
@param[in] prev_space Pointer to the previous fil_space_t.
If NULL, use the first fil_space_t on fil_system->space_list.
@return pointer to the next fil_space_t.
@retval NULL if this was the last*/
fil_space_t*
fil_space_keyrotate_next(
fil_space_t* prev_space)
{
fil_space_t* space = prev_space;
fil_space_t* old = NULL;
mutex_enter(&fil_system->mutex);
space = fil_space_get_by_id(id);
if (space != NULL) {
if (space->crypt_data != NULL) {
/* Here we need to release fil_system mutex to
avoid mutex deadlock assertion. Here we would
taje mutexes in order fil_system, crypt_data and
in fil_crypt_start_encrypting_space we would
take them in order crypt_data, fil_system
at fil_space_get_flags -> fil_space_get_space */
mutex_exit(&fil_system->mutex);
fil_space_merge_crypt_data(space->crypt_data,
crypt_data);
ret_crypt_data = space->crypt_data;
free_crypt_data = crypt_data;
} else {
space->crypt_data = crypt_data;
ret_crypt_data = space->crypt_data;
mutex_exit(&fil_system->mutex);
if (UT_LIST_GET_LEN(fil_system->rotation_list) == 0) {
if (space) {
ut_ad(space->n_pending_ops > 0);
space->n_pending_ops--;
fil_space_remove_from_keyrotation(space);
}
} else {
/* there is a small risk that tablespace has been deleted */
free_crypt_data = crypt_data;
mutex_exit(&fil_system->mutex);
return(NULL);
}
if (free_crypt_data != NULL) {
/* there was already crypt data present and the new crypt
* data provided as argument to this function has been merged
* into that => free new crypt data
*/
fil_space_destroy_crypt_data(&free_crypt_data);
if (prev_space == NULL) {
space = UT_LIST_GET_FIRST(fil_system->rotation_list);
/* We can trust that space is not NULL because we
checked list length above */
} else {
ut_ad(space->n_pending_ops > 0);
/* Move on to the next fil_space_t */
space->n_pending_ops--;
old = space;
space = UT_LIST_GET_NEXT(rotation_list, space);
fil_space_remove_from_keyrotation(old);
}
return ret_crypt_data;
/* Skip spaces that are being created by fil_ibd_create(),
or dropped. Note that rotation_list contains only
space->purpose == FIL_TABLESPACE. */
while (space != NULL
&& (UT_LIST_GET_LEN(space->chain) == 0
|| space->stop_new_ops)) {
old = space;
space = UT_LIST_GET_NEXT(rotation_list, space);
fil_space_remove_from_keyrotation(old);
}
if (space != NULL) {
space->n_pending_ops++;
}
mutex_exit(&fil_system->mutex);
return(space);
}

40
storage/innobase/fsp/fsp0fsp.cc
View file

@ -677,7 +677,7 @@ UNIV_INTERN
void
fsp_header_init(
/*============*/
ulint space, /*!< in: space id */
ulint space_id, /*!< in: space id */
ulint size, /*!< in: current size in blocks */
mtr_t* mtr) /*!< in/out: mini-transaction */
{
@ -689,11 +689,11 @@ fsp_header_init(
ut_ad(mtr);
mtr_x_lock(fil_space_get_latch(space, &flags), mtr);
mtr_x_lock(fil_space_get_latch(space_id, &flags), mtr);
zip_size = fsp_flags_get_zip_size(flags);
block = buf_page_create(space, 0, zip_size, mtr);
buf_page_get(space, zip_size, 0, RW_X_LATCH, mtr);
block = buf_page_create(space_id, 0, zip_size, mtr);
buf_page_get(space_id, zip_size, 0, RW_X_LATCH, mtr);
buf_block_dbg_add_level(block, SYNC_FSP_PAGE);
/* The prior contents of the file page should be ignored */
@ -706,7 +706,7 @@ fsp_header_init(
header = FSP_HEADER_OFFSET + page;
mlog_write_ulint(header + FSP_SPACE_ID, space, MLOG_4BYTES, mtr);
mlog_write_ulint(header + FSP_SPACE_ID, space_id, MLOG_4BYTES, mtr);
mlog_write_ulint(header + FSP_NOT_USED, 0, MLOG_4BYTES, mtr);
mlog_write_ulint(header + FSP_SIZE, size, MLOG_4BYTES, mtr);
@ -722,18 +722,23 @@ fsp_header_init(
flst_init(header + FSP_SEG_INODES_FREE, mtr);
mlog_write_ull(header + FSP_SEG_ID, 1, mtr);
if (space == 0) {
fsp_fill_free_list(FALSE, space, header, mtr);
if (space_id == 0) {
fsp_fill_free_list(FALSE, space_id, header, mtr);
btr_create(DICT_CLUSTERED | DICT_UNIVERSAL | DICT_IBUF,
0, 0, DICT_IBUF_ID_MIN + space,
0, 0, DICT_IBUF_ID_MIN + space_id,
dict_ind_redundant, mtr);
} else {
fsp_fill_free_list(TRUE, space, header, mtr);
fsp_fill_free_list(TRUE, space_id, header, mtr);
}
ulint maxsize = 0;
ulint offset = fsp_header_get_crypt_offset(zip_size, &maxsize);
fil_space_write_crypt_data(space, page, offset, maxsize, mtr);
fil_space_t* space = fil_space_acquire(space_id);
ut_ad(space);
if (space->crypt_data) {
space->crypt_data->write_page0(page, mtr);
}
fil_space_release(space);
}
#endif /* !UNIV_HOTBACKUP */
@ -4123,12 +4128,11 @@ fsp_print(
/**********************************************************************//**
Compute offset after xdes where crypt data can be stored
@param[in] zip_size Compressed size or 0
@return offset */
ulint
fsp_header_get_crypt_offset(
/*========================*/
ulint zip_size, /*!< in: zip_size */
ulint* max_size) /*!< out: free space available for crypt data */
const ulint zip_size)
{
ulint pageno = 0;
/* compute first page_no that will have xdes stored on page != 0*/
@ -4143,12 +4147,6 @@ fsp_header_get_crypt_offset(
ulint iv_offset = XDES_ARR_OFFSET +
XDES_SIZE * (1 + xdes_calc_descriptor_index(zip_size, pageno));
if (max_size != NULL) {
/* return how much free space there is available on page */
*max_size = (zip_size ? zip_size : UNIV_PAGE_SIZE) -
(FSP_HEADER_OFFSET + iv_offset + FIL_PAGE_DATA_END);
}
return FSP_HEADER_OFFSET + iv_offset;
}

3
storage/innobase/fts/fts0fts.cc
View file

@ -1989,7 +1989,8 @@ fts_create_one_index_table(
dict_mem_table_add_col(new_table, heap, "ilist", DATA_BLOB,
4130048, 0);
error = row_create_table_for_mysql(new_table, trx, false, FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
error = row_create_table_for_mysql(new_table, trx, false,
FIL_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
if (error != DB_SUCCESS) {
trx->error_state = error;

34
storage/innobase/handler/ha_innodb.cc
View file

@ -1084,6 +1084,9 @@ static SHOW_VAR innodb_status_variables[]= {
{"encryption_rotation_estimated_iops",
(char*) &export_vars.innodb_encryption_rotation_estimated_iops,
SHOW_LONG},
{"encryption_key_rotation_list_length",
(char*)&export_vars.innodb_key_rotation_list_length,
SHOW_LONGLONG},
/* scrubing */
{"scrub_background_page_reorganizations",
@ -11752,7 +11755,7 @@ ha_innobase::check_table_options(
atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes;
fil_encryption_t encrypt = (fil_encryption_t)options->encryption;
if (encrypt != FIL_SPACE_ENCRYPTION_DEFAULT && !use_tablespace) {
if (encrypt != FIL_ENCRYPTION_DEFAULT && !use_tablespace) {
push_warning(
thd, Sql_condition::WARN_LEVEL_WARN,
HA_WRONG_CREATE_OPTION,
@ -11760,7 +11763,7 @@ ha_innobase::check_table_options(
return "ENCRYPTED";
}
if (encrypt == FIL_SPACE_ENCRYPTION_OFF && srv_encrypt_tables == 2) {
if (encrypt == FIL_ENCRYPTION_OFF && srv_encrypt_tables == 2) {
push_warning(
thd, Sql_condition::WARN_LEVEL_WARN,
HA_WRONG_CREATE_OPTION,
@ -11841,8 +11844,8 @@ ha_innobase::check_table_options(
}
/* If encryption is set up make sure that used key_id is found */
if (encrypt == FIL_SPACE_ENCRYPTION_ON ||
(encrypt == FIL_SPACE_ENCRYPTION_DEFAULT && srv_encrypt_tables)) {
if (encrypt == FIL_ENCRYPTION_ON ||
(encrypt == FIL_ENCRYPTION_DEFAULT && srv_encrypt_tables)) {
if (!encryption_key_id_exists((unsigned int)options->encryption_key_id)) {
push_warning_printf(
thd, Sql_condition::WARN_LEVEL_WARN,
@ -11855,7 +11858,7 @@ ha_innobase::check_table_options(
}
/* Ignore nondefault key_id if encryption is set off */
if (encrypt == FIL_SPACE_ENCRYPTION_OFF &&
if (encrypt == FIL_ENCRYPTION_OFF &&
options->encryption_key_id != THDVAR(thd, default_encryption_key_id)) {
push_warning_printf(
thd, Sql_condition::WARN_LEVEL_WARN,
@ -11868,7 +11871,7 @@ ha_innobase::check_table_options(
/* If default encryption is used make sure that used kay is found
from key file. */
if (encrypt == FIL_SPACE_ENCRYPTION_DEFAULT &&
if (encrypt == FIL_ENCRYPTION_DEFAULT &&
!srv_encrypt_tables &&
options->encryption_key_id != FIL_DEFAULT_ENCRYPTION_KEY) {
if (!encryption_key_id_exists((unsigned int)options->encryption_key_id)) {
@ -19820,10 +19823,11 @@ static MYSQL_SYSVAR_UINT(encryption_rotate_key_age,
PLUGIN_VAR_RQCMDARG,
"Key rotation - re-encrypt in background "
"all pages that were encrypted with a key that "
"many (or more) versions behind",
"many (or more) versions behind. Value 0 indicates "
"that key rotation is disabled.",
NULL,
innodb_encryption_rotate_key_age_update,
srv_fil_crypt_rotate_key_age, 0, UINT_MAX32, 0);
1, 0, UINT_MAX32, 0);
static MYSQL_SYSVAR_UINT(encryption_rotation_iops, srv_n_fil_crypt_iops,
PLUGIN_VAR_RQCMDARG,
@ -20737,8 +20741,9 @@ innodb_encrypt_tables_validate(
for update function */
struct st_mysql_value* value) /*!< in: incoming string */
{
if (check_sysvar_enum(thd, var, save, value))
if (check_sysvar_enum(thd, var, save, value)) {
return 1;
}
ulong encrypt_tables = *(ulong*)save;
@ -20750,6 +20755,17 @@ innodb_encrypt_tables_validate(
"encryption plugin is not available");
return 1;
}
if (!srv_fil_crypt_rotate_key_age) {
const char *msg = (encrypt_tables ? "enable" : "disable");
push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
HA_ERR_UNSUPPORTED,
"InnoDB: cannot %s encryption, "
"innodb_encryption_rotate_key_age=0"
" i.e. key rotation disabled", msg);
return 1;
}
return 0;
}

6
storage/innobase/handler/handler0alter.cc
View file

@ -2883,9 +2883,11 @@ prepare_inplace_alter_table_dict(
ulint n_cols;
dtuple_t* add_cols;
ulint key_id = FIL_DEFAULT_ENCRYPTION_KEY;
fil_encryption_t mode = FIL_SPACE_ENCRYPTION_DEFAULT;
fil_encryption_t mode = FIL_ENCRYPTION_DEFAULT;
crypt_data = fil_space_get_crypt_data(ctx->prebuilt->table->space);
fil_space_t* space = fil_space_acquire(ctx->prebuilt->table->space);
crypt_data = space->crypt_data;
fil_space_release(space);
if (crypt_data) {
key_id = crypt_data->key_id;

114
storage/innobase/handler/i_s.cc
View file

@ -1,7 +1,7 @@
/*****************************************************************************
Copyright (c) 2007, 2016, Oracle and/or its affiliates.
Copyrigth (c) 2014, 2016, MariaDB Corporation
Copyrigth (c) 2014, 2017, MariaDB Corporation
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -8180,22 +8180,31 @@ static ST_FIELD_INFO innodb_tablespaces_encryption_fields_info[] =
STRUCT_FLD(old_name, ""),
STRUCT_FLD(open_method, SKIP_OPEN_TABLE)},
#define TABLESPACES_ENCRYPTION_ROTATING_OR_FLUSHING 9
{STRUCT_FLD(field_name, "ROTATING_OR_FLUSHING"),
STRUCT_FLD(field_length, 1),
STRUCT_FLD(field_type, MYSQL_TYPE_LONG),
STRUCT_FLD(value, 0),
STRUCT_FLD(field_flags, MY_I_S_UNSIGNED),
STRUCT_FLD(old_name, ""),
STRUCT_FLD(open_method, SKIP_OPEN_TABLE)},
END_OF_ST_FIELD_INFO
};
/**********************************************************************//**
Function to fill INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION
with information collected by scanning SYS_TABLESPACES table and then use
fil_space()
with information collected by scanning SYS_TABLESPACES table.
@param[in] thd thread handle
@param[in] space Tablespace
@param[in] table_to_fill I_S table to fill
@return 0 on success */
static
int
i_s_dict_fill_tablespaces_encryption(
/*==========================*/
THD* thd, /*!< in: thread */
ulint space, /*!< in: space ID */
const char* name, /*!< in: tablespace name */
TABLE* table_to_fill) /*!< in/out: fill this table */
THD* thd,
fil_space_t* space,
TABLE* table_to_fill)
{
Field** fields;
struct fil_space_crypt_status_t status;
@ -8205,10 +8214,11 @@ i_s_dict_fill_tablespaces_encryption(
fields = table_to_fill->field;
fil_space_crypt_get_status(space, &status);
OK(fields[TABLESPACES_ENCRYPTION_SPACE]->store(space));
OK(fields[TABLESPACES_ENCRYPTION_SPACE]->store(space->id));
OK(field_store_string(fields[TABLESPACES_ENCRYPTION_NAME],
name));
space->name));
OK(fields[TABLESPACES_ENCRYPTION_ENCRYPTION_SCHEME]->store(
status.scheme));
@ -8220,6 +8230,9 @@ i_s_dict_fill_tablespaces_encryption(
status.current_key_version));
OK(fields[TABLESPACES_ENCRYPTION_CURRENT_KEY_ID]->store(
status.key_id));
OK(fields[TABLESPACES_ENCRYPTION_ROTATING_OR_FLUSHING]->store(
(status.rotating || status.flushing) ? 1 : 0));
if (status.rotating) {
fields[TABLESPACES_ENCRYPTION_KEY_ROTATION_PAGE_NUMBER]->set_notnull();
OK(fields[TABLESPACES_ENCRYPTION_KEY_ROTATION_PAGE_NUMBER]->store(
@ -8233,6 +8246,7 @@ i_s_dict_fill_tablespaces_encryption(
fields[TABLESPACES_ENCRYPTION_KEY_ROTATION_MAX_PAGE_NUMBER]
->set_null();
}
OK(schema_table_store_record(thd, table_to_fill));
DBUG_RETURN(0);
@ -8272,30 +8286,36 @@ i_s_tablespaces_encryption_fill_table(
while (rec) {
const char* err_msg;
ulint space;
ulint space_id;
const char* name;
ulint flags;
/* Extract necessary information from a SYS_TABLESPACES row */
err_msg = dict_process_sys_tablespaces(
heap, rec, &space, &name, &flags);
heap, rec, &space_id, &name, &flags);
mtr_commit(&mtr);
mutex_exit(&dict_sys->mutex);
if (space == 0) {
if (space_id == 0) {
found_space_0 = true;
}
if (!err_msg) {
fil_space_t* space = fil_space_acquire_silent(space_id);
if (!err_msg && space) {
i_s_dict_fill_tablespaces_encryption(
thd, space, name, tables->table);
thd, space, tables->table);
} else {
push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
ER_CANT_FIND_SYSTEM_REC, "%s",
err_msg);
}
if (space) {
fil_space_release(space);
}
mem_heap_empty(heap);
/* Get the next record */
@ -8311,10 +8331,13 @@ i_s_tablespaces_encryption_fill_table(
if (found_space_0 == false) {
/* space 0 does for what ever unknown reason not show up
* in iteration above, add it manually */
ulint space = 0;
const char* name = NULL;
fil_space_t* space = fil_space_acquire_silent(0);
i_s_dict_fill_tablespaces_encryption(
thd, space, name, tables->table);
thd, space, tables->table);
fil_space_release(space);
}
DBUG_RETURN(0);
@ -8465,22 +8488,32 @@ static ST_FIELD_INFO innodb_tablespaces_scrubbing_fields_info[] =
STRUCT_FLD(old_name, ""),
STRUCT_FLD(open_method, SKIP_OPEN_TABLE)},
#define TABLESPACES_ENCRYPTION_ROTATING_OR_FLUSHING 9
{STRUCT_FLD(field_name, "ROTATING_OR_FLUSHING"),
STRUCT_FLD(field_length, MY_INT32_NUM_DECIMAL_DIGITS),
STRUCT_FLD(field_type, MYSQL_TYPE_LONG),
STRUCT_FLD(value, 0),
STRUCT_FLD(field_flags, MY_I_S_UNSIGNED),
STRUCT_FLD(old_name, ""),
STRUCT_FLD(open_method, SKIP_OPEN_TABLE)},
END_OF_ST_FIELD_INFO
};
/**********************************************************************//**
Function to fill INFORMATION_SCHEMA.INNODB_TABLESPACES_SCRUBBING
with information collected by scanning SYS_TABLESPACES table and then use
fil_space()
with information collected by scanning SYS_TABLESPACES table and
fil_space.
@param[in] thd Thread handle
@param[in] space Tablespace
@param[in] table_to_fill I_S table
@return 0 on success */
static
int
i_s_dict_fill_tablespaces_scrubbing(
/*==========================*/
THD* thd, /*!< in: thread */
ulint space, /*!< in: space ID */
const char* name, /*!< in: tablespace name */
TABLE* table_to_fill) /*!< in/out: fill this table */
THD* thd,
fil_space_t* space,
TABLE* table_to_fill)
{
Field** fields;
struct fil_space_scrub_status_t status;
@ -8490,10 +8523,11 @@ i_s_dict_fill_tablespaces_scrubbing(
fields = table_to_fill->field;
fil_space_get_scrub_status(space, &status);
OK(fields[TABLESPACES_SCRUBBING_SPACE]->store(space));
OK(fields[TABLESPACES_SCRUBBING_SPACE]->store(space->id));
OK(field_store_string(fields[TABLESPACES_SCRUBBING_NAME],
name));
space->name));
OK(fields[TABLESPACES_SCRUBBING_COMPRESSED]->store(
status.compressed ? 1 : 0));
@ -8513,6 +8547,7 @@ i_s_dict_fill_tablespaces_scrubbing(
TABLESPACES_SCRUBBING_CURRENT_SCRUB_ACTIVE_THREADS,
TABLESPACES_SCRUBBING_CURRENT_SCRUB_PAGE_NUMBER,
TABLESPACES_SCRUBBING_CURRENT_SCRUB_MAX_PAGE_NUMBER };
if (status.scrubbing) {
for (uint i = 0; i < array_elements(field_numbers); i++) {
fields[field_numbers[i]]->set_notnull();
@ -8532,6 +8567,7 @@ i_s_dict_fill_tablespaces_scrubbing(
fields[field_numbers[i]]->set_null();
}
}
OK(schema_table_store_record(thd, table_to_fill));
DBUG_RETURN(0);
@ -8571,30 +8607,36 @@ i_s_tablespaces_scrubbing_fill_table(
while (rec) {
const char* err_msg;
ulint space;
ulint space_id;
const char* name;
ulint flags;
/* Extract necessary information from a SYS_TABLESPACES row */
err_msg = dict_process_sys_tablespaces(
heap, rec, &space, &name, &flags);
heap, rec, &space_id, &name, &flags);
mtr_commit(&mtr);
mutex_exit(&dict_sys->mutex);
if (space == 0) {
if (space_id == 0) {
found_space_0 = true;
}
if (!err_msg) {
fil_space_t* space = fil_space_acquire_silent(space_id);
if (!err_msg && space) {
i_s_dict_fill_tablespaces_scrubbing(
thd, space, name, tables->table);
thd, space, tables->table);
} else {
push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
ER_CANT_FIND_SYSTEM_REC, "%s",
err_msg);
}
if (space) {
fil_space_release(space);
}
mem_heap_empty(heap);
/* Get the next record */
@ -8610,10 +8652,12 @@ i_s_tablespaces_scrubbing_fill_table(
if (found_space_0 == false) {
/* space 0 does for what ever unknown reason not show up
* in iteration above, add it manually */
ulint space = 0;
const char* name = NULL;
fil_space_t* space = fil_space_acquire_silent(0);
i_s_dict_fill_tablespaces_scrubbing(
thd, space, name, tables->table);
thd, space, tables->table);
fil_space_release(space);
}
DBUG_RETURN(0);

47
storage/innobase/ibuf/ibuf0ibuf.cc
View file

@ -1,7 +1,7 @@
/*****************************************************************************
Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2016, MariaDB Corporation.
Copyright (c) 2016, 2017, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -4554,7 +4554,7 @@ ibuf_merge_or_delete_for_page(
buf_block_t* block, /*!< in: if page has been read from
disk, pointer to the page x-latched,
else NULL */
ulint space, /*!< in: space id of the index page */
ulint space_id,/*!< in: space id of the index page */
ulint page_no,/*!< in: page number of the index page */
ulint zip_size,/*!< in: compressed page size in bytes,
or 0 */
@ -4571,21 +4571,21 @@ ibuf_merge_or_delete_for_page(
ulint volume = 0;
#endif
page_zip_des_t* page_zip = NULL;
ibool tablespace_being_deleted = FALSE;
ibool corruption_noticed = FALSE;
mtr_t mtr;
fil_space_t* space = NULL;
/* Counts for merged & discarded operations. */
ulint mops[IBUF_OP_COUNT];
ulint dops[IBUF_OP_COUNT];
ut_ad(!block || buf_block_get_space(block) == space);
ut_ad(!block || buf_block_get_space(block) == space_id);
ut_ad(!block || buf_block_get_page_no(block) == page_no);
ut_ad(!block || buf_block_get_zip_size(block) == zip_size);
ut_ad(!block || buf_block_get_io_fix(block) == BUF_IO_READ);
if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE
|| trx_sys_hdr_page(space, page_no)) {
|| trx_sys_hdr_page(space_id, page_no)) {
return;
}
@ -4599,7 +4599,7 @@ ibuf_merge_or_delete_for_page(
uncompressed page size always is a power-of-2 multiple of the
compressed page size. */
if (ibuf_fixed_addr_page(space, 0, page_no)
if (ibuf_fixed_addr_page(space_id, 0, page_no)
|| fsp_descr_page(0, page_no)) {
return;
}
@ -4607,19 +4607,19 @@ ibuf_merge_or_delete_for_page(
if (UNIV_LIKELY(update_ibuf_bitmap)) {
ut_a(ut_is_2pow(zip_size));
if (ibuf_fixed_addr_page(space, zip_size, page_no)
if (ibuf_fixed_addr_page(space_id, zip_size, page_no)
|| fsp_descr_page(zip_size, page_no)) {
return;
}
/* If the following returns FALSE, we get the counter
/* If the following returns space, we get the counter
incremented, and must decrement it when we leave this
function. When the counter is > 0, that prevents tablespace
from being dropped. */
tablespace_being_deleted = fil_inc_pending_ops(space, true);
space = fil_space_acquire(space_id);
if (UNIV_UNLIKELY(tablespace_being_deleted)) {
if (UNIV_UNLIKELY(!space)) {
/* Do not try to read the bitmap page from space;
just delete the ibuf records for the page */
@ -4632,7 +4632,7 @@ ibuf_merge_or_delete_for_page(
ibuf_mtr_start(&mtr);
bitmap_page = ibuf_bitmap_get_map_page(
space, page_no, zip_size, &mtr);
space_id, page_no, zip_size, &mtr);
if (bitmap_page &&
fil_page_get_type(bitmap_page) != FIL_PAGE_TYPE_ALLOCATED) {
@ -4646,15 +4646,15 @@ ibuf_merge_or_delete_for_page(
if (!bitmap_bits) {
/* No inserts buffered for this page */
if (!tablespace_being_deleted) {
fil_decr_pending_ops(space);
if (space) {
fil_space_release(space);
}
return;
}
}
} else if (block
&& (ibuf_fixed_addr_page(space, zip_size, page_no)
&& (ibuf_fixed_addr_page(space_id, zip_size, page_no)
|| fsp_descr_page(zip_size, page_no))) {
return;
@ -4662,7 +4662,7 @@ ibuf_merge_or_delete_for_page(
heap = mem_heap_create(512);
search_tuple = ibuf_search_tuple_build(space, page_no, heap);
search_tuple = ibuf_search_tuple_build(space_id, page_no, heap);
if (block) {
/* Move the ownership of the x-latch on the page to this OS
@ -4688,7 +4688,7 @@ ibuf_merge_or_delete_for_page(
fputs(" InnoDB: Dump of the ibuf bitmap page:\n",
stderr);
bitmap_page = ibuf_bitmap_get_map_page(space, page_no,
bitmap_page = ibuf_bitmap_get_map_page(space_id, page_no,
zip_size, &mtr);
if (bitmap_page == NULL)
{
@ -4772,7 +4772,7 @@ loop:
/* Check if the entry is for this index page */
if (ibuf_rec_get_page_no(&mtr, rec) != page_no
|| ibuf_rec_get_space(&mtr, rec) != space) {
|| ibuf_rec_get_space(&mtr, rec) != space_id) {
if (block) {
page_header_reset_last_insert(
@ -4839,7 +4839,7 @@ loop:
ut_ad(page_rec_is_user_rec(rec));
ut_ad(ibuf_rec_get_page_no(&mtr, rec)
== page_no);
ut_ad(ibuf_rec_get_space(&mtr, rec) == space);
ut_ad(ibuf_rec_get_space(&mtr, rec) == space_id);
/* Mark the change buffer record processed,
so that it will not be merged again in case
@ -4869,7 +4869,7 @@ loop:
buf_block_dbg_add_level(
block, SYNC_IBUF_TREE_NODE);
if (!ibuf_restore_pos(space, page_no,
if (!ibuf_restore_pos(space_id, page_no,
search_tuple,
BTR_MODIFY_LEAF,
&pcur, &mtr)) {
@ -4893,7 +4893,7 @@ loop:
}
/* Delete the record from ibuf */
if (ibuf_delete_rec(space, page_no, &pcur, search_tuple,
if (ibuf_delete_rec(space_id, page_no, &pcur, search_tuple,
&mtr)) {
/* Deletion was pessimistic and mtr was committed:
we start from the beginning again */
@ -4913,7 +4913,7 @@ reset_bit:
page_t* bitmap_page;
bitmap_page = ibuf_bitmap_get_map_page(
space, page_no, zip_size, &mtr);
space_id, page_no, zip_size, &mtr);
ibuf_bitmap_page_set_bits(
bitmap_page, page_no, zip_size,
@ -4954,9 +4954,8 @@ reset_bit:
mutex_exit(&ibuf_mutex);
#endif /* HAVE_ATOMIC_BUILTINS */
if (update_ibuf_bitmap && !tablespace_being_deleted) {
fil_decr_pending_ops(space);
if (space) {
fil_space_release(space);
}
#ifdef UNIV_IBUF_COUNT_DEBUG

582
storage/innobase/include/fil0crypt.h
View file

@ -39,14 +39,6 @@ static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = {
/* This key will be used if nothing else is given */
#define FIL_DEFAULT_ENCRYPTION_KEY ENCRYPTION_KEY_SYSTEM_DATA
/** Enum values for encryption table option */
typedef enum {
FIL_SPACE_ENCRYPTION_DEFAULT = 0, /* Tablespace encrypted if
srv_encrypt_tables = ON */
FIL_SPACE_ENCRYPTION_ON = 1, /* Tablespace is encrypted always */
FIL_SPACE_ENCRYPTION_OFF = 2 /* Tablespace is not encrypted */
} fil_encryption_t;
extern os_event_t fil_crypt_threads_event;
/**
@ -110,23 +102,21 @@ struct fil_space_rotate_state_t
} scrubbing;
};
struct fil_space_crypt_struct : st_encryption_scheme
struct fil_space_crypt_t : st_encryption_scheme
{
public:
/** Constructor. Does not initialize the members!
The object is expected to be placed in a buffer that
has been zero-initialized. */
fil_space_crypt_struct(
fil_space_crypt_t(
ulint new_type,
uint new_min_key_version,
uint new_key_id,
ulint offset,
fil_encryption_t new_encryption)
: st_encryption_scheme(),
min_key_version(new_min_key_version),
page0_offset(offset),
page0_offset(0),
encryption(new_encryption),
closing(false),
key_found(),
rotate_state()
{
@ -138,9 +128,9 @@ struct fil_space_crypt_struct : st_encryption_scheme
locker = crypt_data_scheme_locker;
type = new_type;
if (new_encryption == FIL_SPACE_ENCRYPTION_OFF ||
if (new_encryption == FIL_ENCRYPTION_OFF ||
(!srv_encrypt_tables &&
new_encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
new_encryption == FIL_ENCRYPTION_DEFAULT)) {
type = CRYPT_SCHEME_UNENCRYPTED;
} else {
type = CRYPT_SCHEME_1;
@ -149,9 +139,8 @@ struct fil_space_crypt_struct : st_encryption_scheme
}
/** Destructor */
~fil_space_crypt_struct()
~fil_space_crypt_t()
{
closing = true;
mutex_free(&mutex);
}
@ -169,45 +158,36 @@ struct fil_space_crypt_struct : st_encryption_scheme
/** Returns true if tablespace should be encrypted */
bool should_encrypt() const {
return ((encryption == FIL_SPACE_ENCRYPTION_ON) ||
return ((encryption == FIL_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
encryption == FIL_SPACE_ENCRYPTION_DEFAULT));
encryption == FIL_ENCRYPTION_DEFAULT));
}
/** Return true if tablespace is encrypted. */
bool is_encrypted() const {
return (encryption != FIL_SPACE_ENCRYPTION_OFF);
return (encryption != FIL_ENCRYPTION_OFF);
}
/** Return true if default tablespace encryption is used, */
bool is_default_encryption() const {
return (encryption == FIL_SPACE_ENCRYPTION_DEFAULT);
return (encryption == FIL_ENCRYPTION_DEFAULT);
}
/** Return true if tablespace is not encrypted. */
bool not_encrypted() const {
return (encryption == FIL_SPACE_ENCRYPTION_OFF);
return (encryption == FIL_ENCRYPTION_OFF);
}
/** Is this tablespace closing. */
bool is_closing(bool is_fixed) {
bool closed;
if (!is_fixed) {
mutex_enter(&mutex);
}
closed = closing;
if (!is_fixed) {
mutex_exit(&mutex);
}
return closed;
}
/** Write crypt data to a page (0)
@param[in,out] page0 Page 0 where to write
@param[in,out] mtr Minitransaction */
void write_page0(byte* page0, mtr_t* mtr);
uint min_key_version; // min key version for this space
ulint page0_offset; // byte offset on page 0 for crypt data
fil_encryption_t encryption; // Encryption setup
ib_mutex_t mutex; // mutex protecting following variables
bool closing; // is tablespace being closed
/** Return code from encryption_key_get_latest_version.
If ENCRYPTION_KEY_VERSION_INVALID encryption plugin
@ -219,238 +199,7 @@ struct fil_space_crypt_struct : st_encryption_scheme
fil_space_rotate_state_t rotate_state;
};
/* structure containing encryption specification */
typedef struct fil_space_crypt_struct fil_space_crypt_t;
/*********************************************************************
Init global resources needed for tablespace encryption/decryption */
UNIV_INTERN
void
fil_space_crypt_init();
/*********************************************************************
Cleanup global resources needed for tablespace encryption/decryption */
UNIV_INTERN
void
fil_space_crypt_cleanup();
/*********************************************************************
Create crypt data, i.e data that is used for a single tablespace */
UNIV_INTERN
fil_space_crypt_t *
fil_space_create_crypt_data(
/*========================*/
fil_encryption_t encrypt_mode, /*!< in: encryption mode */
uint key_id); /*!< in: encryption key id */
/*********************************************************************
Destroy crypt data */
UNIV_INTERN
void
fil_space_destroy_crypt_data(
/*=========================*/
fil_space_crypt_t **crypt_data); /*!< in/out: crypt data */
/*********************************************************************
Get crypt data for a space*/
UNIV_INTERN
fil_space_crypt_t *
fil_space_get_crypt_data(
/*=====================*/
ulint space); /*!< in: tablespace id */
/*********************************************************************
Set crypt data for a space*/
UNIV_INTERN
fil_space_crypt_t*
fil_space_set_crypt_data(
/*=====================*/
ulint space, /*!< in: tablespace id */
fil_space_crypt_t* crypt_data); /*!< in: crypt data to set */
/*********************************************************************
Merge crypt data */
UNIV_INTERN
void
fil_space_merge_crypt_data(
/*=======================*/
fil_space_crypt_t* dst_crypt_data, /*!< in: crypt_data */
const fil_space_crypt_t* src_crypt_data); /*!< in: crypt data */
/*********************************************************************
Read crypt data from buffer page */
UNIV_INTERN
fil_space_crypt_t *
fil_space_read_crypt_data(
/*======================*/
ulint space, /*!< in: tablespace id */
const byte* page, /*!< in: buffer page */
ulint offset); /*!< in: offset where crypt data is stored */
/*********************************************************************
Write crypt data to buffer page */
UNIV_INTERN
void
fil_space_write_crypt_data(
/*=======================*/
ulint space, /*!< in: tablespace id */
byte* page, /*!< in: buffer page */
ulint offset, /*!< in: offset where to store data */
ulint maxsize, /*!< in: max space available to store crypt data in */
mtr_t * mtr); /*!< in: mini-transaction */
/*********************************************************************
Clear crypt data from page 0 (used for import tablespace) */
UNIV_INTERN
void
fil_space_clear_crypt_data(
/*=======================*/
byte* page, /*!< in: buffer page */
ulint offset); /*!< in: offset where crypt data is stored */
/*********************************************************************
Parse crypt data log record */
UNIV_INTERN
byte*
fil_parse_write_crypt_data(
/*=======================*/
byte* ptr, /*!< in: start of log record */
byte* end_ptr, /*!< in: end of log record */
buf_block_t*); /*!< in: buffer page to apply record to */
/*********************************************************************
Check if extra buffer shall be allocated for decrypting after read */
UNIV_INTERN
bool
fil_space_check_encryption_read(
/*============================*/
ulint space); /*!< in: tablespace id */
/******************************************************************
Decrypt a page
@return true if page decrypted, false if not.*/
UNIV_INTERN
bool
fil_space_decrypt(
/*==============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
byte* tmp_frame, /*!< in: temporary buffer */
ulint page_size, /*!< in: page size */
byte* src_frame, /*!< in: out: page buffer */
dberr_t* err) /*!< in: out: DB_SUCCESS or
error code */
MY_ATTRIBUTE((warn_unused_result));
/*********************************************************************
Encrypt buffer page
@return encrypted page, or original not encrypted page if encrypt
is not needed. */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
ulint space, /*!< in: tablespace id */
ulint offset, /*!< in: page no */
lsn_t lsn, /*!< in: page lsn */
byte* src_frame, /*!< in: page frame */
ulint size, /*!< in: size of data to encrypt */
byte* dst_frame); /*!< in: where to encrypt to */
/******************************************************************
Decrypt a page
@param[in] space Tablespace id
@param[in] tmp_frame Temporary buffer used for decrypting
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] decrypted true if page was decrypted
@return decrypted page, or original not encrypted page if decryption is
not needed.*/
UNIV_INTERN
byte*
fil_space_decrypt(
/*==============*/
ulint space,
byte* src_frame,
ulint page_size,
byte* dst_frame,
bool* decrypted)
__attribute__((warn_unused_result));
/*********************************************************************
Verify that post encryption checksum match calculated checksum.
This function should be called only if tablespace contains crypt_data
metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
const fil_space_t* space,
ulint pageno)
__attribute__((warn_unused_result));
/*********************************************************************
Init threads for key rotation */
UNIV_INTERN
void
fil_crypt_threads_init();
/*********************************************************************
Set thread count (e.g start or stops threads) used for key rotation */
UNIV_INTERN
void
fil_crypt_set_thread_cnt(
/*=====================*/
uint new_cnt); /*!< in: requested #threads */
/*********************************************************************
Cleanup resources for threads for key rotation */
UNIV_INTERN
void
fil_crypt_threads_cleanup();
/*********************************************************************
Set rotate key age */
UNIV_INTERN
void
fil_crypt_set_rotate_key_age(
/*=========================*/
uint rotate_age); /*!< in: requested rotate age */
/*********************************************************************
Set rotation threads iops */
UNIV_INTERN
void
fil_crypt_set_rotation_iops(
/*========================*/
uint iops); /*!< in: requested iops */
/*********************************************************************
Mark a space as closing */
UNIV_INTERN
void
fil_space_crypt_mark_space_closing(
/*===============================*/
ulint space, /*!< in: tablespace id */
fil_space_crypt_t* crypt_data); /*!< in: crypt_data or NULL */
/*********************************************************************
Wait for crypt threads to stop accessing space */
UNIV_INTERN
void
fil_space_crypt_close_tablespace(
/*=============================*/
ulint space); /*!< in: tablespace id */
/** Struct for retreiving info about encryption */
/** Status info about encryption */
struct fil_space_crypt_status_t {
ulint space; /*!< tablespace id */
ulint scheme; /*!< encryption scheme */
@ -464,17 +213,7 @@ struct fil_space_crypt_status_t {
ulint rotate_max_page_number; /*!< max page if key rotating */
};
/*********************************************************************
Get crypt status for a space
@return 0 if crypt data found */
UNIV_INTERN
int
fil_space_crypt_get_status(
/*=======================*/
ulint id, /*!< in: space id */
struct fil_space_crypt_status_t * status); /*!< out: status */
/** Struct for retreiving statistics about encryption key rotation */
/** Statistics about encryption key rotation */
struct fil_crypt_stat_t {
ulint pages_read_from_cache;
ulint pages_read_from_disk;
@ -483,15 +222,7 @@ struct fil_crypt_stat_t {
ulint estimated_iops;
};
/*********************************************************************
Get crypt rotation statistics */
UNIV_INTERN
void
fil_crypt_total_stat(
/*==================*/
fil_crypt_stat_t* stat); /*!< out: crypt stat */
/** Struct for retreiving info about scrubbing */
/** Status info about scrubbing */
struct fil_space_scrub_status_t {
ulint space; /*!< tablespace id */
bool compressed; /*!< is space compressed */
@ -504,48 +235,271 @@ struct fil_space_scrub_status_t {
};
/*********************************************************************
Get scrub status for a space
@return 0 if no scrub info found */
UNIV_INTERN
int
fil_space_get_scrub_status(
/*=======================*/
ulint id, /*!< in: space id */
struct fil_space_scrub_status_t * status); /*!< out: status */
/*********************************************************************
Adjust encrypt tables */
Init space crypt */
UNIV_INTERN
void
fil_crypt_set_encrypt_tables(
/*=========================*/
uint val); /*!< in: New srv_encrypt_tables setting */
fil_space_crypt_init();
/*********************************************************************
Cleanup space crypt */
UNIV_INTERN
void
fil_space_crypt_cleanup();
/******************************************************************
Encrypt a buffer */
Create a fil_space_crypt_t object
@param[in] encrypt_mode FIL_ENCRYPTION_DEFAULT or
FIL_ENCRYPTION_ON or
FIL_ENCRYPTION_OFF
@param[in] key_id Encryption key id
@return crypt object */
UNIV_INTERN
fil_space_crypt_t*
fil_space_create_crypt_data(
fil_encryption_t encrypt_mode,
uint key_id)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Merge fil_space_crypt_t object
@param[in,out] dst Destination cryp data
@param[in] src Source crypt data */
UNIV_INTERN
void
fil_space_merge_crypt_data(
fil_space_crypt_t* dst,
const fil_space_crypt_t* src);
/******************************************************************
Read crypt data from a page (0)
@param[in] space space_id
@param[in] page Page 0
@param[in] offset Offset to crypt data
@return crypt data from page 0 or NULL. */
UNIV_INTERN
fil_space_crypt_t*
fil_space_read_crypt_data(
ulint space,
const byte* page,
ulint offset)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Free a crypt data object
@param[in,out] crypt_data crypt data to be freed */
UNIV_INTERN
void
fil_space_destroy_crypt_data(
fil_space_crypt_t **crypt_data);
/******************************************************************
Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry
@param[in] ptr Log entry start
@param[in] end_ptr Log entry end
@param[in] block buffer block
@return position on log buffer */
UNIV_INTERN
const byte*
fil_parse_write_crypt_data(
const byte* ptr,
const byte* end_ptr,
const buf_block_t* block)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Encrypt a buffer
@param[in,out] crypt_data Crypt data
@param[in] space space_id
@param[in] offset Page offset
@param[in] lsn Log sequence number
@param[in] src_frame Page to encrypt
@param[in] zip_size Compressed size or 0
@param[in,out] dst_frame Output buffer
@return encrypted buffer or NULL */
UNIV_INTERN
byte*
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame); /*!< in: outbut buffer */
fil_space_crypt_t* crypt_data,
ulint space,
ulint offset,
lsn_t lsn,
const byte* src_frame,
ulint zip_size,
byte* dst_frame)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Encrypt a page
@param[in] space Tablespace
@param[in] offset Page offset
@param[in] lsn Log sequence number
@param[in] src_frame Page to encrypt
@param[in,out] dst_frame Output buffer
@return encrypted buffer or NULL */
UNIV_INTERN
byte*
fil_space_encrypt(
const fil_space_t* space,
ulint offset,
lsn_t lsn,
byte* src_frame,
byte* dst_frame)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Decrypt a page
@param[in,out] crypt_data crypt_data
@param[in] tmp_frame Temporary buffer
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] err DB_SUCCESS or error
@return true if page decrypted, false if not.*/
UNIV_INTERN
bool
fil_space_decrypt(
fil_space_crypt_t* crypt_data,
byte* tmp_frame,
ulint page_size,
byte* src_frame,
dberr_t* err);
/******************************************************************
Decrypt a page
@param[in] space Tablespace
@param[in] tmp_frame Temporary buffer used for decrypting
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] decrypted true if page was decrypted
@return decrypted page, or original not encrypted page if decryption is
not needed.*/
UNIV_INTERN
byte*
fil_space_decrypt(
const fil_space_t* space,
byte* tmp_frame,
byte* src_frame,
bool* decrypted)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Calculate post encryption checksum
@param[in] zip_size zip_size or 0
@param[in] dst_frame Block where checksum is calculated
@return page checksum or BUF_NO_CHECKSUM_MAGIC
not needed. */
UNIV_INTERN
ulint
fil_crypt_calculate_checksum(
/*=========================*/
ulint zip_size, /*!< in: zip_size or 0 */
byte* dst_frame); /*!< in: page where to calculate */
ulint zip_size,
const byte* dst_frame)
MY_ATTRIBUTE((warn_unused_result));
/*********************************************************************
Verify that post encryption checksum match calculated checksum.
This function should be called only if tablespace contains crypt_data
metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
const fil_space_t* space,
ulint pageno)
MY_ATTRIBUTE((warn_unused_result));
/*********************************************************************
Adjust thread count for key rotation
@param[in] enw_cnt Number of threads to be used */
UNIV_INTERN
void
fil_crypt_set_thread_cnt(
uint new_cnt);
/*********************************************************************
Adjust max key age
@param[in] val New max key age */
UNIV_INTERN
void
fil_crypt_set_rotate_key_age(
uint val);
/*********************************************************************
Adjust rotation iops
@param[in] val New max roation iops */
UNIV_INTERN
void
fil_crypt_set_rotation_iops(
uint val);
/*********************************************************************
Adjust encrypt tables
@param[in] val New setting for innodb-encrypt-tables */
UNIV_INTERN
void
fil_crypt_set_encrypt_tables(
uint val);
/*********************************************************************
Init threads for key rotation */
UNIV_INTERN
void
fil_crypt_threads_init();
/*********************************************************************
Clean up key rotation threads resources */
UNIV_INTERN
void
fil_crypt_threads_cleanup();
/*********************************************************************
Wait for crypt threads to stop accessing space
@param[in] space Tablespace */
UNIV_INTERN
void
fil_space_crypt_close_tablespace(
const fil_space_t* space);
/*********************************************************************
Get crypt status for a space (used by information_schema)
@param[in] space Tablespace
@param[out] status Crypt status
return 0 if crypt data present */
UNIV_INTERN
void
fil_space_crypt_get_status(
const fil_space_t* space,
struct fil_space_crypt_status_t* status);
/*********************************************************************
Return crypt statistics
@param[out] stat Crypt statistics */
UNIV_INTERN
void
fil_crypt_total_stat(
fil_crypt_stat_t *stat);
/*********************************************************************
Get scrub status for a space (used by information_schema)
@param[in] space Tablespace
@param[out] status Scrub status
return 0 if data found */
UNIV_INTERN
void
fil_space_get_scrub_status(
const fil_space_t* space,
struct fil_space_scrub_status_t* status);
#ifndef UNIV_NONINL
#include "fil0crypt.ic"

32
storage/innobase/include/fil0crypt.ic
View file

@ -34,35 +34,3 @@ fil_page_is_encrypted(
{
return(mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0);
}
/*******************************************************************//**
Find out whether the page can be decrypted.
The function for decrypting the page should already be executed before this.
@return 1 if key provider not available or key is not available
0 if decryption should be possible
*/
UNIV_INLINE
bool
fil_page_encryption_status(
/*===================*/
const byte *buf, /*!< in: page */
ulint space_id) /*!< in: space_id */
{
fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space_id);
ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
if (page_type == FIL_PAGE_TYPE_FSP_HDR) {
if (crypt_data != NULL) {
if (!encryption_key_id_exists(crypt_data->key_id)) {
/* accessing table would surely fail, because no key or no key provider available */
return 1;
}
}
} else {
ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
if (!encryption_key_version_exists(crypt_data->key_id, key)) {
return 1;
}
}
return 0;
}

113
storage/innobase/include/fil0fil.h
View file

@ -185,8 +185,18 @@ extern fil_addr_t fil_addr_null;
#ifndef UNIV_INNOCHECKSUM
/* structure containing encryption specification */
typedef struct fil_space_crypt_struct fil_space_crypt_t;
/** Structure containing encryption specification */
struct fil_space_crypt_t;
/** Enum values for encryption table option */
enum fil_encryption_t {
/** Encrypted if innodb_encrypt_tables=ON (srv_encrypt_tables) */
FIL_ENCRYPTION_DEFAULT,
/** Encrypted */
FIL_ENCRYPTION_ON,
/** Not encrypted */
FIL_ENCRYPTION_OFF
};
/** The number of fsyncs done to the log */
extern ulint fil_n_log_flushes;
@ -276,7 +286,7 @@ struct fil_space_t {
.ibd file of tablespace and want to
stop temporarily posting of new i/o
requests on the file */
ibool stop_new_ops;
bool stop_new_ops;
/*!< we set this TRUE when we start
deleting a single-table tablespace.
When this is set following new ops
@ -343,7 +353,22 @@ struct fil_space_t {
UT_LIST_NODE_T(fil_space_t) space_list;
/*!< list of all spaces */
/*!< Protected by fil_system */
UT_LIST_NODE_T(fil_space_t) rotation_list;
/*!< list of spaces needing
key rotation */
bool is_in_rotation_list;
/*!< true if this space is
currently in key rotation list */
ulint magic_n;/*!< FIL_SPACE_MAGIC_N */
/** @return whether the tablespace is about to be dropped or truncated */
bool is_stopping() const
{
return stop_new_ops;
}
};
/** Value of fil_space_t::magic_n */
@ -399,6 +424,11 @@ struct fil_system_t {
request */
UT_LIST_BASE_NODE_T(fil_space_t) space_list;
/*!< list of all file spaces */
UT_LIST_BASE_NODE_T(fil_space_t) rotation_list;
/*!< list of all file spaces needing
key rotation.*/
ibool space_id_reuse_warned;
/* !< TRUE if fil_space_create()
has issued a warning about
@ -470,18 +500,24 @@ fil_space_truncate_start(
/*******************************************************************//**
Creates a space memory object and puts it to the 'fil system' hash table.
If there is an error, prints an error message to the .err log.
@param[in] name Space name
@param[in] id Space id
@param[in] flags Tablespace flags
@param[in] purpose FIL_TABLESPACE or FIL_LOG if log
@param[in] crypt_data Encryption information
@param[in] create_table True if this is create table
@param[in] mode Encryption mode
@return TRUE if success */
UNIV_INTERN
ibool
bool
fil_space_create(
/*=============*/
const char* name, /*!< in: space name */
ulint id, /*!< in: space id */
ulint zip_size,/*!< in: compressed page size, or
0 for uncompressed tablespaces */
ulint purpose, /*!< in: FIL_TABLESPACE, or FIL_LOG if log */
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
bool create_table); /*!< in: true if create table */
const char* name,
ulint id,
ulint flags,
ulint purpose,
fil_space_crypt_t* crypt_data,
bool create_table,
fil_encryption_t mode = FIL_ENCRYPTION_DEFAULT);
/*******************************************************************//**
Assigns a new space id for a new single-table tablespace. This works simply by
@ -604,6 +640,59 @@ fil_write_flushed_lsn_to_data_files(
/*================================*/
lsn_t lsn, /*!< in: lsn to write */
ulint arch_log_no); /*!< in: latest archived log file number */
/** Acquire a tablespace when it could be dropped concurrently.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@return the tablespace, or NULL if missing or being deleted */
fil_space_t*
fil_space_acquire(
ulint id)
MY_ATTRIBUTE((warn_unused_result));
/** Acquire a tablespace that may not exist.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@return the tablespace, or NULL if missing or being deleted */
fil_space_t*
fil_space_acquire_silent(
ulint id)
MY_ATTRIBUTE((warn_unused_result));
/** Release a tablespace acquired with fil_space_acquire().
@param[in,out] space tablespace to release */
void
fil_space_release(
fil_space_t* space);
/** Return the next fil_space_t.
Once started, the caller must keep calling this until it returns NULL.
fil_space_acquire() and fil_space_release() are invoked here which
blocks a concurrent operation from dropping the tablespace.
@param[in,out] prev_space Pointer to the previous fil_space_t.
If NULL, use the first fil_space_t on fil_system->space_list.
@return pointer to the next fil_space_t.
@retval NULL if this was the last */
fil_space_t*
fil_space_next(
fil_space_t* prev_space)
MY_ATTRIBUTE((warn_unused_result));
/** Return the next fil_space_t from key rotation list.
Once started, the caller must keep calling this until it returns NULL.
fil_space_acquire() and fil_space_release() are invoked here which
blocks a concurrent operation from dropping the tablespace.
@param[in,out] prev_space Pointer to the previous fil_space_t.
If NULL, use the first fil_space_t on fil_system->space_list.
@return pointer to the next fil_space_t.
@retval NULL if this was the last*/
fil_space_t*
fil_space_keyrotate_next(
fil_space_t* prev_space)
MY_ATTRIBUTE((warn_unused_result));
/*******************************************************************//**
Reads the flushed lsn, arch no, and tablespace flag fields from a data
file at database startup.

11
storage/innobase/include/fsp0fsp.h
View file

@ -1038,14 +1038,15 @@ fsp_flags_get_page_size(
/*====================*/
ulint flags); /*!< in: tablespace flags */
/*********************************************************************/
/* @return offset into fsp header where crypt data is stored */
/*********************************************************************
Compute offset after xdes where crypt data can be stored
@param[in] zip_size Compressed size or 0
@return offset */
UNIV_INTERN
ulint
fsp_header_get_crypt_offset(
/*========================*/
ulint zip_size, /*!< in: zip_size */
ulint* max_size); /*!< out: free space after offset */
const ulint zip_size)
MY_ATTRIBUTE((warn_unused_result));
#define fsp_page_is_free(space,page,mtr) \
fsp_page_is_free_func(space,page,mtr, __FILE__, __LINE__)

4
storage/innobase/include/srv0srv.h
View file

@ -185,6 +185,9 @@ struct srv_stats_t {
/** Number of encryption_get_latest_key_version calls */
ulint_ctr_64_t n_key_requests;
/** Number of spaces in keyrotation list */
ulint_ctr_64_t key_rotation_list_length;
};
extern const char* srv_main_thread_op_info;
@ -1032,6 +1035,7 @@ struct export_var_t{
ulint innodb_encryption_rotation_pages_flushed;
ulint innodb_encryption_rotation_estimated_iops;
ib_int64_t innodb_encryption_key_requests;
ib_int64_t innodb_key_rotation_list_length;
ulint innodb_scrub_page_reorganizations;
ulint innodb_scrub_page_splits;

42
storage/innobase/lock/lock0lock.cc
View file

@ -1,7 +1,7 @@
/*****************************************************************************
Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2014, 2016, MariaDB Corporation
Copyright (c) 2014, 2017, MariaDB Corporation
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -6398,12 +6398,13 @@ loop:
if (lock_get_type_low(lock) == LOCK_REC) {
if (load_page_first) {
ulint space = lock->un_member.rec_lock.space;
ulint zip_size= fil_space_get_zip_size(space);
ulint space_id = lock->un_member.rec_lock.space;
/* Check if the space is exists or not. only
when the space is valid, try to get the page. */
fil_space_t* space = fil_space_acquire(space_id);
ulint page_no = lock->un_member.rec_lock.page_no;
ibool tablespace_being_deleted = FALSE;
if (UNIV_UNLIKELY(zip_size == ULINT_UNDEFINED)) {
if (!space) {
/* It is a single table tablespace and
the .ibd file is missing (TRUNCATE
@ -6412,37 +6413,33 @@ loop:
load the page in the buffer pool. */
fprintf(file, "RECORD LOCKS on"
" non-existing space %lu\n",
(ulong) space);
" non-existing space: " ULINTPF "\n",
space_id);
goto print_rec;
}
const ulint zip_size = fsp_flags_get_zip_size(space->flags);
lock_mutex_exit();
mutex_exit(&trx_sys->mutex);
DEBUG_SYNC_C("innodb_monitor_before_lock_page_read");
/* Check if the space is exists or not. only when the space
is valid, try to get the page. */
tablespace_being_deleted = fil_inc_pending_ops(space, false);
if (!tablespace_being_deleted) {
if (space) {
mtr_start(&mtr);
buf_page_get_gen(space, zip_size, page_no,
buf_page_get_gen(space_id, zip_size, page_no,
RW_NO_LATCH, NULL,
BUF_GET_POSSIBLY_FREED,
__FILE__, __LINE__, &mtr);
mtr_commit(&mtr);
fil_decr_pending_ops(space);
} else {
fprintf(file, "RECORD LOCKS on"
" non-existing space %lu\n",
(ulong) space);
}
fil_space_release(space);
load_page_first = FALSE;
lock_mutex_enter();
@ -6868,7 +6865,7 @@ static
void
lock_rec_block_validate(
/*====================*/
ulint space,
ulint space_id,
ulint page_no)
{
/* The lock and the block that it is referring to may be freed at
@ -6881,10 +6878,11 @@ lock_rec_block_validate(
/* Make sure that the tablespace is not deleted while we are
trying to access the page. */
if (!fil_inc_pending_ops(space, true)) {
if (fil_space_t* space = fil_space_acquire(space_id)) {
mtr_start(&mtr);
block = buf_page_get_gen(
space, fil_space_get_zip_size(space),
space_id, fsp_flags_get_zip_size(space->flags),
page_no, RW_X_LATCH, NULL,
BUF_GET_POSSIBLY_FREED,
__FILE__, __LINE__, &mtr);
@ -6894,7 +6892,7 @@ lock_rec_block_validate(
ut_ad(lock_rec_validate_page(block));
mtr_commit(&mtr);
fil_decr_pending_ops(space);
fil_space_release(space);
}
}

2
storage/innobase/log/log0recv.cc
View file

@ -1297,7 +1297,7 @@ recv_parse_or_apply_log_rec_body(
}
break;
case MLOG_FILE_WRITE_CRYPT_DATA:
ptr = fil_parse_write_crypt_data(ptr, end_ptr, block);
ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block));
break;
default:
ptr = NULL;

2
storage/innobase/pars/pars0pars.cc
View file

@ -2020,7 +2020,7 @@ pars_create_table(
}
node = tab_create_graph_create(table, pars_sym_tab_global->heap, true,
FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
FIL_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
table_sym->resolved = TRUE;
table_sym->token_type = SYM_TABLE;

9
storage/innobase/row/row0ftsort.cc
View file

@ -221,7 +221,14 @@ row_fts_psort_info_init(
common_info->sort_event = os_event_create();
common_info->merge_event = os_event_create();
common_info->opt_doc_id_size = opt_doc_id_size;
crypt_data = fil_space_get_crypt_data(new_table->space);
/* Theoretically the tablespace can be dropped straight away.
In practice, the DDL completion will wait for this thread to
finish. */
if (fil_space_t* space = fil_space_acquire(new_table->space)) {
crypt_data = space->crypt_data;
fil_space_release(space);
}
if (crypt_data && crypt_data->should_encrypt()) {
common_info->crypt_data = crypt_data;

18
storage/innobase/row/row0merge.cc
View file

@ -3948,7 +3948,7 @@ row_merge_build_indexes(
{
merge_file_t* merge_files;
row_merge_block_t* block;
row_merge_block_t* crypt_block;
row_merge_block_t* crypt_block = NULL;
ulint block_size;
ulint i;
ulint j;
@ -3984,9 +3984,15 @@ row_merge_build_indexes(
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
/* Get crypt data from tablespace if present. */
crypt_data = fil_space_get_crypt_data(new_table->space);
crypt_block = NULL;
/* Get crypt data from tablespace if present. We should be protected
from concurrent DDL (e.g. drop table) by MDL-locks. */
fil_space_t* space = fil_space_acquire(new_table->space);
if (space) {
crypt_data = space->crypt_data;
} else {
DBUG_RETURN(DB_TABLESPACE_NOT_FOUND);
}
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
@ -4350,5 +4356,9 @@ func_exit:
}
}
if (space) {
fil_space_release(space);
}
DBUG_RETURN(error);
}

59
storage/innobase/row/row0mysql.cc
View file

@ -3289,21 +3289,17 @@ void
fil_wait_crypt_bg_threads(
dict_table_t* table)
{
uint start = time(0);
uint last = start;
if (table->space != 0) {
fil_space_crypt_mark_space_closing(table->space, table->crypt_data);
}
time_t start = time(0);
time_t last = start;
while (table->n_ref_count > 0) {
dict_mutex_exit_for_mysql();
os_thread_sleep(20000);
dict_mutex_enter_for_mysql();
uint now = time(0);
time_t now = time(0);
if (now >= last + 30) {
fprintf(stderr,
"WARNING: waited %u seconds "
"WARNING: waited %ld seconds "
"for ref-count on table: %s space: %u\n",
now - start, table->name, table->space);
last = now;
@ -3311,7 +3307,7 @@ fil_wait_crypt_bg_threads(
if (now >= start + 300) {
fprintf(stderr,
"WARNING: after %u seconds, gave up waiting "
"WARNING: after %ld seconds, gave up waiting "
"for ref-count on table: %s space: %u\n",
now - start, table->name, table->space);
break;
@ -3507,35 +3503,40 @@ row_truncate_table_for_mysql(
if (table->space && !DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) {
/* Discard and create the single-table tablespace. */
fil_space_crypt_t* crypt_data;
ulint space = table->space;
ulint flags = fil_space_get_flags(space);
ulint space_id = table->space;
ulint flags = ULINT_UNDEFINED;
ulint key_id = FIL_DEFAULT_ENCRYPTION_KEY;
fil_encryption_t mode = FIL_SPACE_ENCRYPTION_DEFAULT;
fil_encryption_t mode = FIL_ENCRYPTION_DEFAULT;
dict_get_and_save_data_dir_path(table, true);
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data) {
key_id = crypt_data->key_id;
mode = crypt_data->encryption;
if (fil_space_t* space = fil_space_acquire(space_id)) {
fil_space_crypt_t* crypt_data = space->crypt_data;
if (crypt_data) {
key_id = crypt_data->key_id;
mode = crypt_data->encryption;
}
flags = space->flags;
fil_space_release(space);
}
if (flags != ULINT_UNDEFINED
&& fil_discard_tablespace(space) == DB_SUCCESS) {
&& fil_discard_tablespace(space_id) == DB_SUCCESS) {
dict_index_t* index;
dict_hdr_get_new_id(NULL, NULL, &space);
dict_hdr_get_new_id(NULL, NULL, &space_id);
/* Lock all index trees for this table. We must
do so after dict_hdr_get_new_id() to preserve
the latch order */
dict_table_x_lock_indexes(table);
if (space == ULINT_UNDEFINED
if (space_id == ULINT_UNDEFINED
|| fil_create_new_single_table_tablespace(
space, table->name,
space_id, table->name,
table->data_dir_path,
flags, table->flags2,
FIL_IBD_FILE_INITIAL_SIZE,
@ -3553,21 +3554,21 @@ row_truncate_table_for_mysql(
goto funct_exit;
}
recreate_space = space;
recreate_space = space_id;
/* Replace the space_id in the data dictionary cache.
The persisent data dictionary (SYS_TABLES.SPACE
and SYS_INDEXES.SPACE) are updated later in this
function. */
table->space = space;
table->space = space_id;
index = dict_table_get_first_index(table);
do {
index->space = space;
index->space = space_id;
index = dict_table_get_next_index(index);
} while (index);
mtr_start_trx(&mtr, trx);
fsp_header_init(space,
fsp_header_init(space_id,
FIL_IBD_FILE_INITIAL_SIZE, &mtr);
mtr_commit(&mtr);
}
@ -4235,7 +4236,13 @@ row_drop_table_for_mysql(
/* If table has not yet have crypt_data, try to read it to
make freeing the table easier. */
if (!table->crypt_data) {
table->crypt_data = fil_space_get_crypt_data(table->space);
if (fil_space_t* space = fil_space_acquire_silent(table->space)) {
/* We use crypt data in dict_table_t in ha_innodb.cc
to push warnings to user thread. */
table->crypt_data = space->crypt_data;
fil_space_release(space);
}
}
/* We use the private SQL parser of Innobase to generate the

2
storage/innobase/srv/srv0srv.cc
View file

@ -1689,6 +1689,8 @@ srv_export_innodb_status(void)
crypt_stat.estimated_iops;
export_vars.innodb_encryption_key_requests =
srv_stats.n_key_requests;
export_vars.innodb_key_rotation_list_length =
srv_stats.key_rotation_list_length;
export_vars.innodb_scrub_page_reorganizations =
scrub_stat.page_reorganizations;

6
storage/innobase/srv/srv0start.cc
View file

@ -665,6 +665,7 @@ create_log_files(
FIL_LOG,
NULL /* no encryption yet */,
true /* this is create */);
ut_a(fil_validate());
logfile0 = fil_node_create(
@ -1153,13 +1154,14 @@ check_first_page:
if (i == 0) {
if (!crypt_data) {
crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
crypt_data = fil_space_create_crypt_data(FIL_ENCRYPTION_DEFAULT,
FIL_DEFAULT_ENCRYPTION_KEY);
}
flags = FSP_FLAGS_PAGE_SSIZE();
fil_space_create(name, 0, flags, FIL_TABLESPACE,
crypt_data, (*create_new_db) == true);
crypt_data, (*create_new_db) == true);
}
ut_a(fil_validate());

48
storage/xtradb/btr/btr0scrub.cc
View file

@ -111,11 +111,18 @@ log_scrub_failure(
Lock dict mutexes */
static
bool
btr_scrub_lock_dict_func(ulint space, bool lock_to_close_table,
btr_scrub_lock_dict_func(ulint space_id, bool lock_to_close_table,
const char * file, uint line)
{
uint start = time(0);
uint last = start;
time_t start = time(0);
time_t last = start;
/* FIXME: this is not the proper way of doing things. The
dict_sys->mutex should not be held by any thread for longer
than a few microseconds. It must not be held during I/O,
for example. So, what is the purpose for this busy-waiting?
This function should be rewritten as part of MDEV-8139:
Fix scrubbing tests. */
while (mutex_enter_nowait_func(&(dict_sys->mutex), file, line)) {
/* if we lock to close a table, we wait forever
@ -123,19 +130,24 @@ btr_scrub_lock_dict_func(ulint space, bool lock_to_close_table,
* is closing, and then instead give up
*/
if (lock_to_close_table == false) {
if (fil_crypt_is_closing(space)) {
fil_space_t* space = fil_space_acquire(space_id);
if (!space || space->stop_new_ops) {
if (space) {
fil_space_release(space);
}
return false;
}
fil_space_release(space);
}
os_thread_sleep(250000);
uint now = time(0);
time_t now = time(0);
if (now >= last + 30) {
fprintf(stderr,
"WARNING: %s:%u waited %u seconds for"
"WARNING: %s:%u waited %ld seconds for"
" dict_sys lock, space: %lu"
" lock_to_close_table: %u\n",
file, line, now - start, space,
" lock_to_close_table: %d\n",
file, line, now - start, space_id,
lock_to_close_table);
last = now;
@ -181,16 +193,24 @@ void
btr_scrub_table_close_for_thread(
btr_scrub_t *scrub_data)
{
if (scrub_data->current_table == NULL)
if (scrub_data->current_table == NULL) {
return;
}
bool lock_for_close = true;
btr_scrub_lock_dict(scrub_data->space, lock_for_close);
fil_space_t* space = fil_space_acquire(scrub_data->space);
/* perform the actual closing */
btr_scrub_table_close(scrub_data->current_table);
/* If tablespace is not marked as stopping perform
the actual close. */
if (space && !space->is_stopping()) {
mutex_enter(&dict_sys->mutex);
/* perform the actual closing */
btr_scrub_table_close(scrub_data->current_table);
mutex_exit(&dict_sys->mutex);
}
btr_scrub_unlock_dict();
if (space) {
fil_space_release(space);
}
scrub_data->current_table = NULL;
scrub_data->current_index = NULL;

78
storage/xtradb/buf/buf0buf.cc
View file

@ -692,7 +692,7 @@ buf_page_is_corrupted(
ulint checksum_field2;
ulint space_id = mach_read_from_4(
read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint page_type = mach_read_from_4(
ulint page_type = mach_read_from_2(
read_buf + FIL_PAGE_TYPE);
/* We can trust page type if page compression is set on tablespace
@ -4571,11 +4571,14 @@ buf_page_check_corrupt(
byte* dst_frame = (zip_size) ? bpage->zip.data :
((buf_block_t*) bpage)->frame;
ulint space_id = bpage->space;
fil_space_t* space = fil_space_found_by_id(space_id);
fil_space_crypt_t* crypt_data = space->crypt_data;
fil_space_t* space = fil_space_acquire_silent(space_id);
bool still_encrypted = false;
bool corrupted = false;
ulint page_type = mach_read_from_2(dst_frame + FIL_PAGE_TYPE);
fil_space_crypt_t* crypt_data = NULL;
ut_ad(space);
crypt_data = space->crypt_data;
/* In buf_decrypt_after_read we have either decrypted the page if
page post encryption checksum matches and used key_id is found
@ -4605,7 +4608,7 @@ buf_page_check_corrupt(
ib_logf(IB_LOG_LEVEL_ERROR,
"%s: Block in space_id " ULINTPF " in file %s corrupted.",
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ? "Maybe corruption" : "Corruption",
space_id, space->name ? space->name : "NULL");
space_id, (space && space->name) ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"Based on page type %s (" ULINTPF ")",
fil_get_page_type_name(page_type), page_type);
@ -4615,7 +4618,7 @@ buf_page_check_corrupt(
ib_logf(IB_LOG_LEVEL_ERROR,
"Block in space_id " ULINTPF " in file %s encrypted.",
space_id, space->name ? space->name : "NULL");
space_id, (space && space->name) ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"However key management plugin or used key_version %u is not found or"
" used encryption algorithm or method does not match.",
@ -4627,6 +4630,10 @@ buf_page_check_corrupt(
}
}
if (space) {
fil_space_release(space);
}
return corrupted;
}
@ -6248,13 +6255,6 @@ buf_page_encrypt_before_write(
byte* src_frame,
ulint space_id)
{
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
ulint zip_size = buf_page_get_zip_size(bpage);
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
bool page_compressed = fil_space_is_page_compressed(bpage->space);
bool encrypted = true;
bpage->real_size = UNIV_PAGE_SIZE;
fil_page_type_validate(src_frame);
@ -6271,7 +6271,20 @@ buf_page_encrypt_before_write(
return src_frame;
}
if (crypt_data != NULL && crypt_data->not_encrypted()) {
fil_space_t* space = fil_space_acquire_silent(space_id);
/* Tablespace must exist during write operation */
if (!space) {
/* This could be true on discard if we have injected a error
case e.g. in innodb.innodb-wl5522-debug-zip so that space
is already marked as stop_new_ops = true. */
return src_frame;
}
fil_space_crypt_t* crypt_data = space->crypt_data;
bool encrypted = true;
if (space->crypt_data != NULL && space->crypt_data->not_encrypted()) {
/* Encryption is disabled */
encrypted = false;
}
@ -6288,11 +6301,17 @@ buf_page_encrypt_before_write(
encrypted = false;
}
bool page_compressed = fil_space_is_page_compressed(bpage->space);
if (!encrypted && !page_compressed) {
/* No need to encrypt or page compress the page */
fil_space_release(space);
return src_frame;
}
ulint zip_size = buf_page_get_zip_size(bpage);
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
/* Find free slot from temporary memory array */
buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
slot->out_buf = NULL;
@ -6302,11 +6321,10 @@ buf_page_encrypt_before_write(
if (!page_compressed) {
/* Encrypt page content */
byte* tmp = fil_space_encrypt(bpage->space,
byte* tmp = fil_space_encrypt(space,
bpage->offset,
bpage->newest_modification,
src_frame,
zip_size,
dst_frame);
ulint key_version = mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
@ -6344,11 +6362,10 @@ buf_page_encrypt_before_write(
if(encrypted) {
/* And then we encrypt the page content */
tmp = fil_space_encrypt(bpage->space,
tmp = fil_space_encrypt(space,
bpage->offset,
bpage->newest_modification,
tmp,
zip_size,
dst_frame);
}
@ -6359,6 +6376,7 @@ buf_page_encrypt_before_write(
fil_page_type_validate(dst_frame);
#endif
fil_space_release(space);
// return dst_frame which will be written
return dst_frame;
}
@ -6384,16 +6402,6 @@ buf_page_decrypt_after_read(
bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame);
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
bool success = true;
ulint space_id = mach_read_from_4(
dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
/* Page is encrypted if encryption information is found from
tablespace and page contains used key_version. This is true
also for pages first compressed and then encrypted. */
if (!crypt_data) {
key_version = 0;
}
bpage->key_version = key_version;
@ -6402,6 +6410,17 @@ buf_page_decrypt_after_read(
return (true);
}
fil_space_t* space = fil_space_acquire(bpage->space);
fil_space_crypt_t* crypt_data = space->crypt_data;
/* Page is encrypted if encryption information is found from
tablespace and page contains used key_version. This is true
also for pages first compressed and then encrypted. */
if (!crypt_data) {
key_version = 0;
}
if (page_compressed) {
/* the page we read is unencrypted */
/* Find free slot from temporary memory array */
@ -6443,15 +6462,15 @@ buf_page_decrypt_after_read(
#endif
/* decrypt using crypt_buf to dst_frame */
byte* res = fil_space_decrypt(bpage->space,
byte* res = fil_space_decrypt(space,
slot->crypt_buf,
size,
dst_frame,
&bpage->encrypted);
if (!res) {
success = false;
}
#ifdef UNIV_DEBUG
fil_page_type_validate(dst_frame);
#endif
@ -6482,5 +6501,6 @@ buf_page_decrypt_after_read(
}
}
fil_space_release(space);
return (success);
}

1140
storage/xtradb/fil/fil0crypt.cc
View file

File diff suppressed because it is too large Load diff

453
storage/xtradb/fil/fil0fil.cc
View file

@ -157,7 +157,11 @@ UNIV_INTERN mysql_pfs_key_t fil_space_latch_key;
/** The tablespace memory cache. This variable is NULL before the module is
initialized. */
fil_system_t* fil_system = NULL;
UNIV_INTERN fil_system_t* fil_system = NULL;
/** At this age or older a space/page will be rotated */
UNIV_INTERN extern uint srv_fil_crypt_rotate_key_age;
UNIV_INTERN extern ib_mutex_t fil_crypt_threads_mutex;
/** Determine if (i) is a user tablespace id or not. */
# define fil_is_user_tablespace_id(i) ((i) > srv_undo_tablespaces_open)
@ -169,7 +173,7 @@ fil_system_t* fil_system = NULL;
&& srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC)\
|| ((s)->purpose == FIL_LOG \
&& srv_unix_file_flush_method == SRV_UNIX_ALL_O_DIRECT))
#else /* __WIN__ */
# define fil_buffering_disabled(s) (0)
#endif /* __WIN__ */
@ -618,6 +622,7 @@ fil_node_open_file(
node->handle = os_file_create_simple_no_error_handling(
innodb_file_data_key, node->name, OS_FILE_OPEN,
OS_FILE_READ_ONLY, &success, 0);
if (!success) {
/* The following call prints an error message */
os_file_get_last_error(true);
@ -669,6 +674,16 @@ fil_node_open_file(
const ulint space_id = fsp_header_get_space_id(page);
ulint flags = fsp_header_get_flags(page);
/* Try to read crypt_data from page 0 if it is not yet
read. */
if (!node->space->page_0_crypt_read) {
ulint offset = fsp_header_get_crypt_offset(
fsp_flags_get_zip_size(flags));
ut_ad(node->space->crypt_data == NULL);
node->space->crypt_data = fil_space_read_crypt_data(space_id, page, offset);
node->space->page_0_crypt_read = true;
}
ut_free(buf2);
os_file_close(node->handle);
@ -1491,17 +1506,24 @@ fil_space_contains_node(
/*******************************************************************//**
Creates a space memory object and puts it to the 'fil system' hash table.
If there is an error, prints an error message to the .err log.
@param[in] name Space name
@param[in] id Space id
@param[in] flags Tablespace flags
@param[in] purpose FIL_TABLESPACE or FIL_LOG if log
@param[in] crypt_data Encryption information
@param[in] create_table True if this is create table
@param[in] mode Encryption mode
@return TRUE if success */
UNIV_INTERN
ibool
bool
fil_space_create(
/*=============*/
const char* name, /*!< in: space name */
ulint id, /*!< in: space id */
ulint flags, /*!< in: tablespace flags */
ulint purpose,/*!< in: FIL_TABLESPACE, or FIL_LOG if log */
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
bool create_table) /*!< in: true if create table */
const char* name,
ulint id,
ulint flags,
ulint purpose,
fil_space_crypt_t* crypt_data,
bool create_table,
fil_encryption_t mode)
{
fil_space_t* space;
@ -1525,7 +1547,7 @@ fil_space_create(
mutex_exit(&fil_system->mutex);
return(FALSE);
return(false);
}
ib_logf(IB_LOG_LEVEL_WARN,
@ -1552,7 +1574,7 @@ fil_space_create(
mutex_exit(&fil_system->mutex);
return(FALSE);
return(false);
}
space = static_cast<fil_space_t*>(mem_zalloc(sizeof(*space)));
@ -1583,17 +1605,6 @@ fil_space_create(
space->flags = flags;
space->magic_n = FIL_SPACE_MAGIC_N;
space->printed_compression_failure = false;
rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP);
HASH_INSERT(fil_space_t, hash, fil_system->spaces, id, space);
HASH_INSERT(fil_space_t, name_hash, fil_system->name_hash,
ut_fold_string(name), space);
space->is_in_unflushed_spaces = false;
space->is_corrupt = FALSE;
space->crypt_data = crypt_data;
/* In create table we write page 0 so we have already
@ -1612,11 +1623,33 @@ fil_space_create(
space->crypt_data ? space->crypt_data->encryption : 0);
#endif
rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP);
HASH_INSERT(fil_space_t, hash, fil_system->spaces, id, space);
HASH_INSERT(fil_space_t, name_hash, fil_system->name_hash,
ut_fold_string(name), space);
UT_LIST_ADD_LAST(space_list, fil_system->space_list, space);
mutex_exit(&fil_system->mutex);
/* Inform key rotation that there could be something
to do */
if (purpose == FIL_TABLESPACE && !srv_fil_crypt_rotate_key_age && fil_crypt_threads_event &&
(mode == FIL_ENCRYPTION_ON || mode == FIL_ENCRYPTION_OFF ||
srv_encrypt_tables)) {
/* Key rotation is not enabled, need to inform background
encryption threads. */
UT_LIST_ADD_LAST(rotation_list, fil_system->rotation_list, space);
mutex_exit(&fil_system->mutex);
space->is_in_rotation_list = true;
mutex_enter(&fil_crypt_threads_mutex);
os_event_set(fil_crypt_threads_event);
mutex_exit(&fil_crypt_threads_mutex);
} else {
mutex_exit(&fil_system->mutex);
}
return(TRUE);
return(true);
}
/*******************************************************************//**
@ -1728,6 +1761,11 @@ fil_space_free(
space);
}
if (space->is_in_rotation_list) {
space->is_in_rotation_list = false;
UT_LIST_REMOVE(rotation_list, fil_system->rotation_list, space);
}
UT_LIST_REMOVE(space_list, fil_system->space_list, space);
ut_a(space->magic_n == FIL_SPACE_MAGIC_N);
@ -2390,7 +2428,6 @@ fil_read_first_page(
const char* check_msg = NULL;
fil_space_crypt_t* cdata;
buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
/* Align the memory for a possible read from a raw device */
@ -2430,7 +2467,7 @@ fil_read_first_page(
ulint space = fsp_header_get_space_id(page);
ulint offset = fsp_header_get_crypt_offset(
fsp_flags_get_zip_size(*flags), NULL);
fsp_flags_get_zip_size(*flags));
cdata = fil_space_read_crypt_data(space, page, offset);
@ -2809,7 +2846,7 @@ fil_op_log_parse_or_replay(
space_id, name, path, flags,
DICT_TF2_USE_TABLESPACE,
FIL_IBD_FILE_INITIAL_SIZE,
FIL_SPACE_ENCRYPTION_DEFAULT,
FIL_ENCRYPTION_DEFAULT,
FIL_DEFAULT_ENCRYPTION_KEY) != DB_SUCCESS) {
ut_error;
}
@ -2933,16 +2970,27 @@ fil_check_pending_operations(
*space = 0;
/* Wait for crypt threads to stop accessing space */
fil_space_crypt_close_tablespace(id);
mutex_enter(&fil_system->mutex);
fil_space_t* sp = fil_space_get_by_id(id);
if (sp) {
sp->stop_new_ops = TRUE;
/* space could be freed by other threads as soon
as n_pending_ops reaches 0, thus increment pending
ops here. */
sp->n_pending_ops++;
}
mutex_exit(&fil_system->mutex);
/* Wait for crypt threads to stop accessing space */
if (sp) {
fil_space_crypt_close_tablespace(sp);
/* We have "acquired" this space and must
free it now as below we compare n_pending_ops. */
fil_space_release(sp);
}
/* Check for pending change buffer merges. */
do {
@ -3985,13 +4033,13 @@ fil_create_new_single_table_tablespace(
/* Create crypt data if the tablespace is either encrypted or user has
requested it to remain unencrypted. */
if (mode == FIL_SPACE_ENCRYPTION_ON || mode == FIL_SPACE_ENCRYPTION_OFF ||
if (mode == FIL_ENCRYPTION_ON || mode == FIL_ENCRYPTION_OFF ||
srv_encrypt_tables) {
crypt_data = fil_space_create_crypt_data(mode, key_id);
}
success = fil_space_create(tablename, space_id, flags, FIL_TABLESPACE,
crypt_data, true);
crypt_data, true, mode);
if (!success || !fil_node_create(path, size, space_id, FALSE)) {
err = DB_ERROR;
@ -6699,7 +6747,8 @@ fil_iterate(
page_type == FIL_PAGE_PAGE_COMPRESSED);
/* If tablespace is encrypted, we need to decrypt
the page. */
the page. Note that tablespaces are not in
fil_system during import. */
if (encrypted) {
decrypted = fil_space_decrypt(
iter.crypt_data,
@ -6952,8 +7001,11 @@ fil_tablespace_iterate(
iter.n_io_buffers = n_io_buffers;
iter.page_size = callback.get_page_size();
/* In MariaDB/MySQL 5.6 tablespace does not exist
during import, therefore we can't use space directly
here. */
ulint crypt_data_offset = fsp_header_get_crypt_offset(
callback.get_zip_size(), 0);
callback.get_zip_size());
/* read (optional) crypt data */
iter.crypt_data = fil_space_read_crypt_data(
@ -6995,7 +7047,7 @@ fil_tablespace_iterate(
mem_free(io_buffer);
if (iter.crypt_data != NULL) {
if (crypt_io_buffer != NULL) {
mem_free(crypt_io_buffer);
iter.crypt_io_buffer = NULL;
fil_space_destroy_crypt_data(&iter.crypt_data);
@ -7254,7 +7306,7 @@ fil_space_set_corrupt(
space = fil_space_get_by_id(space_id);
if (space) {
space->is_corrupt = TRUE;
space->is_corrupt = true;
}
mutex_exit(&fil_system->mutex);
@ -7289,36 +7341,6 @@ fil_get_first_space()
return out_id;
}
/******************************************************************
Get id of first tablespace that has node or ULINT_UNDEFINED if none */
UNIV_INTERN
ulint
fil_get_first_space_safe()
/*======================*/
{
ulint out_id = ULINT_UNDEFINED;
fil_space_t* space;
mutex_enter(&fil_system->mutex);
space = UT_LIST_GET_FIRST(fil_system->space_list);
if (space != NULL) {
do
{
if (!space->stop_new_ops && UT_LIST_GET_LEN(space->chain) > 0) {
out_id = space->id;
break;
}
space = UT_LIST_GET_NEXT(space_list, space);
} while (space != NULL);
}
mutex_exit(&fil_system->mutex);
return out_id;
}
/******************************************************************
Get id of next tablespace or ULINT_UNDEFINED if none */
UNIV_INTERN
@ -7360,165 +7382,206 @@ fil_get_next_space(
return out_id;
}
/******************************************************************
Get id of next tablespace that has node or ULINT_UNDEFINED if none */
UNIV_INTERN
ulint
fil_get_next_space_safe(
/*====================*/
ulint id) /*!< in: previous space id */
/** Acquire a tablespace when it could be dropped concurrently.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@param[in] silent whether to silently ignore missing tablespaces
@return the tablespace, or NULL if missing or being deleted */
inline
fil_space_t*
fil_space_acquire_low(
ulint id,
bool silent)
{
bool found;
fil_space_t* space;
ulint out_id = ULINT_UNDEFINED;
fil_space_t* space;
mutex_enter(&fil_system->mutex);
space = fil_space_get_by_id(id);
if (space == NULL) {
/* we didn't find it...search for space with space->id > id */
found = false;
if (!silent) {
ib_logf(IB_LOG_LEVEL_WARN, "Trying to access missing"
" tablespace " ULINTPF ".", id);
ut_error;
}
} else if (space->stop_new_ops) {
space = NULL;
} else {
space->n_pending_ops++;
}
mutex_exit(&fil_system->mutex);
return(space);
}
/** Acquire a tablespace when it could be dropped concurrently.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@return the tablespace, or NULL if missing or being deleted */
fil_space_t*
fil_space_acquire(
ulint id)
{
return(fil_space_acquire_low(id, false));
}
/** Acquire a tablespace that may not exist.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@return the tablespace, or NULL if missing or being deleted */
fil_space_t*
fil_space_acquire_silent(
ulint id)
{
return(fil_space_acquire_low(id, true));
}
/** Release a tablespace acquired with fil_space_acquire().
@param[in,out] space tablespace to release */
void
fil_space_release(
fil_space_t* space)
{
mutex_enter(&fil_system->mutex);
ut_ad(space->magic_n == FIL_SPACE_MAGIC_N);
ut_ad(space->n_pending_ops > 0);
space->n_pending_ops--;
mutex_exit(&fil_system->mutex);
}
/** Return the next fil_space_t.
Once started, the caller must keep calling this until it returns NULL.
fil_space_acquire() and fil_space_release() are invoked here which
blocks a concurrent operation from dropping the tablespace.
@param[in] prev_space Pointer to the previous fil_space_t.
If NULL, use the first fil_space_t on fil_system->space_list.
@return pointer to the next fil_space_t.
@retval NULL if this was the last*/
fil_space_t*
fil_space_next(
fil_space_t* prev_space)
{
fil_space_t* space=prev_space;
mutex_enter(&fil_system->mutex);
if (prev_space == NULL) {
space = UT_LIST_GET_FIRST(fil_system->space_list);
/* We can trust that space is not NULL because at least the
system tablespace is always present and loaded first. */
space->n_pending_ops++;
} else {
/* we found it, take next available space */
found = true;
}
ut_ad(space->n_pending_ops > 0);
while ((space = UT_LIST_GET_NEXT(space_list, space)) != NULL) {
/* Move on to the next fil_space_t */
space->n_pending_ops--;
space = UT_LIST_GET_NEXT(space_list, space);
if (!found && space->id <= id)
continue;
/* Skip spaces that are being created by
fil_ibd_create(), or dropped, or !tablespace. */
while (space != NULL
&& (UT_LIST_GET_LEN(space->chain) == 0
|| space->stop_new_ops
|| space->purpose != FIL_TABLESPACE)) {
space = UT_LIST_GET_NEXT(space_list, space);
}
if (!space->stop_new_ops) {
/* inc reference to prevent drop */
out_id = space->id;
break;
if (space != NULL) {
space->n_pending_ops++;
}
}
mutex_exit(&fil_system->mutex);
return out_id;
return(space);
}
/******************************************************************
Get crypt data for a tablespace */
UNIV_INTERN
fil_space_crypt_t*
fil_space_get_crypt_data(
/*=====================*/
ulint id) /*!< in: space id */
/**
Remove space from key rotation list if there are no more
pending operations.
@param[in] space Tablespace */
static
void
fil_space_remove_from_keyrotation(
fil_space_t* space)
{
fil_space_t* space;
fil_space_crypt_t* crypt_data = NULL;
ut_ad(mutex_own(&fil_system->mutex));
ut_ad(space);
ut_ad(fil_system);
mutex_enter(&fil_system->mutex);
space = fil_space_get_by_id(id);
mutex_exit(&fil_system->mutex);
if (space != NULL) {
/* If we have not yet read the page0
of this tablespace we will do it now. */
if (!space->crypt_data && !space->page_0_crypt_read) {
ulint space_id = space->id;
fil_node_t* node;
ut_a(space->crypt_data == NULL);
node = UT_LIST_GET_FIRST(space->chain);
byte *buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
byte *page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE));
fil_read(true, space_id, 0, 0, 0, UNIV_PAGE_SIZE, page,
NULL, NULL);
ulint offset = fsp_header_get_crypt_offset(
fsp_header_get_zip_size(page), NULL);
space->crypt_data = fil_space_read_crypt_data(space_id, page, offset);
ut_free(buf);
#ifdef UNIV_DEBUG
ib_logf(IB_LOG_LEVEL_INFO,
"Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d.",
space_id,
space->name,
space->crypt_data ? space->crypt_data->key_id : 0,
space->crypt_data ? space->crypt_data->encryption : 0,
node->handle);
#endif
ut_a(space->id == space_id);
space->page_0_crypt_read = true;
}
crypt_data = space->crypt_data;
if (!space->page_0_crypt_read) {
ib_logf(IB_LOG_LEVEL_WARN,
"Space %lu name %s contains encryption %d information for key_id %u but page0 is not read.",
space->id,
space->name,
space->crypt_data ? space->crypt_data->encryption : 0,
space->crypt_data ? space->crypt_data->key_id : 0);
}
if (space->n_pending_ops == 0) {
space->is_in_rotation_list = false;
UT_LIST_REMOVE(rotation_list, fil_system->rotation_list, space);
}
return(crypt_data);
}
/******************************************************************
Get crypt data for a tablespace */
UNIV_INTERN
fil_space_crypt_t*
fil_space_set_crypt_data(
/*=====================*/
ulint id, /*!< in: space id */
fil_space_crypt_t* crypt_data) /*!< in: crypt data */
{
fil_space_t* space;
fil_space_crypt_t* free_crypt_data = NULL;
fil_space_crypt_t* ret_crypt_data = NULL;
ut_ad(fil_system);
/** Return the next fil_space_t from key rotation list.
Once started, the caller must keep calling this until it returns NULL.
fil_space_acquire() and fil_space_release() are invoked here which
blocks a concurrent operation from dropping the tablespace.
@param[in] prev_space Pointer to the previous fil_space_t.
If NULL, use the first fil_space_t on fil_system->space_list.
@return pointer to the next fil_space_t.
@retval NULL if this was the last*/
fil_space_t*
fil_space_keyrotate_next(
fil_space_t* prev_space)
{
fil_space_t* space = prev_space;
fil_space_t* old = NULL;
mutex_enter(&fil_system->mutex);
space = fil_space_get_by_id(id);
if (space != NULL) {
if (space->crypt_data != NULL) {
/* Here we need to release fil_system mutex to
avoid mutex deadlock assertion. Here we would
take mutexes in order fil_system, crypt_data and
in fil_crypt_start_encrypting_space we would
take them in order crypt_data, fil_system
at fil_space_get_flags -> fil_space_get_space */
mutex_exit(&fil_system->mutex);
fil_space_merge_crypt_data(space->crypt_data,
crypt_data);
ret_crypt_data = space->crypt_data;
free_crypt_data = crypt_data;
} else {
space->crypt_data = crypt_data;
ret_crypt_data = space->crypt_data;
mutex_exit(&fil_system->mutex);
if (UT_LIST_GET_LEN(fil_system->rotation_list) == 0) {
if (space) {
ut_ad(space->n_pending_ops > 0);
space->n_pending_ops--;
fil_space_remove_from_keyrotation(space);
}
} else {
/* there is a small risk that tablespace has been deleted */
free_crypt_data = crypt_data;
mutex_exit(&fil_system->mutex);
return(NULL);
}
if (free_crypt_data != NULL) {
/* there was already crypt data present and the new crypt
* data provided as argument to this function has been merged
* into that => free new crypt data
*/
fil_space_destroy_crypt_data(&free_crypt_data);
if (prev_space == NULL) {
space = UT_LIST_GET_FIRST(fil_system->rotation_list);
/* We can trust that space is not NULL because we
checked list length above */
} else {
ut_ad(space->n_pending_ops > 0);
/* Move on to the next fil_space_t */
space->n_pending_ops--;
old = space;
space = UT_LIST_GET_NEXT(rotation_list, space);
fil_space_remove_from_keyrotation(old);
}
return ret_crypt_data;
/* Skip spaces that are being created by fil_ibd_create(),
or dropped. Note that rotation_list contains only
space->purpose == FIL_TABLESPACE. */
while (space != NULL
&& (UT_LIST_GET_LEN(space->chain) == 0
|| space->stop_new_ops)) {
old = space;
space = UT_LIST_GET_NEXT(rotation_list, space);
fil_space_remove_from_keyrotation(old);
}
if (space != NULL) {
space->n_pending_ops++;
}
mutex_exit(&fil_system->mutex);
return(space);
}

40
storage/xtradb/fsp/fsp0fsp.cc
View file

@ -680,7 +680,7 @@ UNIV_INTERN
void
fsp_header_init(
/*============*/
ulint space, /*!< in: space id */
ulint space_id, /*!< in: space id */
ulint size, /*!< in: current size in blocks */
mtr_t* mtr) /*!< in/out: mini-transaction */
{
@ -692,11 +692,11 @@ fsp_header_init(
ut_ad(mtr);
mtr_x_lock(fil_space_get_latch(space, &flags), mtr);
mtr_x_lock(fil_space_get_latch(space_id, &flags), mtr);
zip_size = fsp_flags_get_zip_size(flags);
block = buf_page_create(space, 0, zip_size, mtr);
buf_page_get(space, zip_size, 0, RW_X_LATCH, mtr);
block = buf_page_create(space_id, 0, zip_size, mtr);
buf_page_get(space_id, zip_size, 0, RW_X_LATCH, mtr);
buf_block_dbg_add_level(block, SYNC_FSP_PAGE);
/* The prior contents of the file page should be ignored */
@ -709,7 +709,7 @@ fsp_header_init(
header = FSP_HEADER_OFFSET + page;
mlog_write_ulint(header + FSP_SPACE_ID, space, MLOG_4BYTES, mtr);
mlog_write_ulint(header + FSP_SPACE_ID, space_id, MLOG_4BYTES, mtr);
mlog_write_ulint(header + FSP_NOT_USED, 0, MLOG_4BYTES, mtr);
mlog_write_ulint(header + FSP_SIZE, size, MLOG_4BYTES, mtr);
@ -725,18 +725,23 @@ fsp_header_init(
flst_init(header + FSP_SEG_INODES_FREE, mtr);
mlog_write_ull(header + FSP_SEG_ID, 1, mtr);
if (space == 0) {
fsp_fill_free_list(FALSE, space, header, mtr);
if (space_id == 0) {
fsp_fill_free_list(FALSE, space_id, header, mtr);
btr_create(DICT_CLUSTERED | DICT_UNIVERSAL | DICT_IBUF,
0, 0, DICT_IBUF_ID_MIN + space,
0, 0, DICT_IBUF_ID_MIN + space_id,
dict_ind_redundant, mtr);
} else {
fsp_fill_free_list(TRUE, space, header, mtr);
fsp_fill_free_list(TRUE, space_id, header, mtr);
}
ulint maxsize = 0;
ulint offset = fsp_header_get_crypt_offset(zip_size, &maxsize);
fil_space_write_crypt_data(space, page, offset, maxsize, mtr);
fil_space_t* space = fil_space_acquire(space_id);
ut_ad(space);
if (space->crypt_data) {
space->crypt_data->write_page0(page, mtr);
}
fil_space_release(space);
}
#endif /* !UNIV_HOTBACKUP */
@ -4149,12 +4154,11 @@ fsp_print(
/**********************************************************************//**
Compute offset after xdes where crypt data can be stored
@param[in] zip_size Compressed size or 0
@return offset */
ulint
fsp_header_get_crypt_offset(
/*========================*/
ulint zip_size, /*!< in: zip_size */
ulint* max_size) /*!< out: free space available for crypt data */
const ulint zip_size)
{
ulint pageno = 0;
/* compute first page_no that will have xdes stored on page != 0*/
@ -4169,12 +4173,6 @@ fsp_header_get_crypt_offset(
ulint iv_offset = XDES_ARR_OFFSET +
XDES_SIZE * (1 + xdes_calc_descriptor_index(zip_size, pageno));
if (max_size != NULL) {
/* return how much free space there is available on page */
*max_size = (zip_size ? zip_size : UNIV_PAGE_SIZE) -
(FSP_HEADER_OFFSET + iv_offset + FIL_PAGE_DATA_END);
}
return FSP_HEADER_OFFSET + iv_offset;
}

2
storage/xtradb/fts/fts0fts.cc
View file

@ -1989,7 +1989,7 @@ fts_create_one_index_table(
dict_mem_table_add_col(new_table, heap, "ilist", DATA_BLOB,
4130048, 0);
error = row_create_table_for_mysql(new_table, trx, false, FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
error = row_create_table_for_mysql(new_table, trx, false, FIL_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
if (error != DB_SUCCESS) {
trx->error_state = error;

34
storage/xtradb/handler/ha_innodb.cc
View file

@ -1290,6 +1290,9 @@ static SHOW_VAR innodb_status_variables[]= {
{"encryption_rotation_estimated_iops",
(char*) &export_vars.innodb_encryption_rotation_estimated_iops,
SHOW_LONG},
{"encryption_key_rotation_list_length",
(char*)&export_vars.innodb_key_rotation_list_length,
SHOW_LONGLONG},
/* Scrubing feature */
{"scrub_background_page_reorganizations",
@ -12383,7 +12386,7 @@ ha_innobase::check_table_options(
atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes;
fil_encryption_t encrypt = (fil_encryption_t)options->encryption;
if (encrypt != FIL_SPACE_ENCRYPTION_DEFAULT && !use_tablespace) {
if (encrypt != FIL_ENCRYPTION_DEFAULT && !use_tablespace) {
push_warning(
thd, Sql_condition::WARN_LEVEL_WARN,
HA_WRONG_CREATE_OPTION,
@ -12391,7 +12394,7 @@ ha_innobase::check_table_options(
return "ENCRYPTED";
}
if (encrypt == FIL_SPACE_ENCRYPTION_OFF && srv_encrypt_tables == 2) {
if (encrypt == FIL_ENCRYPTION_OFF && srv_encrypt_tables == 2) {
push_warning(
thd, Sql_condition::WARN_LEVEL_WARN,
HA_WRONG_CREATE_OPTION,
@ -12472,8 +12475,8 @@ ha_innobase::check_table_options(
}
/* If encryption is set up make sure that used key_id is found */
if (encrypt == FIL_SPACE_ENCRYPTION_ON ||
(encrypt == FIL_SPACE_ENCRYPTION_DEFAULT && srv_encrypt_tables)) {
if (encrypt == FIL_ENCRYPTION_ON ||
(encrypt == FIL_ENCRYPTION_DEFAULT && srv_encrypt_tables)) {
if (!encryption_key_id_exists((unsigned int)options->encryption_key_id)) {
push_warning_printf(
thd, Sql_condition::WARN_LEVEL_WARN,
@ -12487,7 +12490,7 @@ ha_innobase::check_table_options(
}
/* Ignore nondefault key_id if encryption is set off */
if (encrypt == FIL_SPACE_ENCRYPTION_OFF &&
if (encrypt == FIL_ENCRYPTION_OFF &&
options->encryption_key_id != THDVAR(thd, default_encryption_key_id)) {
push_warning_printf(
thd, Sql_condition::WARN_LEVEL_WARN,
@ -12500,7 +12503,7 @@ ha_innobase::check_table_options(
/* If default encryption is used make sure that used kay is found
from key file. */
if (encrypt == FIL_SPACE_ENCRYPTION_DEFAULT &&
if (encrypt == FIL_ENCRYPTION_DEFAULT &&
!srv_encrypt_tables &&
options->encryption_key_id != FIL_DEFAULT_ENCRYPTION_KEY) {
if (!encryption_key_id_exists((unsigned int)options->encryption_key_id)) {
@ -21290,10 +21293,11 @@ static MYSQL_SYSVAR_UINT(encryption_rotate_key_age,
PLUGIN_VAR_RQCMDARG,
"Key rotation - re-encrypt in background "
"all pages that were encrypted with a key that "
"many (or more) versions behind",
"many (or more) versions behind. Value 0 indicates "
"that key rotation is disabled.",
NULL,
innodb_encryption_rotate_key_age_update,
srv_fil_crypt_rotate_key_age, 0, UINT_MAX32, 0);
1, 0, UINT_MAX32, 0);
static MYSQL_SYSVAR_UINT(encryption_rotation_iops, srv_n_fil_crypt_iops,
PLUGIN_VAR_RQCMDARG,
@ -22254,8 +22258,9 @@ innodb_encrypt_tables_validate(
for update function */
struct st_mysql_value* value) /*!< in: incoming string */
{
if (check_sysvar_enum(thd, var, save, value))
if (check_sysvar_enum(thd, var, save, value)) {
return 1;
}
ulong encrypt_tables = *(ulong*)save;
@ -22267,6 +22272,17 @@ innodb_encrypt_tables_validate(
"encryption plugin is not available");
return 1;
}
if (!srv_fil_crypt_rotate_key_age) {
const char *msg = (encrypt_tables ? "enable" : "disable");
push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
HA_ERR_UNSUPPORTED,
"InnoDB: cannot %s encryption, "
"innodb_encryption_rotate_key_age=0"
" i.e. key rotation disabled", msg);
return 1;
}
return 0;
}

6
storage/xtradb/handler/handler0alter.cc
View file

@ -2891,9 +2891,11 @@ prepare_inplace_alter_table_dict(
ulint n_cols;
dtuple_t* add_cols;
ulint key_id = FIL_DEFAULT_ENCRYPTION_KEY;
fil_encryption_t mode = FIL_SPACE_ENCRYPTION_DEFAULT;
fil_encryption_t mode = FIL_ENCRYPTION_DEFAULT;
crypt_data = fil_space_get_crypt_data(ctx->prebuilt->table->space);
fil_space_t* space = fil_space_acquire(ctx->prebuilt->table->space);
crypt_data = space->crypt_data;
fil_space_release(space);
if (crypt_data) {
key_id = crypt_data->key_id;

105
storage/xtradb/handler/i_s.cc
View file

@ -1,7 +1,7 @@
/*****************************************************************************
Copyright (c) 2007, 2016, Oracle and/or its affiliates.
Copyrigth (c) 2014, 2016, MariaDB Corporation
Copyrigth (c) 2014, 2017, MariaDB Corporation
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -8492,22 +8492,31 @@ static ST_FIELD_INFO innodb_tablespaces_encryption_fields_info[] =
STRUCT_FLD(old_name, ""),
STRUCT_FLD(open_method, SKIP_OPEN_TABLE)},
#define TABLESPACES_ENCRYPTION_ROTATING_OR_FLUSHING 9
{STRUCT_FLD(field_name, "ROTATING_OR_FLUSHING"),
STRUCT_FLD(field_length, 1),
STRUCT_FLD(field_type, MYSQL_TYPE_LONG),
STRUCT_FLD(value, 0),
STRUCT_FLD(field_flags, MY_I_S_UNSIGNED),
STRUCT_FLD(old_name, ""),
STRUCT_FLD(open_method, SKIP_OPEN_TABLE)},
END_OF_ST_FIELD_INFO
};
/**********************************************************************//**
Function to fill INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION
with information collected by scanning SYS_TABLESPACES table and then use
fil_space()
with information collected by scanning SYS_TABLESPACES table.
@param[in] thd thread handle
@param[in] space Tablespace
@param[in] table_to_fill I_S table to fill
@return 0 on success */
static
int
i_s_dict_fill_tablespaces_encryption(
/*==========================*/
THD* thd, /*!< in: thread */
ulint space, /*!< in: space ID */
const char* name, /*!< in: tablespace name */
TABLE* table_to_fill) /*!< in/out: fill this table */
THD* thd,
fil_space_t* space,
TABLE* table_to_fill)
{
Field** fields;
struct fil_space_crypt_status_t status;
@ -8517,10 +8526,11 @@ i_s_dict_fill_tablespaces_encryption(
fields = table_to_fill->field;
fil_space_crypt_get_status(space, &status);
OK(fields[TABLESPACES_ENCRYPTION_SPACE]->store(space));
OK(fields[TABLESPACES_ENCRYPTION_SPACE]->store(space->id));
OK(field_store_string(fields[TABLESPACES_ENCRYPTION_NAME],
name));
space->name));
OK(fields[TABLESPACES_ENCRYPTION_ENCRYPTION_SCHEME]->store(
status.scheme));
@ -8532,6 +8542,9 @@ i_s_dict_fill_tablespaces_encryption(
status.current_key_version));
OK(fields[TABLESPACES_ENCRYPTION_CURRENT_KEY_ID]->store(
status.key_id));
OK(fields[TABLESPACES_ENCRYPTION_ROTATING_OR_FLUSHING]->store(
(status.rotating || status.flushing) ? 1 : 0));
if (status.rotating) {
fields[TABLESPACES_ENCRYPTION_KEY_ROTATION_PAGE_NUMBER]->set_notnull();
OK(fields[TABLESPACES_ENCRYPTION_KEY_ROTATION_PAGE_NUMBER]->store(
@ -8545,6 +8558,7 @@ i_s_dict_fill_tablespaces_encryption(
fields[TABLESPACES_ENCRYPTION_KEY_ROTATION_MAX_PAGE_NUMBER]
->set_null();
}
OK(schema_table_store_record(thd, table_to_fill));
DBUG_RETURN(0);
@ -8584,30 +8598,36 @@ i_s_tablespaces_encryption_fill_table(
while (rec) {
const char* err_msg;
ulint space;
ulint space_id;
const char* name;
ulint flags;
/* Extract necessary information from a SYS_TABLESPACES row */
err_msg = dict_process_sys_tablespaces(
heap, rec, &space, &name, &flags);
heap, rec, &space_id, &name, &flags);
mtr_commit(&mtr);
mutex_exit(&dict_sys->mutex);
if (space == 0) {
if (space_id == 0) {
found_space_0 = true;
}
if (!err_msg) {
fil_space_t* space = fil_space_acquire_silent(space_id);
if (!err_msg && space) {
i_s_dict_fill_tablespaces_encryption(
thd, space, name, tables->table);
thd, space, tables->table);
} else {
push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
ER_CANT_FIND_SYSTEM_REC, "%s",
err_msg);
}
if (space) {
fil_space_release(space);
}
mem_heap_empty(heap);
/* Get the next record */
@ -8623,10 +8643,13 @@ i_s_tablespaces_encryption_fill_table(
if (found_space_0 == false) {
/* space 0 does for what ever unknown reason not show up
* in iteration above, add it manually */
ulint space = 0;
const char* name = NULL;
fil_space_t* space = fil_space_acquire_silent(0);
i_s_dict_fill_tablespaces_encryption(
thd, space, name, tables->table);
thd, space, tables->table);
fil_space_release(space);
}
DBUG_RETURN(0);
@ -8780,17 +8803,18 @@ static ST_FIELD_INFO innodb_tablespaces_scrubbing_fields_info[] =
/**********************************************************************//**
Function to fill INFORMATION_SCHEMA.INNODB_TABLESPACES_SCRUBBING
with information collected by scanning SYS_TABLESPACES table and then use
fil_space()
with information collected by scanning SYS_TABLESPACES table and
fil_space.
@param[in] thd Thread handle
@param[in] space Tablespace
@param[in] table_to_fill I_S table
@return 0 on success */
static
int
i_s_dict_fill_tablespaces_scrubbing(
/*==========================*/
THD* thd, /*!< in: thread */
ulint space, /*!< in: space ID */
const char* name, /*!< in: tablespace name */
TABLE* table_to_fill) /*!< in/out: fill this table */
THD* thd,
fil_space_t* space,
TABLE* table_to_fill)
{
Field** fields;
struct fil_space_scrub_status_t status;
@ -8800,10 +8824,11 @@ i_s_dict_fill_tablespaces_scrubbing(
fields = table_to_fill->field;
fil_space_get_scrub_status(space, &status);
OK(fields[TABLESPACES_SCRUBBING_SPACE]->store(space));
OK(fields[TABLESPACES_SCRUBBING_SPACE]->store(space->id));
OK(field_store_string(fields[TABLESPACES_SCRUBBING_NAME],
name));
space->name));
OK(fields[TABLESPACES_SCRUBBING_COMPRESSED]->store(
status.compressed ? 1 : 0));
@ -8823,6 +8848,7 @@ i_s_dict_fill_tablespaces_scrubbing(
TABLESPACES_SCRUBBING_CURRENT_SCRUB_ACTIVE_THREADS,
TABLESPACES_SCRUBBING_CURRENT_SCRUB_PAGE_NUMBER,
TABLESPACES_SCRUBBING_CURRENT_SCRUB_MAX_PAGE_NUMBER };
if (status.scrubbing) {
for (uint i = 0; i < array_elements(field_numbers); i++) {
fields[field_numbers[i]]->set_notnull();
@ -8842,6 +8868,7 @@ i_s_dict_fill_tablespaces_scrubbing(
fields[field_numbers[i]]->set_null();
}
}
OK(schema_table_store_record(thd, table_to_fill));
DBUG_RETURN(0);
@ -8881,30 +8908,36 @@ i_s_tablespaces_scrubbing_fill_table(
while (rec) {
const char* err_msg;
ulint space;
ulint space_id;
const char* name;
ulint flags;
/* Extract necessary information from a SYS_TABLESPACES row */
err_msg = dict_process_sys_tablespaces(
heap, rec, &space, &name, &flags);
heap, rec, &space_id, &name, &flags);
mtr_commit(&mtr);
mutex_exit(&dict_sys->mutex);
if (space == 0) {
if (space_id == 0) {
found_space_0 = true;
}
if (!err_msg) {
fil_space_t* space = fil_space_acquire_silent(space_id);
if (!err_msg && space) {
i_s_dict_fill_tablespaces_scrubbing(
thd, space, name, tables->table);
thd, space, tables->table);
} else {
push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
ER_CANT_FIND_SYSTEM_REC, "%s",
err_msg);
}
if (space) {
fil_space_release(space);
}
mem_heap_empty(heap);
/* Get the next record */
@ -8920,10 +8953,12 @@ i_s_tablespaces_scrubbing_fill_table(
if (found_space_0 == false) {
/* space 0 does for what ever unknown reason not show up
* in iteration above, add it manually */
ulint space = 0;
const char* name = NULL;
fil_space_t* space = fil_space_acquire_silent(0);
i_s_dict_fill_tablespaces_scrubbing(
thd, space, name, tables->table);
thd, space, tables->table);
fil_space_release(space);
}
DBUG_RETURN(0);

49
storage/xtradb/ibuf/ibuf0ibuf.cc
View file

@ -1,7 +1,7 @@
/*****************************************************************************
Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2016, MariaDB Corporation.
Copyright (c) 2016, 2017, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -4596,7 +4596,7 @@ ibuf_merge_or_delete_for_page(
buf_block_t* block, /*!< in: if page has been read from
disk, pointer to the page x-latched,
else NULL */
ulint space, /*!< in: space id of the index page */
ulint space_id,/*!< in: space id of the index page */
ulint page_no,/*!< in: page number of the index page */
ulint zip_size,/*!< in: compressed page size in bytes,
or 0 */
@ -4613,21 +4613,21 @@ ibuf_merge_or_delete_for_page(
ulint volume = 0;
#endif
page_zip_des_t* page_zip = NULL;
ibool tablespace_being_deleted = FALSE;
ibool corruption_noticed = FALSE;
mtr_t mtr;
fil_space_t* space = NULL;
/* Counts for merged & discarded operations. */
ulint mops[IBUF_OP_COUNT];
ulint dops[IBUF_OP_COUNT];
ut_ad(!block || buf_block_get_space(block) == space);
ut_ad(!block || buf_block_get_space(block) == space_id);
ut_ad(!block || buf_block_get_page_no(block) == page_no);
ut_ad(!block || buf_block_get_zip_size(block) == zip_size);
ut_ad(!block || buf_block_get_io_fix_unlocked(block) == BUF_IO_READ);
if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE
|| trx_sys_hdr_page(space, page_no)) {
|| trx_sys_hdr_page(space_id, page_no)) {
return;
}
@ -4641,7 +4641,7 @@ ibuf_merge_or_delete_for_page(
uncompressed page size always is a power-of-2 multiple of the
compressed page size. */
if (ibuf_fixed_addr_page(space, 0, page_no)
if (ibuf_fixed_addr_page(space_id, 0, page_no)
|| fsp_descr_page(0, page_no)) {
return;
}
@ -4649,19 +4649,19 @@ ibuf_merge_or_delete_for_page(
if (UNIV_LIKELY(update_ibuf_bitmap)) {
ut_a(ut_is_2pow(zip_size));
if (ibuf_fixed_addr_page(space, zip_size, page_no)
if (ibuf_fixed_addr_page(space_id, zip_size, page_no)
|| fsp_descr_page(zip_size, page_no)) {
return;
}
/* If the following returns FALSE, we get the counter
/* If the following returns space, we get the counter
incremented, and must decrement it when we leave this
function. When the counter is > 0, that prevents tablespace
from being dropped. */
tablespace_being_deleted = fil_inc_pending_ops(space, true);
space = fil_space_acquire(space_id);
if (UNIV_UNLIKELY(tablespace_being_deleted)) {
if (UNIV_UNLIKELY(!space)) {
/* Do not try to read the bitmap page from space;
just delete the ibuf records for the page */
@ -4674,7 +4674,7 @@ ibuf_merge_or_delete_for_page(
ibuf_mtr_start(&mtr);
bitmap_page = ibuf_bitmap_get_map_page(
space, page_no, zip_size, &mtr);
space_id, page_no, zip_size, &mtr);
if (bitmap_page &&
fil_page_get_type(bitmap_page) != FIL_PAGE_TYPE_ALLOCATED) {
@ -4688,15 +4688,15 @@ ibuf_merge_or_delete_for_page(
if (!bitmap_bits) {
/* No inserts buffered for this page */
if (!tablespace_being_deleted) {
fil_decr_pending_ops(space);
if (space) {
fil_space_release(space);
}
return;
}
}
} else if (block
&& (ibuf_fixed_addr_page(space, zip_size, page_no)
&& (ibuf_fixed_addr_page(space_id, zip_size, page_no)
|| fsp_descr_page(zip_size, page_no))) {
return;
@ -4704,7 +4704,7 @@ ibuf_merge_or_delete_for_page(
heap = mem_heap_create(512);
search_tuple = ibuf_search_tuple_build(space, page_no, heap);
search_tuple = ibuf_search_tuple_build(space_id, page_no, heap);
if (block) {
/* Move the ownership of the x-latch on the page to this OS
@ -4730,7 +4730,7 @@ ibuf_merge_or_delete_for_page(
fputs(" InnoDB: Dump of the ibuf bitmap page:\n",
stderr);
bitmap_page = ibuf_bitmap_get_map_page(space, page_no,
bitmap_page = ibuf_bitmap_get_map_page(space_id, page_no,
zip_size, &mtr);
if (bitmap_page == NULL)
{
@ -4814,7 +4814,7 @@ loop:
/* Check if the entry is for this index page */
if (ibuf_rec_get_page_no(&mtr, rec) != page_no
|| ibuf_rec_get_space(&mtr, rec) != space) {
|| ibuf_rec_get_space(&mtr, rec) != space_id) {
if (block) {
page_header_reset_last_insert(
@ -4881,7 +4881,7 @@ loop:
ut_ad(page_rec_is_user_rec(rec));
ut_ad(ibuf_rec_get_page_no(&mtr, rec)
== page_no);
ut_ad(ibuf_rec_get_space(&mtr, rec) == space);
ut_ad(ibuf_rec_get_space(&mtr, rec) == space_id);
/* Mark the change buffer record processed,
so that it will not be merged again in case
@ -4911,7 +4911,7 @@ loop:
buf_block_dbg_add_level(
block, SYNC_IBUF_TREE_NODE);
if (!ibuf_restore_pos(space, page_no,
if (!ibuf_restore_pos(space_id, page_no,
search_tuple,
BTR_MODIFY_LEAF,
&pcur, &mtr)) {
@ -4935,7 +4935,7 @@ loop:
}
/* Delete the record from ibuf */
if (ibuf_delete_rec(space, page_no, &pcur, search_tuple,
if (ibuf_delete_rec(space_id, page_no, &pcur, search_tuple,
&mtr)) {
/* Deletion was pessimistic and mtr was committed:
we start from the beginning again */
@ -4955,7 +4955,7 @@ reset_bit:
page_t* bitmap_page;
bitmap_page = ibuf_bitmap_get_map_page(
space, page_no, zip_size, &mtr);
space_id, page_no, zip_size, &mtr);
ibuf_bitmap_page_set_bits(
bitmap_page, page_no, zip_size,
@ -4996,13 +4996,12 @@ reset_bit:
mutex_exit(&ibuf_mutex);
#endif /* HAVE_ATOMIC_BUILTINS */
if (update_ibuf_bitmap && !tablespace_being_deleted) {
fil_decr_pending_ops(space);
if (space) {
fil_space_release(space);
}
#ifdef UNIV_IBUF_COUNT_DEBUG
ut_a(ibuf_count_get(space, page_no) == 0);
ut_a(ibuf_count_get(space_id, page_no) == 0);
#endif
}

582
storage/xtradb/include/fil0crypt.h
View file

@ -39,14 +39,6 @@ static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = {
/* This key will be used if nothing else is given */
#define FIL_DEFAULT_ENCRYPTION_KEY ENCRYPTION_KEY_SYSTEM_DATA
/** Enum values for encryption table option */
typedef enum {
FIL_SPACE_ENCRYPTION_DEFAULT = 0, /* Tablespace encrypted if
srv_encrypt_tables = ON */
FIL_SPACE_ENCRYPTION_ON = 1, /* Tablespace is encrypted always */
FIL_SPACE_ENCRYPTION_OFF = 2 /* Tablespace is not encrypted */
} fil_encryption_t;
extern os_event_t fil_crypt_threads_event;
/**
@ -110,23 +102,21 @@ struct fil_space_rotate_state_t
} scrubbing;
};
struct fil_space_crypt_struct : st_encryption_scheme
struct fil_space_crypt_t : st_encryption_scheme
{
public:
/** Constructor. Does not initialize the members!
The object is expected to be placed in a buffer that
has been zero-initialized. */
fil_space_crypt_struct(
fil_space_crypt_t(
ulint new_type,
uint new_min_key_version,
uint new_key_id,
ulint offset,
fil_encryption_t new_encryption)
: st_encryption_scheme(),
min_key_version(new_min_key_version),
page0_offset(offset),
page0_offset(0),
encryption(new_encryption),
closing(false),
key_found(),
rotate_state()
{
@ -138,9 +128,9 @@ struct fil_space_crypt_struct : st_encryption_scheme
locker = crypt_data_scheme_locker;
type = new_type;
if (new_encryption == FIL_SPACE_ENCRYPTION_OFF ||
if (new_encryption == FIL_ENCRYPTION_OFF ||
(!srv_encrypt_tables &&
new_encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
new_encryption == FIL_ENCRYPTION_DEFAULT)) {
type = CRYPT_SCHEME_UNENCRYPTED;
} else {
type = CRYPT_SCHEME_1;
@ -149,9 +139,8 @@ struct fil_space_crypt_struct : st_encryption_scheme
}
/** Destructor */
~fil_space_crypt_struct()
~fil_space_crypt_t()
{
closing = true;
mutex_free(&mutex);
}
@ -169,45 +158,36 @@ struct fil_space_crypt_struct : st_encryption_scheme
/** Returns true if tablespace should be encrypted */
bool should_encrypt() const {
return ((encryption == FIL_SPACE_ENCRYPTION_ON) ||
return ((encryption == FIL_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
encryption == FIL_SPACE_ENCRYPTION_DEFAULT));
encryption == FIL_ENCRYPTION_DEFAULT));
}
/** Return true if tablespace is encrypted. */
bool is_encrypted() const {
return (encryption != FIL_SPACE_ENCRYPTION_OFF);
return (encryption != FIL_ENCRYPTION_OFF);
}
/** Return true if default tablespace encryption is used, */
bool is_default_encryption() const {
return (encryption == FIL_SPACE_ENCRYPTION_DEFAULT);
return (encryption == FIL_ENCRYPTION_DEFAULT);
}
/** Return true if tablespace is not encrypted. */
bool not_encrypted() const {
return (encryption == FIL_SPACE_ENCRYPTION_OFF);
return (encryption == FIL_ENCRYPTION_OFF);
}
/** Is this tablespace closing. */
bool is_closing(bool is_fixed) {
bool closed;
if (!is_fixed) {
mutex_enter(&mutex);
}
closed = closing;
if (!is_fixed) {
mutex_exit(&mutex);
}
return closed;
}
/** Write crypt data to a page (0)
@param[in,out] page0 Page 0 where to write
@param[in,out] mtr Minitransaction */
void write_page0(byte* page0, mtr_t* mtr);
uint min_key_version; // min key version for this space
ulint page0_offset; // byte offset on page 0 for crypt data
fil_encryption_t encryption; // Encryption setup
ib_mutex_t mutex; // mutex protecting following variables
bool closing; // is tablespace being closed
/** Return code from encryption_key_get_latest_version.
If ENCRYPTION_KEY_VERSION_INVALID encryption plugin
@ -219,238 +199,7 @@ struct fil_space_crypt_struct : st_encryption_scheme
fil_space_rotate_state_t rotate_state;
};
/* structure containing encryption specification */
typedef struct fil_space_crypt_struct fil_space_crypt_t;
/*********************************************************************
Init global resources needed for tablespace encryption/decryption */
UNIV_INTERN
void
fil_space_crypt_init();
/*********************************************************************
Cleanup global resources needed for tablespace encryption/decryption */
UNIV_INTERN
void
fil_space_crypt_cleanup();
/*********************************************************************
Create crypt data, i.e data that is used for a single tablespace */
UNIV_INTERN
fil_space_crypt_t *
fil_space_create_crypt_data(
/*========================*/
fil_encryption_t encrypt_mode, /*!< in: encryption mode */
uint key_id); /*!< in: encryption key id */
/*********************************************************************
Destroy crypt data */
UNIV_INTERN
void
fil_space_destroy_crypt_data(
/*=========================*/
fil_space_crypt_t **crypt_data); /*!< in/out: crypt data */
/*********************************************************************
Get crypt data for a space*/
UNIV_INTERN
fil_space_crypt_t *
fil_space_get_crypt_data(
/*=====================*/
ulint space); /*!< in: tablespace id */
/*********************************************************************
Set crypt data for a space*/
UNIV_INTERN
fil_space_crypt_t*
fil_space_set_crypt_data(
/*=====================*/
ulint space, /*!< in: tablespace id */
fil_space_crypt_t* crypt_data); /*!< in: crypt data to set */
/*********************************************************************
Merge crypt data */
UNIV_INTERN
void
fil_space_merge_crypt_data(
/*=======================*/
fil_space_crypt_t* dst_crypt_data, /*!< in: crypt_data */
const fil_space_crypt_t* src_crypt_data); /*!< in: crypt data */
/*********************************************************************
Read crypt data from buffer page */
UNIV_INTERN
fil_space_crypt_t *
fil_space_read_crypt_data(
/*======================*/
ulint space, /*!< in: tablespace id */
const byte* page, /*!< in: buffer page */
ulint offset); /*!< in: offset where crypt data is stored */
/*********************************************************************
Write crypt data to buffer page */
UNIV_INTERN
void
fil_space_write_crypt_data(
/*=======================*/
ulint space, /*!< in: tablespace id */
byte* page, /*!< in: buffer page */
ulint offset, /*!< in: offset where to store data */
ulint maxsize, /*!< in: max space available to store crypt data in */
mtr_t * mtr); /*!< in: mini-transaction */
/*********************************************************************
Clear crypt data from page 0 (used for import tablespace) */
UNIV_INTERN
void
fil_space_clear_crypt_data(
/*=======================*/
byte* page, /*!< in: buffer page */
ulint offset); /*!< in: offset where crypt data is stored */
/*********************************************************************
Parse crypt data log record */
UNIV_INTERN
byte*
fil_parse_write_crypt_data(
/*=======================*/
byte* ptr, /*!< in: start of log record */
byte* end_ptr, /*!< in: end of log record */
buf_block_t*); /*!< in: buffer page to apply record to */
/*********************************************************************
Check if extra buffer shall be allocated for decrypting after read */
UNIV_INTERN
bool
fil_space_check_encryption_read(
/*============================*/
ulint space); /*!< in: tablespace id */
/******************************************************************
Decrypt a page
@return true if page decrypted, false if not.*/
UNIV_INTERN
bool
fil_space_decrypt(
/*==============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
byte* tmp_frame, /*!< in: temporary buffer */
ulint page_size, /*!< in: page size */
byte* src_frame, /*!< in: out: page buffer */
dberr_t* err) /*!< in: out: DB_SUCCESS or
error code */
MY_ATTRIBUTE((warn_unused_result));
/*********************************************************************
Encrypt buffer page
@return encrypted page, or original not encrypted page if encrypt
is not needed. */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
ulint space, /*!< in: tablespace id */
ulint offset, /*!< in: page no */
lsn_t lsn, /*!< in: page lsn */
byte* src_frame, /*!< in: page frame */
ulint size, /*!< in: size of data to encrypt */
byte* dst_frame); /*!< in: where to encrypt to */
/******************************************************************
Decrypt a page
@param[in] space Tablespace id
@param[in] tmp_frame Temporary buffer used for decrypting
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] decrypted true if page was decrypted
@return decrypted page, or original not encrypted page if decryption is
not needed.*/
UNIV_INTERN
byte*
fil_space_decrypt(
/*==============*/
ulint space,
byte* src_frame,
ulint page_size,
byte* dst_frame,
bool* decrypted)
__attribute__((warn_unused_result));
/*********************************************************************
Verify that post encryption checksum match calculated checksum.
This function should be called only if tablespace contains crypt_data
metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
const fil_space_t* space,
ulint pageno)
__attribute__((warn_unused_result));
/*********************************************************************
Init threads for key rotation */
UNIV_INTERN
void
fil_crypt_threads_init();
/*********************************************************************
Set thread count (e.g start or stops threads) used for key rotation */
UNIV_INTERN
void
fil_crypt_set_thread_cnt(
/*=====================*/
uint new_cnt); /*!< in: requested #threads */
/*********************************************************************
Cleanup resources for threads for key rotation */
UNIV_INTERN
void
fil_crypt_threads_cleanup();
/*********************************************************************
Set rotate key age */
UNIV_INTERN
void
fil_crypt_set_rotate_key_age(
/*=========================*/
uint rotate_age); /*!< in: requested rotate age */
/*********************************************************************
Set rotation threads iops */
UNIV_INTERN
void
fil_crypt_set_rotation_iops(
/*========================*/
uint iops); /*!< in: requested iops */
/*********************************************************************
Mark a space as closing */
UNIV_INTERN
void
fil_space_crypt_mark_space_closing(
/*===============================*/
ulint space, /*!< in: tablespace id */
fil_space_crypt_t* crypt_data); /*!< in: crypt_data or NULL */
/*********************************************************************
Wait for crypt threads to stop accessing space */
UNIV_INTERN
void
fil_space_crypt_close_tablespace(
/*=============================*/
ulint space); /*!< in: tablespace id */
/** Struct for retreiving info about encryption */
/** Status info about encryption */
struct fil_space_crypt_status_t {
ulint space; /*!< tablespace id */
ulint scheme; /*!< encryption scheme */
@ -464,17 +213,7 @@ struct fil_space_crypt_status_t {
ulint rotate_max_page_number; /*!< max page if key rotating */
};
/*********************************************************************
Get crypt status for a space
@return 0 if crypt data found */
UNIV_INTERN
int
fil_space_crypt_get_status(
/*=======================*/
ulint id, /*!< in: space id */
struct fil_space_crypt_status_t * status); /*!< out: status */
/** Struct for retreiving statistics about encryption key rotation */
/** Statistics about encryption key rotation */
struct fil_crypt_stat_t {
ulint pages_read_from_cache;
ulint pages_read_from_disk;
@ -483,15 +222,7 @@ struct fil_crypt_stat_t {
ulint estimated_iops;
};
/*********************************************************************
Get crypt rotation statistics */
UNIV_INTERN
void
fil_crypt_total_stat(
/*==================*/
fil_crypt_stat_t* stat); /*!< out: crypt stat */
/** Struct for retreiving info about scrubbing */
/** Status info about scrubbing */
struct fil_space_scrub_status_t {
ulint space; /*!< tablespace id */
bool compressed; /*!< is space compressed */
@ -504,48 +235,271 @@ struct fil_space_scrub_status_t {
};
/*********************************************************************
Get scrub status for a space
@return 0 if no scrub info found */
UNIV_INTERN
int
fil_space_get_scrub_status(
/*=======================*/
ulint id, /*!< in: space id */
struct fil_space_scrub_status_t * status); /*!< out: status */
/*********************************************************************
Adjust encrypt tables */
Init space crypt */
UNIV_INTERN
void
fil_crypt_set_encrypt_tables(
/*=========================*/
uint val); /*!< in: New srv_encrypt_tables setting */
fil_space_crypt_init();
/*********************************************************************
Cleanup space crypt */
UNIV_INTERN
void
fil_space_crypt_cleanup();
/******************************************************************
Encrypt a buffer */
Create a fil_space_crypt_t object
@param[in] encrypt_mode FIL_ENCRYPTION_DEFAULT or
FIL_ENCRYPTION_ON or
FIL_ENCRYPTION_OFF
@param[in] key_id Encryption key id
@return crypt object */
UNIV_INTERN
fil_space_crypt_t*
fil_space_create_crypt_data(
fil_encryption_t encrypt_mode,
uint key_id)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Merge fil_space_crypt_t object
@param[in,out] dst Destination cryp data
@param[in] src Source crypt data */
UNIV_INTERN
void
fil_space_merge_crypt_data(
fil_space_crypt_t* dst,
const fil_space_crypt_t* src);
/******************************************************************
Read crypt data from a page (0)
@param[in] space space_id
@param[in] page Page 0
@param[in] offset Offset to crypt data
@return crypt data from page 0 or NULL. */
UNIV_INTERN
fil_space_crypt_t*
fil_space_read_crypt_data(
ulint space,
const byte* page,
ulint offset)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Free a crypt data object
@param[in,out] crypt_data crypt data to be freed */
UNIV_INTERN
void
fil_space_destroy_crypt_data(
fil_space_crypt_t **crypt_data);
/******************************************************************
Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry
@param[in] ptr Log entry start
@param[in] end_ptr Log entry end
@param[in] block buffer block
@return position on log buffer */
UNIV_INTERN
const byte*
fil_parse_write_crypt_data(
const byte* ptr,
const byte* end_ptr,
const buf_block_t* block)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Encrypt a buffer
@param[in,out] crypt_data Crypt data
@param[in] space space_id
@param[in] offset Page offset
@param[in] lsn Log sequence number
@param[in] src_frame Page to encrypt
@param[in] zip_size Compressed size or 0
@param[in,out] dst_frame Output buffer
@return encrypted buffer or NULL */
UNIV_INTERN
byte*
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame); /*!< in: outbut buffer */
fil_space_crypt_t* crypt_data,
ulint space,
ulint offset,
lsn_t lsn,
const byte* src_frame,
ulint zip_size,
byte* dst_frame)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Encrypt a page
@param[in] space Tablespace
@param[in] offset Page offset
@param[in] lsn Log sequence number
@param[in] src_frame Page to encrypt
@param[in,out] dst_frame Output buffer
@return encrypted buffer or NULL */
UNIV_INTERN
byte*
fil_space_encrypt(
const fil_space_t* space,
ulint offset,
lsn_t lsn,
byte* src_frame,
byte* dst_frame)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Decrypt a page
@param[in,out] crypt_data crypt_data
@param[in] tmp_frame Temporary buffer
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] err DB_SUCCESS or error
@return true if page decrypted, false if not.*/
UNIV_INTERN
bool
fil_space_decrypt(
fil_space_crypt_t* crypt_data,
byte* tmp_frame,
ulint page_size,
byte* src_frame,
dberr_t* err);
/******************************************************************
Decrypt a page
@param[in] space Tablespace
@param[in] tmp_frame Temporary buffer used for decrypting
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] decrypted true if page was decrypted
@return decrypted page, or original not encrypted page if decryption is
not needed.*/
UNIV_INTERN
byte*
fil_space_decrypt(
const fil_space_t* space,
byte* tmp_frame,
byte* src_frame,
bool* decrypted)
MY_ATTRIBUTE((warn_unused_result));
/******************************************************************
Calculate post encryption checksum
@param[in] zip_size zip_size or 0
@param[in] dst_frame Block where checksum is calculated
@return page checksum or BUF_NO_CHECKSUM_MAGIC
not needed. */
UNIV_INTERN
ulint
fil_crypt_calculate_checksum(
/*=========================*/
ulint zip_size, /*!< in: zip_size or 0 */
byte* dst_frame); /*!< in: page where to calculate */
ulint zip_size,
const byte* dst_frame)
MY_ATTRIBUTE((warn_unused_result));
/*********************************************************************
Verify that post encryption checksum match calculated checksum.
This function should be called only if tablespace contains crypt_data
metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
const fil_space_t* space,
ulint pageno)
MY_ATTRIBUTE((warn_unused_result));
/*********************************************************************
Adjust thread count for key rotation
@param[in] enw_cnt Number of threads to be used */
UNIV_INTERN
void
fil_crypt_set_thread_cnt(
uint new_cnt);
/*********************************************************************
Adjust max key age
@param[in] val New max key age */
UNIV_INTERN
void
fil_crypt_set_rotate_key_age(
uint val);
/*********************************************************************
Adjust rotation iops
@param[in] val New max roation iops */
UNIV_INTERN
void
fil_crypt_set_rotation_iops(
uint val);
/*********************************************************************
Adjust encrypt tables
@param[in] val New setting for innodb-encrypt-tables */
UNIV_INTERN
void
fil_crypt_set_encrypt_tables(
uint val);
/*********************************************************************
Init threads for key rotation */
UNIV_INTERN
void
fil_crypt_threads_init();
/*********************************************************************
Clean up key rotation threads resources */
UNIV_INTERN
void
fil_crypt_threads_cleanup();
/*********************************************************************
Wait for crypt threads to stop accessing space
@param[in] space Tablespace */
UNIV_INTERN
void
fil_space_crypt_close_tablespace(
const fil_space_t* space);
/*********************************************************************
Get crypt status for a space (used by information_schema)
@param[in] space Tablespace
@param[out] status Crypt status
return 0 if crypt data present */
UNIV_INTERN
void
fil_space_crypt_get_status(
const fil_space_t* space,
struct fil_space_crypt_status_t* status);
/*********************************************************************
Return crypt statistics
@param[out] stat Crypt statistics */
UNIV_INTERN
void
fil_crypt_total_stat(
fil_crypt_stat_t *stat);
/*********************************************************************
Get scrub status for a space (used by information_schema)
@param[in] space Tablespace
@param[out] status Scrub status
return 0 if data found */
UNIV_INTERN
void
fil_space_get_scrub_status(
const fil_space_t* space,
struct fil_space_scrub_status_t* status);
#ifndef UNIV_NONINL
#include "fil0crypt.ic"

32
storage/xtradb/include/fil0crypt.ic
View file

@ -34,35 +34,3 @@ fil_page_is_encrypted(
{
return(mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0);
}
/*******************************************************************//**
Find out whether the page can be decrypted.
The function for decrypting the page should already be executed before this.
@return 1 if key provider not available or key is not available
0 if decryption should be possible
*/
UNIV_INLINE
bool
fil_page_encryption_status(
/*===================*/
const byte *buf, /*!< in: page */
ulint space_id) /*!< in: space_id */
{
fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space_id);
ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
if (page_type == FIL_PAGE_TYPE_FSP_HDR) {
if (crypt_data != NULL) {
if (!encryption_key_id_exists(crypt_data->key_id)) {
/* accessing table would surely fail, because no key or no key provider available */
return 1;
}
}
} else {
ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
if (!encryption_key_version_exists(crypt_data->key_id, key)) {
return 1;
}
}
return 0;
}

120
storage/xtradb/include/fil0fil.h
View file

@ -181,8 +181,18 @@ extern fil_addr_t fil_addr_null;
#define FIL_LOG 502 /*!< redo log */
/* @} */
/* structure containing encryption specification */
typedef struct fil_space_crypt_struct fil_space_crypt_t;
/** Structure containing encryption specification */
struct fil_space_crypt_t;
/** Enum values for encryption table option */
enum fil_encryption_t {
/** Encrypted if innodb_encrypt_tables=ON (srv_encrypt_tables) */
FIL_ENCRYPTION_DEFAULT,
/** Encrypted */
FIL_ENCRYPTION_ON,
/** Not encrypted */
FIL_ENCRYPTION_OFF
};
/** The number of fsyncs done to the log */
extern ulint fil_n_log_flushes;
@ -269,8 +279,8 @@ struct fil_space_t {
.ibd file of tablespace and want to
stop temporarily posting of new i/o
requests on the file */
ibool stop_new_ops;
/*!< we set this TRUE when we start
bool stop_new_ops;
/*!< we set this true when we start
deleting a single-table tablespace.
When this is set following new ops
are not allowed:
@ -316,13 +326,16 @@ struct fil_space_t {
prio_rw_lock_t latch; /*!< latch protecting the file space storage
allocation */
#endif /* !UNIV_HOTBACKUP */
UT_LIST_NODE_T(fil_space_t) unflushed_spaces;
/*!< list of spaces with at least one unflushed
file we have written to */
bool is_in_unflushed_spaces;
/*!< true if this space is currently in
unflushed_spaces */
ibool is_corrupt;
/** True if srv_pass_corrupt_table=true and tablespace contains
corrupted page. */
bool is_corrupt;
/*!< true if tablespace corrupted */
bool printed_compression_failure;
/*!< true if we have already printed
@ -338,7 +351,22 @@ struct fil_space_t {
UT_LIST_NODE_T(fil_space_t) space_list;
/*!< list of all spaces */
/*!< Protected by fil_system */
UT_LIST_NODE_T(fil_space_t) rotation_list;
/*!< list of spaces needing
key rotation */
bool is_in_rotation_list;
/*!< true if this space is
currently in key rotation list */
ulint magic_n;/*!< FIL_SPACE_MAGIC_N */
/** @return whether the tablespace is about to be dropped or truncated */
bool is_stopping() const
{
return stop_new_ops;
}
};
/** Value of fil_space_t::magic_n */
@ -394,6 +422,11 @@ struct fil_system_t {
request */
UT_LIST_BASE_NODE_T(fil_space_t) space_list;
/*!< list of all file spaces */
UT_LIST_BASE_NODE_T(fil_space_t) rotation_list;
/*!< list of all file spaces needing
key rotation.*/
ibool space_id_reuse_warned;
/* !< TRUE if fil_space_create()
has issued a warning about
@ -472,18 +505,24 @@ fil_space_contains_node(
/*******************************************************************//**
Creates a space memory object and puts it to the 'fil system' hash table.
If there is an error, prints an error message to the .err log.
@param[in] name Space name
@param[in] id Space id
@param[in] flags Tablespace flags
@param[in] purpose FIL_TABLESPACE or FIL_LOG if log
@param[in] crypt_data Encryption information
@param[in] create_table True if this is create table
@param[in] mode Encryption mode
@return TRUE if success */
UNIV_INTERN
ibool
bool
fil_space_create(
/*=============*/
const char* name, /*!< in: space name */
ulint id, /*!< in: space id */
ulint zip_size,/*!< in: compressed page size, or
0 for uncompressed tablespaces */
ulint purpose, /*!< in: FIL_TABLESPACE, or FIL_LOG if log */
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
bool create_table); /*!< in: true if create table */
const char* name,
ulint id,
ulint flags,
ulint purpose,
fil_space_crypt_t* crypt_data,
bool create_table,
fil_encryption_t mode = FIL_ENCRYPTION_DEFAULT);
/*******************************************************************//**
Assigns a new space id for a new single-table tablespace. This works simply by
@ -606,6 +645,59 @@ fil_write_flushed_lsn_to_data_files(
/*================================*/
lsn_t lsn, /*!< in: lsn to write */
ulint arch_log_no); /*!< in: latest archived log file number */
/** Acquire a tablespace when it could be dropped concurrently.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@return the tablespace, or NULL if missing or being deleted */
fil_space_t*
fil_space_acquire(
ulint id)
MY_ATTRIBUTE((warn_unused_result));
/** Acquire a tablespace that may not exist.
Used by background threads that do not necessarily hold proper locks
for concurrency control.
@param[in] id tablespace ID
@return the tablespace, or NULL if missing or being deleted */
fil_space_t*
fil_space_acquire_silent(
ulint id)
MY_ATTRIBUTE((warn_unused_result));
/** Release a tablespace acquired with fil_space_acquire().
@param[in,out] space tablespace to release */
void
fil_space_release(
fil_space_t* space);
/** Return the next fil_space_t.
Once started, the caller must keep calling this until it returns NULL.
fil_space_acquire() and fil_space_release() are invoked here which
blocks a concurrent operation from dropping the tablespace.
@param[in,out] prev_space Pointer to the previous fil_space_t.
If NULL, use the first fil_space_t on fil_system->space_list.
@return pointer to the next fil_space_t.
@retval NULL if this was the last */
fil_space_t*
fil_space_next(
fil_space_t* prev_space)
MY_ATTRIBUTE((warn_unused_result));
/** Return the next fil_space_t from key rotation list.
Once started, the caller must keep calling this until it returns NULL.
fil_space_acquire() and fil_space_release() are invoked here which
blocks a concurrent operation from dropping the tablespace.
@param[in,out] prev_space Pointer to the previous fil_space_t.
If NULL, use the first fil_space_t on fil_system->space_list.
@return pointer to the next fil_space_t.
@retval NULL if this was the last*/
fil_space_t*
fil_space_keyrotate_next(
fil_space_t* prev_space)
MY_ATTRIBUTE((warn_unused_result));
/*******************************************************************//**
Reads the flushed lsn, arch no, and tablespace flag fields from a data
file at database startup.

11
storage/xtradb/include/fsp0fsp.h
View file

@ -1037,14 +1037,15 @@ fsp_flags_get_page_size(
/*====================*/
ulint flags); /*!< in: tablespace flags */
/*********************************************************************/
/* @return offset into fsp header where crypt data is stored */
/*********************************************************************
Compute offset after xdes where crypt data can be stored
@param[in] zip_size Compressed size or 0
@return offset */
UNIV_INTERN
ulint
fsp_header_get_crypt_offset(
/*========================*/
ulint zip_size, /*!< in: zip_size */
ulint* max_size); /*!< out: free space after offset */
const ulint zip_size)
MY_ATTRIBUTE((warn_unused_result));
#define fsp_page_is_free(space,page,mtr) \
fsp_page_is_free_func(space,page,mtr, __FILE__, __LINE__)

4
storage/xtradb/include/srv0srv.h
View file

@ -192,6 +192,9 @@ struct srv_stats_t {
/** Number of encryption_get_latest_key_version calls */
ulint_ctr_64_t n_key_requests;
/** Number of spaces in keyrotation list */
ulint_ctr_64_t key_rotation_list_length;
};
extern const char* srv_main_thread_op_info;
@ -1265,6 +1268,7 @@ struct export_var_t{
ulint innodb_encryption_rotation_pages_flushed;
ulint innodb_encryption_rotation_estimated_iops;
ib_int64_t innodb_encryption_key_requests;
ib_int64_t innodb_key_rotation_list_length;
ulint innodb_scrub_page_reorganizations;
ulint innodb_scrub_page_splits;

42
storage/xtradb/lock/lock0lock.cc
View file

@ -1,7 +1,7 @@
/*****************************************************************************
Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2014, 2015, MariaDB Corporation
Copyright (c) 2014, 2017, MariaDB Corporation
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -6450,12 +6450,13 @@ loop:
if (lock_get_type_low(lock) == LOCK_REC) {
if (load_page_first) {
ulint space = lock->un_member.rec_lock.space;
ulint zip_size= fil_space_get_zip_size(space);
ulint space_id = lock->un_member.rec_lock.space;
/* Check if the space is exists or not. only
when the space is valid, try to get the page. */
fil_space_t* space = fil_space_acquire(space_id);
ulint page_no = lock->un_member.rec_lock.page_no;
ibool tablespace_being_deleted = FALSE;
if (UNIV_UNLIKELY(zip_size == ULINT_UNDEFINED)) {
if (!space) {
/* It is a single table tablespace and
the .ibd file is missing (TRUNCATE
@ -6464,11 +6465,13 @@ loop:
load the page in the buffer pool. */
fprintf(file, "RECORD LOCKS on"
" non-existing space %lu\n",
(ulong) space);
" non-existing space: " ULINTPF "\n",
space_id);
goto print_rec;
}
const ulint zip_size = fsp_flags_get_zip_size(space->flags);
lock_mutex_exit();
mutex_exit(&trx_sys->mutex);
@ -6476,15 +6479,10 @@ loop:
DEBUG_SYNC_C("innodb_monitor_before_lock_page_read");
/* Check if the space is exists or not. only
when the space is valid, try to get the page. */
tablespace_being_deleted
= fil_inc_pending_ops(space, false);
if (!tablespace_being_deleted) {
if (space) {
mtr_start(&mtr);
buf_page_get_gen(space, zip_size,
buf_page_get_gen(space_id, zip_size,
page_no, RW_NO_LATCH,
NULL,
BUF_GET_POSSIBLY_FREED,
@ -6493,14 +6491,11 @@ loop:
mtr_commit(&mtr);
fil_decr_pending_ops(space);
} else {
fprintf(file, "RECORD LOCKS on"
" non-existing space %lu\n",
(ulong) space);
}
}
fil_space_release(space);
load_page_first = FALSE;
lock_mutex_enter();
@ -6929,7 +6924,7 @@ static
void
lock_rec_block_validate(
/*====================*/
ulint space,
ulint space_id,
ulint page_no)
{
/* The lock and the block that it is referring to may be freed at
@ -6942,10 +6937,11 @@ lock_rec_block_validate(
/* Make sure that the tablespace is not deleted while we are
trying to access the page. */
if (!fil_inc_pending_ops(space, true)) {
if (fil_space_t* space = fil_space_acquire(space_id)) {
mtr_start(&mtr);
block = buf_page_get_gen(
space, fil_space_get_zip_size(space),
space_id, fsp_flags_get_zip_size(space->flags),
page_no, RW_X_LATCH, NULL,
BUF_GET_POSSIBLY_FREED,
__FILE__, __LINE__, &mtr);
@ -6955,7 +6951,7 @@ lock_rec_block_validate(
ut_ad(lock_rec_validate_page(block));
mtr_commit(&mtr);
fil_decr_pending_ops(space);
fil_space_release(space);
}
}

2
storage/xtradb/log/log0recv.cc
View file

@ -1383,7 +1383,7 @@ recv_parse_or_apply_log_rec_body(
}
break;
case MLOG_FILE_WRITE_CRYPT_DATA:
ptr = fil_parse_write_crypt_data(ptr, end_ptr, block);
ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block));
break;
default:
ptr = NULL;

2
storage/xtradb/pars/pars0pars.cc
View file

@ -2020,7 +2020,7 @@ pars_create_table(
}
node = tab_create_graph_create(table, pars_sym_tab_global->heap, true,
FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
FIL_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
table_sym->resolved = TRUE;
table_sym->token_type = SYM_TABLE;

11
storage/xtradb/row/row0ftsort.cc
View file

@ -1,7 +1,7 @@
/*****************************************************************************
Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, 2016, MariaDB Corporation.
Copyright (c) 2015, 2017, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -224,7 +224,14 @@ row_fts_psort_info_init(
common_info->sort_event = os_event_create();
common_info->merge_event = os_event_create();
common_info->opt_doc_id_size = opt_doc_id_size;
crypt_data = fil_space_get_crypt_data(new_table->space);
/* Theoretically the tablespace can be dropped straight away.
In practice, the DDL completion will wait for this thread to
finish. */
if (fil_space_t* space = fil_space_acquire(new_table->space)) {
crypt_data = space->crypt_data;
fil_space_release(space);
}
if (crypt_data && crypt_data->should_encrypt()) {
common_info->crypt_data = crypt_data;

18
storage/xtradb/row/row0merge.cc
View file

@ -3951,7 +3951,7 @@ row_merge_build_indexes(
{
merge_file_t* merge_files;
row_merge_block_t* block;
row_merge_block_t* crypt_block;
row_merge_block_t* crypt_block = NULL;
ulint block_size;
ulint i;
ulint j;
@ -3987,9 +3987,15 @@ row_merge_build_indexes(
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
/* Get crypt data from tablespace if present. */
crypt_data = fil_space_get_crypt_data(new_table->space);
crypt_block = NULL;
/* Get crypt data from tablespace if present. We should be protected
from concurrent DDL (e.g. drop table) by MDL-locks. */
fil_space_t* space = fil_space_acquire(new_table->space);
if (space) {
crypt_data = space->crypt_data;
} else {
DBUG_RETURN(DB_TABLESPACE_NOT_FOUND);
}
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
@ -4353,5 +4359,9 @@ func_exit:
}
}
if (space) {
fil_space_release(space);
}
DBUG_RETURN(error);
}

59
storage/xtradb/row/row0mysql.cc
View file

@ -3310,21 +3310,17 @@ void
fil_wait_crypt_bg_threads(
dict_table_t* table)
{
uint start = time(0);
uint last = start;
if (table->space != 0) {
fil_space_crypt_mark_space_closing(table->space, table->crypt_data);
}
time_t start = time(0);
time_t last = start;
while (table->n_ref_count > 0) {
dict_mutex_exit_for_mysql();
os_thread_sleep(20000);
dict_mutex_enter_for_mysql();
uint now = time(0);
time_t now = time(0);
if (now >= last + 30) {
fprintf(stderr,
"WARNING: waited %u seconds "
"WARNING: waited %ld seconds "
"for ref-count on table: %s space: %u\n",
now - start, table->name, table->space);
last = now;
@ -3332,7 +3328,7 @@ fil_wait_crypt_bg_threads(
if (now >= start + 300) {
fprintf(stderr,
"WARNING: after %u seconds, gave up waiting "
"WARNING: after %ld seconds, gave up waiting "
"for ref-count on table: %s space: %u\n",
now - start, table->name, table->space);
break;
@ -3528,35 +3524,40 @@ row_truncate_table_for_mysql(
if (table->space && !DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) {
/* Discard and create the single-table tablespace. */
fil_space_crypt_t* crypt_data;
ulint space = table->space;
ulint flags = fil_space_get_flags(space);
ulint space_id = table->space;
ulint flags = ULINT_UNDEFINED;
ulint key_id = FIL_DEFAULT_ENCRYPTION_KEY;
fil_encryption_t mode = FIL_SPACE_ENCRYPTION_DEFAULT;
fil_encryption_t mode = FIL_ENCRYPTION_DEFAULT;
dict_get_and_save_data_dir_path(table, true);
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data) {
key_id = crypt_data->key_id;
mode = crypt_data->encryption;
if (fil_space_t* space = fil_space_acquire(space_id)) {
fil_space_crypt_t* crypt_data = space->crypt_data;
if (crypt_data) {
key_id = crypt_data->key_id;
mode = crypt_data->encryption;
}
flags = space->flags;
fil_space_release(space);
}
if (flags != ULINT_UNDEFINED
&& fil_discard_tablespace(space) == DB_SUCCESS) {
&& fil_discard_tablespace(space_id) == DB_SUCCESS) {
dict_index_t* index;
dict_hdr_get_new_id(NULL, NULL, &space);
dict_hdr_get_new_id(NULL, NULL, &space_id);
/* Lock all index trees for this table. We must
do so after dict_hdr_get_new_id() to preserve
the latch order */
dict_table_x_lock_indexes(table);
if (space == ULINT_UNDEFINED
if (space_id == ULINT_UNDEFINED
|| fil_create_new_single_table_tablespace(
space, table->name,
space_id, table->name,
table->data_dir_path,
flags, table->flags2,
FIL_IBD_FILE_INITIAL_SIZE,
@ -3574,21 +3575,21 @@ row_truncate_table_for_mysql(
goto funct_exit;
}
recreate_space = space;
recreate_space = space_id;
/* Replace the space_id in the data dictionary cache.
The persisent data dictionary (SYS_TABLES.SPACE
and SYS_INDEXES.SPACE) are updated later in this
function. */
table->space = space;
table->space = space_id;
index = dict_table_get_first_index(table);
do {
index->space = space;
index->space = space_id;
index = dict_table_get_next_index(index);
} while (index);
mtr_start_trx(&mtr, trx);
fsp_header_init(space,
fsp_header_init(space_id,
FIL_IBD_FILE_INITIAL_SIZE, &mtr);
mtr_commit(&mtr);
}
@ -4262,7 +4263,13 @@ row_drop_table_for_mysql(
/* If table has not yet have crypt_data, try to read it to
make freeing the table easier. */
if (!table->crypt_data) {
table->crypt_data = fil_space_get_crypt_data(table->space);
if (fil_space_t* space = fil_space_acquire_silent(table->space)) {
/* We use crypt data in dict_table_t in ha_innodb.cc
to push warnings to user thread. */
table->crypt_data = space->crypt_data;
fil_space_release(space);
}
}
/* We use the private SQL parser of Innobase to generate the

2
storage/xtradb/srv/srv0srv.cc
View file

@ -2117,6 +2117,8 @@ srv_export_innodb_status(void)
crypt_stat.estimated_iops;
export_vars.innodb_encryption_key_requests =
srv_stats.n_key_requests;
export_vars.innodb_key_rotation_list_length =
srv_stats.key_rotation_list_length;
export_vars.innodb_scrub_page_reorganizations =
scrub_stat.page_reorganizations;

6
storage/xtradb/srv/srv0start.cc
View file

@ -695,6 +695,7 @@ create_log_files(
FIL_LOG,
NULL /* no encryption yet */,
true /* this is create */);
ut_a(fil_validate());
logfile0 = fil_node_create(
@ -1187,13 +1188,14 @@ check_first_page:
if (i == 0) {
if (!crypt_data) {
crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
crypt_data = fil_space_create_crypt_data(FIL_ENCRYPTION_DEFAULT,
FIL_DEFAULT_ENCRYPTION_KEY);
}
flags = FSP_FLAGS_PAGE_SSIZE();
fil_space_create(name, 0, flags, FIL_TABLESPACE,
crypt_data, (*create_new_db) == true);
crypt_data, (*create_new_db) == true);
}
ut_a(fil_validate());