From 04f92dde67a3d4f6fcbdbf65fdb63a54ff6761d8 Mon Sep 17 00:00:00 2001 From: Hartmut Holzgraefe Date: Wed, 21 Sep 2016 10:51:37 +0200 Subject: [PATCH 001/203] MDEV-10853 netcat help output in error log when running xtrabackup SST --- scripts/wsrep_sst_xtrabackup-v2.sh | 2 +- scripts/wsrep_sst_xtrabackup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 962c170dc1b..327c92fb6cf 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -175,7 +175,7 @@ get_transfer() fi wsrep_log_info "Using netcat as streamer" if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - if nc -h | grep -q ncat;then + if nc -h 2>&1 | grep -q ncat;then tcmd="nc -l ${TSST_PORT}" else tcmd="nc -dl ${TSST_PORT}" diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh index 7279c623288..7ba421840ab 100644 --- a/scripts/wsrep_sst_xtrabackup.sh +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -149,7 +149,7 @@ get_transfer() fi wsrep_log_info "Using netcat as streamer" if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - if nc -h | grep -q ncat;then + if nc -h 2>&1 | grep -q ncat;then tcmd="nc -l ${TSST_PORT}" else tcmd="nc -dl ${TSST_PORT}" From fc1798785f4daf878e2c9f4c28ed35ca1af6dcba Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 17 Oct 2016 12:10:12 -0400 Subject: [PATCH 002/203] Adjust test to adapt to a recent change in mysqltest. --- mysql-test/suite/galera/t/galera_concurrent_ctas.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_concurrent_ctas.test b/mysql-test/suite/galera/t/galera_concurrent_ctas.test index f0dcf8e4900..8b9b461d0fd 100644 --- a/mysql-test/suite/galera/t/galera_concurrent_ctas.test +++ b/mysql-test/suite/galera/t/galera_concurrent_ctas.test @@ -43,10 +43,10 @@ let $run=10; while($run) { --error 0,1 - exec $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_1 test - < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql & - $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_2 test - < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql; + exec $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_1 test \ + < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql & \ + $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_2 test \ + < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql; dec $run; } From 1bba40f0dfae5b2c634a50a5de8a44ca63c55510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 9 Nov 2016 08:49:33 +0200 Subject: [PATCH 003/203] MDEV-10544: Galera: Failing assertion: (lock->trx)->wait_lock == lock In Galera case this assertion is too strict as conflicting lock could be same as lock here. --- storage/innobase/lock/lock0lock.c | 1 - storage/xtradb/lock/lock0lock.c | 1 - 2 files changed, 2 deletions(-) diff --git a/storage/innobase/lock/lock0lock.c b/storage/innobase/lock/lock0lock.c index cf77a9ebb8c..3890a2c44ff 100644 --- a/storage/innobase/lock/lock0lock.c +++ b/storage/innobase/lock/lock0lock.c @@ -798,7 +798,6 @@ lock_reset_lock_and_trx_wait( /* Reset the back pointer in trx to this waiting lock request */ if (!(lock->type_mode & LOCK_CONV_BY_OTHER)) { - ut_ad((lock->trx)->wait_lock == lock); (lock->trx)->wait_lock = NULL; } else { ut_ad(lock_get_type_low(lock) == LOCK_REC); diff --git a/storage/xtradb/lock/lock0lock.c b/storage/xtradb/lock/lock0lock.c index c5f8d6770df..b6cee714dec 100644 --- a/storage/xtradb/lock/lock0lock.c +++ b/storage/xtradb/lock/lock0lock.c @@ -799,7 +799,6 @@ lock_reset_lock_and_trx_wait( /* Reset the back pointer in trx to this waiting lock request */ if (!(lock->type_mode & LOCK_CONV_BY_OTHER)) { - ut_ad((lock->trx)->wait_lock == lock); (lock->trx)->wait_lock = NULL; } else { ut_ad(lock_get_type_low(lock) == LOCK_REC); From 72fd15f7c31aa3e3705ae1b005a3247a985c5bb3 Mon Sep 17 00:00:00 2001 From: SachinSetiya Date: Thu, 1 Dec 2016 12:16:13 +0530 Subject: [PATCH 004/203] MDEV-11016 wsrep_node_is_ready() check is too strict Problem:- The condition that checks for node readiness is too strict as it does not allow SELECTs even if these selects do not access any tables. For example,if we run SELECT 1; OR SELECT @@max_allowed_packet; Solution:- We need not to report this error when all_tables(lex->query_tables) is NULL: --- .../suite/galera/r/galera_var_dirty_reads.result | 13 +++++++++++++ .../suite/galera/t/galera_var_dirty_reads.test | 7 +++++++ sql/sql_parse.cc | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 6d703c8cf95..6a2aa1eb5e7 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,6 +18,19 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 +SET @@session.wsrep_dirty_reads=OFF; +SELECT 2; +2 +2 +SELECT @@max_allowed_packet; +@@max_allowed_packet +4194304 +SELECT 2+2 from DUAL; +2+2 +4 +SELECT sysdate() from DUAL; +sysdate() +2016-10-28 23:13:06 SELECT * FROM t1; i 1 diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index 9eea8efdaf3..dd7bc88cb1c 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -39,6 +39,13 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; +#Select query which does not access table should be allowed MDEV-11016 +SET @@session.wsrep_dirty_reads=OFF; +SELECT 2; +SELECT @@max_allowed_packet; +SELECT 2+2 from DUAL; +SELECT sysdate() from DUAL; + --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' --enable_query_log diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 48addaab143..eb4f7141918 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2388,11 +2388,15 @@ mysql_execute_command(THD *thd) /* Bail out if DB snapshot has not been installed. We however, allow SET and SHOW queries. + SHOW and SELECT queries (only if wsrep_dirty_reads is set or when it + does not access any table). */ if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && lex->sql_command != SQLCOM_SET_OPTION && !(thd->variables.wsrep_dirty_reads && lex->sql_command == SQLCOM_SELECT) && + !(lex->sql_command == SQLCOM_SELECT && + !all_tables) && !wsrep_is_show_query(lex->sql_command)) { #if DIRTY_HACK From 313a14f79e6de43acd5c2ceffbcc01af2506d800 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Fri, 9 Dec 2016 12:24:08 -0500 Subject: [PATCH 005/203] Fix test failures. --- .../suite/galera/r/galera_var_dirty_reads.result | 15 +++++++++------ mysql-test/suite/galera/suite.pm | 1 + .../suite/galera/t/galera_var_dirty_reads.test | 12 +++++++++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 6a2aa1eb5e7..c0ffb2d4860 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,19 +18,22 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 +# +# MDEV-11016: wsrep_node_is_ready() check is too strict +# SET @@session.wsrep_dirty_reads=OFF; SELECT 2; 2 2 -SELECT @@max_allowed_packet; -@@max_allowed_packet -4194304 SELECT 2+2 from DUAL; 2+2 4 -SELECT sysdate() from DUAL; -sysdate() -2016-10-28 23:13:06 +SET @VAR=1; +SELECT @VAR; +@VAR +1 +SELECT @@max_allowed_packet; +SELECT SYSDATE() from DUAL; SELECT * FROM t1; i 1 diff --git a/mysql-test/suite/galera/suite.pm b/mysql-test/suite/galera/suite.pm index fdaf64580c7..a790e951762 100644 --- a/mysql-test/suite/galera/suite.pm +++ b/mysql-test/suite/galera/suite.pm @@ -37,6 +37,7 @@ push @::global_suppressions, qr|WSREP: access file\(.*gvwstate.dat\) failed\(No such file or directory\)|, qr|WSREP: no nodes coming from prim view, prim not possible|, qr(WSREP: SYNC message from member .* in non-primary configuration. Ignored.), + qr(WSREP: discarding established .*), ); diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index dd7bc88cb1c..48fbe37139c 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -39,12 +39,18 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; -#Select query which does not access table should be allowed MDEV-11016 +--echo # +--echo # MDEV-11016: wsrep_node_is_ready() check is too strict +--echo # SET @@session.wsrep_dirty_reads=OFF; SELECT 2; -SELECT @@max_allowed_packet; SELECT 2+2 from DUAL; -SELECT sysdate() from DUAL; +SET @VAR=1; +SELECT @VAR; +--disable_result_log +SELECT @@max_allowed_packet; +SELECT SYSDATE() from DUAL; +--enable_result_log --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' From 95422c445db7db60fca688711d98955e59e009e2 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 14 Dec 2016 15:58:14 +0530 Subject: [PATCH 006/203] Revert " MDEV-11016 wsrep_node_is_ready() check is too strict" This reverts commit 72fd15f7c31aa3e3705ae1b005a3247a985c5bb3. # Conflicts: # mysql-test/suite/galera/r/galera_var_dirty_reads.result # mysql-test/suite/galera/t/galera_var_dirty_reads.test --- .../suite/galera/r/galera_var_dirty_reads.result | 16 ---------------- .../suite/galera/t/galera_var_dirty_reads.test | 13 ------------- sql/sql_parse.cc | 4 ---- 3 files changed, 33 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index c0ffb2d4860..6d703c8cf95 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,22 +18,6 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 -# -# MDEV-11016: wsrep_node_is_ready() check is too strict -# -SET @@session.wsrep_dirty_reads=OFF; -SELECT 2; -2 -2 -SELECT 2+2 from DUAL; -2+2 -4 -SET @VAR=1; -SELECT @VAR; -@VAR -1 -SELECT @@max_allowed_packet; -SELECT SYSDATE() from DUAL; SELECT * FROM t1; i 1 diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index 48fbe37139c..9eea8efdaf3 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -39,19 +39,6 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; ---echo # ---echo # MDEV-11016: wsrep_node_is_ready() check is too strict ---echo # -SET @@session.wsrep_dirty_reads=OFF; -SELECT 2; -SELECT 2+2 from DUAL; -SET @VAR=1; -SELECT @VAR; ---disable_result_log -SELECT @@max_allowed_packet; -SELECT SYSDATE() from DUAL; ---enable_result_log - --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' --enable_query_log diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index eb4f7141918..48addaab143 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2388,15 +2388,11 @@ mysql_execute_command(THD *thd) /* Bail out if DB snapshot has not been installed. We however, allow SET and SHOW queries. - SHOW and SELECT queries (only if wsrep_dirty_reads is set or when it - does not access any table). */ if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && lex->sql_command != SQLCOM_SET_OPTION && !(thd->variables.wsrep_dirty_reads && lex->sql_command == SQLCOM_SELECT) && - !(lex->sql_command == SQLCOM_SELECT && - !all_tables) && !wsrep_is_show_query(lex->sql_command)) { #if DIRTY_HACK From 5ddd8149e44122feb0989b9944113d2b92d6f84a Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 14 Dec 2016 17:14:42 +0530 Subject: [PATCH 007/203] MDEV-11479 Improved wsrep_dirty_reads Tasks:- Changes in wsrep_dirty_reads variable 1.) Global + Session scope (Current: session-only) 2.) Can be set using command line. 3.) Allow all commands that do not change data (besides SELECT) 4.) Allow prepared Statements that do not change data 5.) Works with wsrep_sync_wait enabled --- mysql-test/r/mysqld--help.result | 3 + .../galera/r/galera_var_dirty_reads.result | 70 +++++++++++++++++++ .../galera/t/galera_var_dirty_reads.test | 68 ++++++++++++++++++ .../sys_vars/r/wsrep_dirty_reads_basic.result | 23 +++++- .../sys_vars/t/wsrep_dirty_reads_basic.test | 17 ++++- sql/sql_parse.cc | 9 ++- sql/sys_vars.cc | 5 +- sql/wsrep_mysqld.cc | 4 ++ sql/wsrep_mysqld.h | 1 + 9 files changed, 191 insertions(+), 9 deletions(-) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 61243e02363..34d4bae0f28 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -879,6 +879,8 @@ The following options may be given as the first argument: DBUG options to provider library --wsrep-debug To enable debug level logging --wsrep-desync To desynchronize the node from the cluster + --wsrep-dirty-reads Allow reads even when the node is not in the primary + component. --wsrep-drupal-282555-workaround To use a workaround forbad autoincrement value --wsrep-forced-binlog-format=name @@ -1201,6 +1203,7 @@ wsrep-convert-LOCK-to-trx FALSE wsrep-dbug-option wsrep-debug FALSE wsrep-desync FALSE +wsrep-dirty-reads FALSE wsrep-drupal-282555-workaround FALSE wsrep-forced-binlog-format NONE wsrep-load-data-splitting TRUE diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 6d703c8cf95..8a3175912c7 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -3,6 +3,10 @@ INSERT INTO t1 VALUES(1); SELECT * FROM t1; i 1 +create user user1; +grant all privileges on *.* to user1; +create user user2; +grant all privileges on *.* to user2; SET @@global.wsrep_cluster_address = ''; SET @@session.wsrep_dirty_reads=OFF; SET SESSION wsrep_sync_wait=0; @@ -18,8 +22,74 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 +connect con1, localhost, user1,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2; +SET SESSION wsrep_sync_wait=0; +set session wsrep_dirty_reads=1; +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; +set session wsrep_dirty_reads=0; +execute stmt_show; +ERROR 08S01: WSREP has not yet prepared node for application use +execute stmt_select; +ERROR 08S01: WSREP has not yet prepared node for application use +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET wsrep_dirty_reads=ON; +select @@session.wsrep_dirty_reads; +@@session.wsrep_dirty_reads +1 +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET @@global.wsrep_dirty_reads=ON; +connect con2, localhost, user2,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2; +select @@session.wsrep_dirty_reads; +@@session.wsrep_dirty_reads +1 +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET SESSION wsrep_sync_wait=1; +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET SESSION wsrep_sync_wait=7; +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +connection node_2; +SET @@global.wsrep_dirty_reads=OFF; +connection node_1; SELECT * FROM t1; i 1 DROP TABLE t1; +drop user user1; +drop user user2; +disconnect node_2; +disconnect node_1; # End of test diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index 9eea8efdaf3..8fd3b1d22f2 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -20,6 +20,11 @@ CREATE TABLE t1(i INT) ENGINE=INNODB; INSERT INTO t1 VALUES(1); SELECT * FROM t1; +create user user1; +grant all privileges on *.* to user1; +create user user2; +grant all privileges on *.* to user2; + SET @@global.wsrep_cluster_address = ''; SET @@session.wsrep_dirty_reads=OFF; @@ -39,6 +44,67 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; +--enable_connect_log +--connect (con1, localhost, user1,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2) +#Just test the session behavior +SET SESSION wsrep_sync_wait=0; + +set session wsrep_dirty_reads=1; +#Prepared statement creation should be allowed MDEV-11479 +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; +set session wsrep_dirty_reads=0; + +#No Preapare stmt/proceure will be allowed +--error ER_UNKNOWN_COM_ERROR +execute stmt_show; +--error ER_UNKNOWN_COM_ERROR +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +SET wsrep_dirty_reads=ON; +select @@session.wsrep_dirty_reads; +#Only prepare statement which does not change data should be allowed +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; +SET @@global.wsrep_dirty_reads=ON; + +--connect (con2, localhost, user2,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2) +#Just test the session behavior +select @@session.wsrep_dirty_reads; + +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; + +#Only prepare statement which does not change data should be allowed +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +#wsrep_dirty_read should work when wsrep_sync_wait is 1 or non zero +#because we already are disconnected , So It does not make any sense +#to wait for other nodes +SET SESSION wsrep_sync_wait=1; +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +SET SESSION wsrep_sync_wait=7; +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +--connection node_2 +SET @@global.wsrep_dirty_reads=OFF; + --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' --enable_query_log @@ -48,6 +114,8 @@ SELECT * FROM t1; SELECT * FROM t1; # Cleanup DROP TABLE t1; +drop user user1; +drop user user2; --disable_query_log # Restore original auto_increment_offset values. diff --git a/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result b/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result index d2a62d6136f..1968103873a 100644 --- a/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result +++ b/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result @@ -5,12 +5,13 @@ SET @wsrep_dirty_reads_session_saved = @@session.wsrep_dirty_reads; # default SELECT @@global.wsrep_dirty_reads; -ERROR HY000: Variable 'wsrep_dirty_reads' is a SESSION variable +@@global.wsrep_dirty_reads +0 SELECT @@session.wsrep_dirty_reads; @@session.wsrep_dirty_reads 0 -# scope and valid values +# valid values for session SET @@session.wsrep_dirty_reads=OFF; SELECT @@session.wsrep_dirty_reads; @@session.wsrep_dirty_reads @@ -24,11 +25,29 @@ SELECT @@session.wsrep_dirty_reads; @@session.wsrep_dirty_reads 0 +# valid values for global +SET @@global.wsrep_dirty_reads=OFF; +SELECT @@global.wsrep_dirty_reads; +@@global.wsrep_dirty_reads +0 +SET @@global.wsrep_dirty_reads=ON; +SELECT @@global.wsrep_dirty_reads; +@@global.wsrep_dirty_reads +1 +SET @@global.wsrep_dirty_reads=default; +SELECT @@global.wsrep_dirty_reads; +@@global.wsrep_dirty_reads +0 + # invalid values SET @@session.wsrep_dirty_reads=NULL; ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'NULL' SET @@session.wsrep_dirty_reads='junk'; ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'junk' +SET @@global.wsrep_dirty_reads=NULL; +ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'NULL' +SET @@global.wsrep_dirty_reads='junk'; +ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'junk' # restore the initial values SET @@session.wsrep_dirty_reads = @wsrep_dirty_reads_session_saved; diff --git a/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test b/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test index a47524fcfe3..ffe767a051b 100644 --- a/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test +++ b/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test @@ -8,12 +8,12 @@ SET @wsrep_dirty_reads_session_saved = @@session.wsrep_dirty_reads; --echo # default ---error ER_INCORRECT_GLOBAL_LOCAL_VAR + SELECT @@global.wsrep_dirty_reads; SELECT @@session.wsrep_dirty_reads; --echo ---echo # scope and valid values +--echo # valid values for session SET @@session.wsrep_dirty_reads=OFF; SELECT @@session.wsrep_dirty_reads; SET @@session.wsrep_dirty_reads=ON; @@ -21,12 +21,25 @@ SELECT @@session.wsrep_dirty_reads; SET @@session.wsrep_dirty_reads=default; SELECT @@session.wsrep_dirty_reads; +--echo +--echo # valid values for global +SET @@global.wsrep_dirty_reads=OFF; +SELECT @@global.wsrep_dirty_reads; +SET @@global.wsrep_dirty_reads=ON; +SELECT @@global.wsrep_dirty_reads; +SET @@global.wsrep_dirty_reads=default; +SELECT @@global.wsrep_dirty_reads; + --echo --echo # invalid values --error ER_WRONG_VALUE_FOR_VAR SET @@session.wsrep_dirty_reads=NULL; --error ER_WRONG_VALUE_FOR_VAR SET @@session.wsrep_dirty_reads='junk'; +--error ER_WRONG_VALUE_FOR_VAR +SET @@global.wsrep_dirty_reads=NULL; +--error ER_WRONG_VALUE_FOR_VAR +SET @@global.wsrep_dirty_reads='junk'; --echo --echo # restore the initial values diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 48addaab143..ec6eb0e6f87 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2386,13 +2386,16 @@ mysql_execute_command(THD *thd) } /* - Bail out if DB snapshot has not been installed. We however, - allow SET and SHOW queries. + Bail out if DB snapshot has not been installed. SET and SHOW commands, + however, are always allowed. + + We additionally allow all other commands that do not change data in + case wsrep_dirty_reads is enabled. */ if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && lex->sql_command != SQLCOM_SET_OPTION && !(thd->variables.wsrep_dirty_reads && - lex->sql_command == SQLCOM_SELECT) && + !is_update_query(lex->sql_command)) && !wsrep_is_show_query(lex->sql_command)) { #if DIRTY_HACK diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index bdcedfff5a3..e0de7960ec9 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3966,8 +3966,9 @@ static Sys_var_mybool Sys_wsrep_restart_slave( GLOBAL_VAR(wsrep_restart_slave), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); static Sys_var_mybool Sys_wsrep_dirty_reads( - "wsrep_dirty_reads", "Do not reject SELECT queries even when the node " - "is not ready.", SESSION_ONLY(wsrep_dirty_reads), NO_CMD_LINE, + "wsrep_dirty_reads", + "Allow reads even when the node is not in the primary component.", + SESSION_VAR(wsrep_dirty_reads), CMD_LINE(OPT_ARG), DEFAULT(FALSE), NO_MUTEX_GUARD, NOT_IN_BINLOG); #endif /* WITH_WSREP */ diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 48114fcba34..21c7e94096d 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -65,6 +65,8 @@ my_bool wsrep_restart_slave_activated = 0; // node has dropped, and slave // restart will be needed my_bool wsrep_slave_UK_checks = 0; // slave thread does UK checks my_bool wsrep_slave_FK_checks = 0; // slave thread does FK checks +// Allow reads even if the node is not in the primary component. +bool wsrep_dirty_reads = false; /* Set during the creation of first wsrep applier and rollback threads. @@ -894,6 +896,8 @@ bool wsrep_must_sync_wait (THD* thd, uint mask) { return (thd->variables.wsrep_sync_wait & mask) && thd->variables.wsrep_on && + !(thd->variables.wsrep_dirty_reads && + !is_update_query(thd->lex->sql_command)) && !thd->in_active_multi_stmt_transaction() && thd->wsrep_conflict_state != REPLAYING && thd->wsrep_sync_wait_gtid.seqno == WSREP_SEQNO_UNDEFINED; diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 3fb5189937e..85d4cea241f 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -80,6 +80,7 @@ extern ulong wsrep_retry_autocommit; extern my_bool wsrep_auto_increment_control; extern my_bool wsrep_drupal_282555_workaround; extern my_bool wsrep_incremental_data_collection; +extern bool wsrep_dirty_reads; extern const char* wsrep_start_position; extern ulong wsrep_max_ws_size; extern ulong wsrep_max_ws_rows; From ec9a48112b3caac39fb133d524f499172942d43a Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Sun, 15 Jan 2017 22:32:22 -0500 Subject: [PATCH 008/203] Fix bad merge --- scripts/mysqld_safe.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 9ab51ded8a8..5bd3b5b06ac 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -988,7 +988,6 @@ do chmod "$fmode" "$err_log" # wrong owner next time we log, so set fi # it up correctly while we can! - eval_log_error "$cmd" end_time=`date +%M%S` if test ! -f "$pid_file" # This is removed if normal shutdown From aa820f9471ccd68a48ed81c1c8b5ad98f9cf20ba Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 10 Mar 2017 11:49:14 +0100 Subject: [PATCH 009/203] MDEV-12217 misssing full license clarify the license --- policy/selinux/mariadb-server.fc | 2 +- policy/selinux/mariadb-server.te | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/policy/selinux/mariadb-server.fc b/policy/selinux/mariadb-server.fc index 1a69ecc2c40..409f72923aa 100644 --- a/policy/selinux/mariadb-server.fc +++ b/policy/selinux/mariadb-server.fc @@ -1,4 +1,4 @@ -# This SELinux file contexts (.fc) file has been copied under BSD License from +# This SELinux file contexts (.fc) file has been copied under New BSD License from # Percona XtraDB Cluster. /etc/init\.d/rc\.d/mysql -- gen_context(system_u:object_r:mysqld_initrc_exec_t,s0) diff --git a/policy/selinux/mariadb-server.te b/policy/selinux/mariadb-server.te index 34d79326b10..45ef40f4153 100644 --- a/policy/selinux/mariadb-server.te +++ b/policy/selinux/mariadb-server.te @@ -1,4 +1,4 @@ -# This SELinux type enforcement (.te) file has been copied under BSD License +# This SELinux type enforcement (.te) file has been copied under New BSD License # from Percona XtraDB Cluster, along with some additions. module mariadb-server 1.0; From ea2695ccbc4ef80cb8b21a2ca4bf0eedb195adae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 5 Apr 2017 08:54:20 +0300 Subject: [PATCH 010/203] fix warning "ignoring return value" of fwrite. Merge pull request https://github.com/MariaDB/server/pull/343 contributed by Eric Herman. --- sql/wsrep_binlog.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/wsrep_binlog.cc b/sql/wsrep_binlog.cc index af4ec84db85..7feebc63cf4 100644 --- a/sql/wsrep_binlog.cc +++ b/sql/wsrep_binlog.cc @@ -308,9 +308,13 @@ void wsrep_dump_rbr_buf(THD *thd, const void* rbr_buf, size_t buf_len) } FILE *of= fopen(filename, "wb"); + if (of) { - fwrite (rbr_buf, buf_len, 1, of); + if (fwrite(rbr_buf, buf_len, 1, of) == 0) + WSREP_ERROR("Failed to write buffer of length %llu to '%s'", + (unsigned long long)buf_len, filename); + fclose(of); } else From 19b9fe07f5356b1d5965c16b322a0ca7d5fbdfe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 5 Apr 2017 10:50:12 +0300 Subject: [PATCH 011/203] Fix compiler error on gcc 6.x and most of the compiler warnings. --- sql/sql_class.cc | 4 ++-- sql/wsrep_sst.cc | 4 ++-- storage/innobase/btr/btr0btr.c | 1 - storage/innobase/btr/btr0cur.c | 8 +------- storage/innobase/buf/buf0buddy.c | 1 - storage/innobase/dict/dict0crea.c | 10 +++++----- storage/innobase/dict/dict0dict.c | 1 - storage/innobase/dict/dict0load.c | 10 +++++----- storage/innobase/dict/dict0mem.c | 2 +- storage/innobase/dyn/dyn0dyn.c | 1 - storage/innobase/fil/fil0fil.c | 2 +- storage/innobase/fsp/fsp0fsp.c | 5 ----- storage/innobase/handler/ha_innodb.h | 4 ++-- storage/innobase/include/dict0dict.ic | 7 ------- storage/innobase/include/dict0mem.h | 2 +- storage/innobase/include/dyn0dyn.ic | 10 ---------- storage/innobase/include/fsp0fsp.h | 2 +- storage/innobase/include/mach0data.ic | 13 ------------- storage/innobase/include/mtr0mtr.ic | 1 - storage/innobase/include/page0page.ic | 1 - storage/innobase/include/rem0rec.ic | 1 - storage/innobase/include/ut0lst.h | 3 --- storage/innobase/mtr/mtr0mtr.c | 1 - storage/innobase/page/page0page.c | 2 -- storage/innobase/page/page0zip.c | 1 - storage/innobase/rem/rem0rec.c | 3 +-- storage/innobase/row/row0merge.c | 12 ------------ storage/innobase/row/row0purge.c | 3 --- storage/innobase/row/row0upd.c | 5 ----- storage/innobase/trx/trx0trx.c | 1 - storage/xtradb/btr/btr0btr.c | 1 - storage/xtradb/btr/btr0cur.c | 10 ++-------- storage/xtradb/buf/buf0buddy.c | 1 - storage/xtradb/dict/dict0crea.c | 10 +++++----- storage/xtradb/dict/dict0dict.c | 1 - storage/xtradb/dict/dict0load.c | 10 +++++----- storage/xtradb/dict/dict0mem.c | 2 +- storage/xtradb/dyn/dyn0dyn.c | 1 - storage/xtradb/fil/fil0fil.c | 2 +- storage/xtradb/fsp/fsp0fsp.c | 5 ----- storage/xtradb/handler/ha_innodb.h | 4 ++-- storage/xtradb/include/btr0sea.ic | 4 ---- storage/xtradb/include/dict0dict.ic | 7 ------- storage/xtradb/include/dict0mem.h | 2 +- storage/xtradb/include/dyn0dyn.ic | 10 ---------- storage/xtradb/include/fsp0fsp.h | 2 +- storage/xtradb/include/mach0data.ic | 13 ------------- storage/xtradb/include/mtr0mtr.ic | 1 - storage/xtradb/include/page0page.ic | 1 - storage/xtradb/include/rem0rec.ic | 1 - storage/xtradb/include/ut0lst.h | 3 --- storage/xtradb/mtr/mtr0mtr.c | 1 - storage/xtradb/page/page0page.c | 2 -- storage/xtradb/page/page0zip.c | 1 - storage/xtradb/rem/rem0rec.c | 3 +-- storage/xtradb/row/row0merge.c | 12 ------------ storage/xtradb/row/row0purge.c | 3 --- storage/xtradb/row/row0upd.c | 5 ----- storage/xtradb/trx/trx0trx.c | 1 - 59 files changed, 41 insertions(+), 199 deletions(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 7905daa235d..fd2dec5a146 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -853,11 +853,11 @@ extern "C" wsrep_ws_handle_t* wsrep_thd_ws_handle(THD *thd) return &thd->wsrep_ws_handle; } -extern "C"void wsrep_thd_LOCK(THD *thd) +extern "C" void wsrep_thd_LOCK(THD *thd) { mysql_mutex_lock(&thd->LOCK_wsrep_thd); } -extern "C"void wsrep_thd_UNLOCK(THD *thd) +extern "C" void wsrep_thd_UNLOCK(THD *thd) { mysql_mutex_unlock(&thd->LOCK_wsrep_thd); } diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index 8008a977a84..a5fe8a1e2d8 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -748,7 +748,7 @@ static int sst_donate_mysqldump (const char* addr, "%s", host, port, mysqld_port, mysqld_unix_port, mysql_real_data_home, wsrep_defaults_file, uuid_str, - (long long)seqno, bypass ? " "WSREP_SST_OPT_BYPASS : ""); + (long long)seqno, bypass ? " " WSREP_SST_OPT_BYPASS : ""); if (ret < 0 || ret >= cmd_len) { @@ -1023,7 +1023,7 @@ static int sst_donate_other (const char* method, method, addr, mysqld_unix_port, mysql_real_data_home, wsrep_defaults_file, uuid, (long long) seqno, - bypass ? " "WSREP_SST_OPT_BYPASS : ""); + bypass ? " " WSREP_SST_OPT_BYPASS : ""); if (ret < 0 || ret >= cmd_len) { diff --git a/storage/innobase/btr/btr0btr.c b/storage/innobase/btr/btr0btr.c index ede72ba57bb..58d816559d4 100644 --- a/storage/innobase/btr/btr0btr.c +++ b/storage/innobase/btr/btr0btr.c @@ -2932,7 +2932,6 @@ btr_level_list_remove_func( ulint prev_page_no; ulint next_page_no; - ut_ad(page && mtr); ut_ad(mtr_memo_contains_page(mtr, page, MTR_MEMO_PAGE_X_FIX)); ut_ad(space == page_get_space_id(page)); /* Get the previous and next page numbers of page */ diff --git a/storage/innobase/btr/btr0cur.c b/storage/innobase/btr/btr0cur.c index 3b533909e67..2c1ab4bffae 100644 --- a/storage/innobase/btr/btr0cur.c +++ b/storage/innobase/btr/btr0cur.c @@ -1877,7 +1877,6 @@ btr_cur_update_alloc_zip( mtr_t* mtr) /*!< in: mini-transaction */ { ut_a(page_zip == buf_block_get_page_zip(block)); - ut_ad(page_zip); ut_ad(!dict_index_is_ibuf(index)); if (page_zip_available(page_zip, dict_index_is_clust(index), @@ -2837,7 +2836,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(page_is_leaf(page_align(rec))); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && thr) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr), index, "del mark "); rec_print_new(stderr, rec, offsets); } @@ -4128,7 +4127,6 @@ btr_cur_disown_inherited_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(!rec_offs_comp(offsets) || !rec_get_node_ptr_flag(rec)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(mtr); for (i = 0; i < rec_offs_n_fields(offsets); i++) { if (rec_offs_nth_extern(offsets, i) @@ -4191,9 +4189,6 @@ btr_push_update_extern_fields( ulint n; const upd_field_t* uf; - ut_ad(tuple); - ut_ad(update); - uf = update->fields; n = upd_get_n_fields(update); @@ -4366,7 +4361,6 @@ btr_store_big_rec_extern_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(btr_mtr); ut_ad(mtr_memo_contains(btr_mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK)); ut_ad(mtr_memo_contains(btr_mtr, rec_block, MTR_MEMO_PAGE_X_FIX)); diff --git a/storage/innobase/buf/buf0buddy.c b/storage/innobase/buf/buf0buddy.c index 9277a89ce66..fa2515eddc2 100644 --- a/storage/innobase/buf/buf0buddy.c +++ b/storage/innobase/buf/buf0buddy.c @@ -276,7 +276,6 @@ buf_buddy_alloc_low( { buf_block_t* block; - ut_ad(lru); ut_ad(buf_pool_mutex_own(buf_pool)); ut_ad(!mutex_own(&buf_pool->zip_mutex)); ut_ad(i >= buf_buddy_get_slot(PAGE_ZIP_MIN_SIZE)); diff --git a/storage/innobase/dict/dict0crea.c b/storage/innobase/dict/dict0crea.c index f8bcc6f2d5f..4e173fb905d 100644 --- a/storage/innobase/dict/dict0crea.c +++ b/storage/innobase/dict/dict0crea.c @@ -112,13 +112,13 @@ dict_create_sys_tables_tuple( dfield = dtuple_get_nth_field(entry, 3/*TYPE*/); ptr = mem_heap_alloc(heap, 4); - if (table->flags & (~DICT_TF_COMPACT & ~(~0 << DICT_TF_BITS))) { + if (table->flags & (~DICT_TF_COMPACT & ~(~0U << DICT_TF_BITS))) { ut_a(table->flags & DICT_TF_COMPACT); ut_a(dict_table_get_format(table) >= DICT_TF_FORMAT_ZIP); ut_a((table->flags & DICT_TF_ZSSIZE_MASK) <= (DICT_TF_ZSSIZE_MAX << DICT_TF_ZSSIZE_SHIFT)); - ut_a(!(table->flags & (~0 << DICT_TF2_BITS))); - mach_write_to_4(ptr, table->flags & ~(~0 << DICT_TF_BITS)); + ut_a(!(table->flags & (~0U << DICT_TF2_BITS))); + mach_write_to_4(ptr, table->flags & ~(~0U << DICT_TF_BITS)); } else { mach_write_to_4(ptr, DICT_TABLE_ORDINARY); } @@ -306,7 +306,7 @@ dict_build_table_def_step( ut_ad(!dict_table_zip_size(table) || dict_table_get_format(table) >= DICT_TF_FORMAT_ZIP); - flags = table->flags & ~(~0 << DICT_TF_BITS); + flags = table->flags & ~(~0U << DICT_TF_BITS); error = fil_create_new_single_table_tablespace( space, path_or_name, is_path, flags == DICT_TF_COMPACT ? 0 : flags, @@ -325,7 +325,7 @@ dict_build_table_def_step( mtr_commit(&mtr); } else { /* Create in the system tablespace: disallow new features */ - table->flags &= (~0 << DICT_TF_BITS) | DICT_TF_COMPACT; + table->flags &= (~0U << DICT_TF_BITS) | DICT_TF_COMPACT; } row = dict_create_sys_tables_tuple(table, node->heap); diff --git a/storage/innobase/dict/dict0dict.c b/storage/innobase/dict/dict0dict.c index 2a59347bc78..17cf10412a8 100644 --- a/storage/innobase/dict/dict0dict.c +++ b/storage/innobase/dict/dict0dict.c @@ -5868,7 +5868,6 @@ dict_set_corrupted( const char* status; btr_cur_t cursor; - ut_ad(index); ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(!dict_table_is_comp(dict_sys->sys_tables)); ut_ad(!dict_table_is_comp(dict_sys->sys_indexes)); diff --git a/storage/innobase/dict/dict0load.c b/storage/innobase/dict/dict0load.c index 22de23af7ee..351a42f110d 100644 --- a/storage/innobase/dict/dict0load.c +++ b/storage/innobase/dict/dict0load.c @@ -643,7 +643,7 @@ dict_sys_tables_get_flags( return(ULINT_UNDEFINED); } - if (UNIV_UNLIKELY(flags & (~0 << DICT_TF_BITS))) { + if (UNIV_UNLIKELY(flags & (~0U << DICT_TF_BITS))) { /* Some unused bits are set. */ return(ULINT_UNDEFINED); } @@ -1332,7 +1332,7 @@ err_len: goto err_len; } type = mach_read_from_4(field); - if (UNIV_UNLIKELY(type & (~0 << DICT_IT_BITS))) { + if (UNIV_UNLIKELY(type & (~0U << DICT_IT_BITS))) { return("unknown SYS_INDEXES.TYPE bits"); } @@ -1696,7 +1696,7 @@ err_len: flags2 = mach_read_from_4(field); - if (flags2 & (~0 << (DICT_TF2_BITS - DICT_TF2_SHIFT))) { + if (flags2 & (~0U << (DICT_TF2_BITS - DICT_TF2_SHIFT))) { ut_print_timestamp(stderr); fputs(" InnoDB: Warning: table ", stderr); ut_print_filename(stderr, name); @@ -1705,7 +1705,7 @@ err_len: " has unknown flags %lx.\n", (ulong) flags2); - flags2 &= ~(~0 << (DICT_TF2_BITS - DICT_TF2_SHIFT)); + flags2 &= ~(~0U << (DICT_TF2_BITS - DICT_TF2_SHIFT)); } flags |= flags2 << DICT_TF2_SHIFT; @@ -1838,7 +1838,7 @@ err_exit: if (!fil_open_single_table_tablespace( TRUE, table->space, table->flags == DICT_TF_COMPACT ? 0 : - table->flags & ~(~0 << DICT_TF_BITS), name)) { + table->flags & ~(~0U << DICT_TF_BITS), name)) { /* We failed to find a sensible tablespace file */ diff --git a/storage/innobase/dict/dict0mem.c b/storage/innobase/dict/dict0mem.c index cea7a253ba0..87d03eff3c2 100644 --- a/storage/innobase/dict/dict0mem.c +++ b/storage/innobase/dict/dict0mem.c @@ -69,7 +69,7 @@ dict_mem_table_create( DBUG_ENTER("dict_mem_table_create"); ut_ad(name); - ut_a(!(flags & (~0 << DICT_TF2_BITS))); + ut_a(!(flags & (~0U << DICT_TF2_BITS))); heap = mem_heap_create(DICT_HEAP_SIZE); diff --git a/storage/innobase/dyn/dyn0dyn.c b/storage/innobase/dyn/dyn0dyn.c index d0f50ad0c32..b723877d648 100644 --- a/storage/innobase/dyn/dyn0dyn.c +++ b/storage/innobase/dyn/dyn0dyn.c @@ -40,7 +40,6 @@ dyn_array_add_block( mem_heap_t* heap; dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/innobase/fil/fil0fil.c b/storage/innobase/fil/fil0fil.c index 7aa1a42358e..7b29eda02c6 100644 --- a/storage/innobase/fil/fil0fil.c +++ b/storage/innobase/fil/fil0fil.c @@ -3308,7 +3308,7 @@ fil_open_single_table_tablespace( } if (space_id != id - || space_flags != (flags & ~(~0 << DICT_TF_BITS))) { + || space_flags != (flags & ~(~0U << DICT_TF_BITS))) { ut_print_timestamp(stderr); fputs(" InnoDB: Error: tablespace id and flags in file ", diff --git a/storage/innobase/fsp/fsp0fsp.c b/storage/innobase/fsp/fsp0fsp.c index 6d00a1f1d4a..e2e27608e5d 100644 --- a/storage/innobase/fsp/fsp0fsp.c +++ b/storage/innobase/fsp/fsp0fsp.c @@ -1285,7 +1285,6 @@ fsp_fill_free_list( ulint i; mtr_t ibuf_mtr; - ut_ad(header && mtr); ut_ad(page_offset(header) == FSP_HEADER_OFFSET); /* Check if we can fill free list from above the free list limit */ @@ -1584,9 +1583,6 @@ fsp_alloc_free_page( ulint page_no; ulint space_size; - ut_ad(mtr); - ut_ad(init_mtr); - header = fsp_get_space_header(space, zip_size, mtr); /* Get the hinted descriptor */ @@ -2602,7 +2598,6 @@ fseg_alloc_free_page_low( ibool success; ulint n; - ut_ad(mtr); ut_ad((direction >= FSP_UP) && (direction <= FSP_NO_DIR)); ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 8034f4dd535..f9b6d6ff582 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -319,8 +319,8 @@ extern "C" void wsrep_thd_set_conflict_state( extern "C" void wsrep_thd_set_trx_to_replay(THD *thd, uint64 trx_id); -extern "C"void wsrep_thd_LOCK(THD *thd); -extern "C"void wsrep_thd_UNLOCK(THD *thd); +extern "C" void wsrep_thd_LOCK(THD *thd); +extern "C" void wsrep_thd_UNLOCK(THD *thd); extern "C" uint32 wsrep_thd_wsrep_rand(THD *thd); extern "C" time_t wsrep_thd_query_start(THD *thd); extern "C" my_thread_id wsrep_thd_thread_id(THD *thd); diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index b65cae2a1d8..dbc3ce99ab0 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -247,7 +247,6 @@ dict_index_is_clust( /*================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_CLUSTERED)); @@ -261,7 +260,6 @@ dict_index_is_unique( /*=================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_UNIQUE)); @@ -276,7 +274,6 @@ dict_index_is_ibuf( /*===============*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_IBUF)); @@ -293,7 +290,6 @@ dict_index_is_sec_or_ibuf( { ulint type; - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); type = index->type; @@ -311,7 +307,6 @@ dict_table_get_n_user_cols( /*=======================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols - DATA_N_SYS_COLS); @@ -343,7 +338,6 @@ dict_table_get_n_cols( /*==================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols); @@ -937,7 +931,6 @@ dict_index_is_corrupted( /*====================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY((index->type & DICT_CORRUPT) diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index daa3c35ebfb..09834308bb6 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -87,7 +87,7 @@ combination of types */ /* @{ */ #define DICT_TF_FORMAT_SHIFT 5 /* file format */ #define DICT_TF_FORMAT_MASK \ -((~(~0 << (DICT_TF_BITS - DICT_TF_FORMAT_SHIFT))) << DICT_TF_FORMAT_SHIFT) +((~(~0U << (DICT_TF_BITS - DICT_TF_FORMAT_SHIFT))) << DICT_TF_FORMAT_SHIFT) #define DICT_TF_FORMAT_51 0 /*!< InnoDB/MySQL up to 5.1 */ #define DICT_TF_FORMAT_ZIP 1 /*!< InnoDB plugin for 5.1: compressed tables, diff --git a/storage/innobase/include/dyn0dyn.ic b/storage/innobase/include/dyn0dyn.ic index 177877ed1fd..2565a249271 100644 --- a/storage/innobase/include/dyn0dyn.ic +++ b/storage/innobase/include/dyn0dyn.ic @@ -47,8 +47,6 @@ dyn_block_get_used( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((block->used) & ~DYN_BLOCK_FULL_FLAG); } @@ -61,8 +59,6 @@ dyn_block_get_data( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((byte*) block->data); } @@ -76,7 +72,6 @@ dyn_array_create( dyn_array_t* arr) /*!< in/out: memory buffer of size sizeof(dyn_array_t) */ { - ut_ad(arr); #if DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG # error "DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG" #endif @@ -119,7 +114,6 @@ dyn_array_push( dyn_block_t* block; ulint used; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -159,7 +153,6 @@ dyn_array_open( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -195,7 +188,6 @@ dyn_array_close( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); block = dyn_array_get_last_block(arr); @@ -222,7 +214,6 @@ dyn_array_get_element( { const dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); /* Get the first array block */ @@ -260,7 +251,6 @@ dyn_array_get_data_size( const dyn_block_t* block; ulint sum = 0; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index f07e3decc66..0b813182947 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -42,7 +42,7 @@ Created 12/18/1995 Heikki Tuuri #define FSP_FLAGS_POS_PAGE_SSIZE 6 /** Bit mask of the PAGE_SSIZE field */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ + ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) /** Return the value of the PAGE_SSIZE field */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index 238a56577af..fdcb07ca29b 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -50,7 +50,6 @@ mach_read_from_1( /*=============*/ const byte* b) /*!< in: pointer to byte */ { - ut_ad(b); return((ulint)(b[0])); } @@ -143,7 +142,6 @@ mach_read_from_3( /*=============*/ const byte* b) /*!< in: pointer to 3 bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 16) | ((ulint)(b[1]) << 8) | (ulint)(b[2]) @@ -178,7 +176,6 @@ mach_read_from_4( /*=============*/ const byte* b) /*!< in: pointer to four bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 24) | ((ulint)(b[1]) << 16) | ((ulint)(b[2]) << 8) @@ -255,8 +252,6 @@ mach_read_compressed( { ulint flag; - ut_ad(b); - flag = mach_read_from_1(b); if (flag < 0x80UL) { @@ -333,8 +328,6 @@ mach_read_from_7( /*=============*/ const byte* b) /*!< in: pointer to 7 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_3(b), mach_read_from_4(b + 3))); } @@ -364,8 +357,6 @@ mach_read_from_6( /*=============*/ const byte* b) /*!< in: pointer to 6 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_2(b), mach_read_from_4(b + 2))); } @@ -413,8 +404,6 @@ mach_ull_read_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - n = (ib_uint64_t) mach_read_compressed(b); size = mach_get_compressed_size((ulint) n); @@ -480,8 +469,6 @@ mach_ull_read_much_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - if (*b != (byte)0xFF) { n = 0; size = 0; diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic index a03a0271535..c9e2f2e39de 100644 --- a/storage/innobase/include/mtr0mtr.ic +++ b/storage/innobase/include/mtr0mtr.ic @@ -149,7 +149,6 @@ mtr_memo_contains( dyn_array_t* memo; ulint offset; - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE || mtr->state == MTR_COMMITTING); diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 781ad029e87..1689c0ade6b 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -154,7 +154,6 @@ page_header_get_offs( { ulint offs; - ut_ad(page); ut_ad((field == PAGE_FREE) || (field == PAGE_LAST_INSERT) || (field == PAGE_HEAP_TOP)); diff --git a/storage/innobase/include/rem0rec.ic b/storage/innobase/include/rem0rec.ic index 564d2d1b31c..c81388391d7 100644 --- a/storage/innobase/include/rem0rec.ic +++ b/storage/innobase/include/rem0rec.ic @@ -1545,7 +1545,6 @@ rec_copy( ulint extra_len; ulint data_len; - ut_ad(rec && buf); ut_ad(rec_offs_validate((rec_t*) rec, NULL, offsets)); ut_ad(rec_validate(rec, offsets)); diff --git a/storage/innobase/include/ut0lst.h b/storage/innobase/include/ut0lst.h index a010f464570..1fb43fd4805 100644 --- a/storage/innobase/include/ut0lst.h +++ b/storage/innobase/include/ut0lst.h @@ -88,7 +88,6 @@ Adds the node as the first element in a two-way linked list. */ #define UT_LIST_ADD_FIRST(NAME, BASE, N)\ {\ - ut_ad(N);\ ((BASE).count)++;\ ((N)->NAME).next = (BASE).start;\ ((N)->NAME).prev = NULL;\ @@ -134,7 +133,6 @@ Inserts a NODE2 after NODE1 in a list. #define UT_LIST_INSERT_AFTER(NAME, BASE, NODE1, NODE2)\ {\ ut_ad(NODE1);\ - ut_ad(NODE2);\ ut_ad((NODE1) != (NODE2));\ ((BASE).count)++;\ ((NODE2)->NAME).prev = (NODE1);\ @@ -169,7 +167,6 @@ Removes a node from a two-way linked list. */ #define UT_LIST_REMOVE(NAME, BASE, N) \ do { \ - ut_ad(N); \ ut_a((BASE).count > 0); \ ((BASE).count)--; \ if (((N)->NAME).next != NULL) { \ diff --git a/storage/innobase/mtr/mtr0mtr.c b/storage/innobase/mtr/mtr0mtr.c index a5c98761523..ebad6e7bfa1 100644 --- a/storage/innobase/mtr/mtr0mtr.c +++ b/storage/innobase/mtr/mtr0mtr.c @@ -257,7 +257,6 @@ mtr_commit( /*=======*/ mtr_t* mtr) /*!< in: mini-transaction */ { - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE); ut_ad(!mtr->inside_ibuf); diff --git a/storage/innobase/page/page0page.c b/storage/innobase/page/page0page.c index 81051f8f4fe..9af4b1c8faa 100644 --- a/storage/innobase/page/page0page.c +++ b/storage/innobase/page/page0page.c @@ -1345,7 +1345,6 @@ page_dir_split_slot( ulint i; ulint n_owned; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); @@ -1407,7 +1406,6 @@ page_dir_balance_slot( rec_t* old_rec; rec_t* new_rec; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); diff --git a/storage/innobase/page/page0zip.c b/storage/innobase/page/page0zip.c index 7c04e0b9183..faa004be730 100644 --- a/storage/innobase/page/page0zip.c +++ b/storage/innobase/page/page0zip.c @@ -4739,7 +4739,6 @@ page_zip_parse_compress( ulint size; ulint trailer_size; - ut_ad(ptr && end_ptr); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(ptr + (2 + 2) > end_ptr)) { diff --git a/storage/innobase/rem/rem0rec.c b/storage/innobase/rem/rem0rec.c index 5351690dcef..84f04beb61d 100644 --- a/storage/innobase/rem/rem0rec.c +++ b/storage/innobase/rem/rem0rec.c @@ -794,8 +794,7 @@ rec_get_converted_size_comp_prefix_low( ulint extra_size; ulint data_size; ulint i; - ut_ad(index); - ut_ad(fields); + ut_ad(n_fields > 0); ut_ad(n_fields <= dict_index_get_n_fields(index)); ut_ad(!temp || extra); diff --git a/storage/innobase/row/row0merge.c b/storage/innobase/row/row0merge.c index 7d87d1f9c8f..a393254d145 100644 --- a/storage/innobase/row/row0merge.c +++ b/storage/innobase/row/row0merge.c @@ -807,14 +807,8 @@ row_merge_read_rec( ulint data_size; ulint avail_size; - ut_ad(block); - ut_ad(buf); ut_ad(b >= block[0]); ut_ad(b < block[1]); - ut_ad(index); - ut_ad(foffs); - ut_ad(mrec); - ut_ad(offsets); ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); @@ -1175,12 +1169,6 @@ row_merge_read_clustered_index( trx->op_info = "reading clustered index"; - ut_ad(trx); - ut_ad(old_table); - ut_ad(new_table); - ut_ad(index); - ut_ad(files); - /* Create and initialize memory for record buffers */ merge_buf = mem_alloc(n_index * sizeof *merge_buf); diff --git a/storage/innobase/row/row0purge.c b/storage/innobase/row/row0purge.c index 9018582f5d6..5f3e4175544 100644 --- a/storage/innobase/row/row0purge.c +++ b/storage/innobase/row/row0purge.c @@ -751,9 +751,6 @@ row_purge( { ibool updated_extern; - ut_ad(node); - ut_ad(thr); - node->undo_rec = trx_purge_fetch_next_rec(&node->roll_ptr, &node->reservation, node->heap); diff --git a/storage/innobase/row/row0upd.c b/storage/innobase/row/row0upd.c index 05924958edd..3560a8bff1a 100644 --- a/storage/innobase/row/row0upd.c +++ b/storage/innobase/row/row0upd.c @@ -1235,8 +1235,6 @@ row_upd_index_replace_new_col_vals_index_pos( ulint n_fields; const ulint zip_size = dict_table_zip_size(index->table); - ut_ad(index); - dtuple_set_info_bits(entry, update->info_bits); if (order_only) { @@ -1419,9 +1417,6 @@ row_upd_changes_ord_field_binary_func( ulint i; const dict_index_t* clust_index; - ut_ad(index); - ut_ad(update); - ut_ad(thr); ut_ad(thr->graph); ut_ad(thr->graph->trx); diff --git a/storage/innobase/trx/trx0trx.c b/storage/innobase/trx/trx0trx.c index ce43c80e826..558cd74cfca 100644 --- a/storage/innobase/trx/trx0trx.c +++ b/storage/innobase/trx/trx0trx.c @@ -97,7 +97,6 @@ trx_create( trx_t* trx; ut_ad(mutex_own(&kernel_mutex)); - ut_ad(sess); trx = mem_alloc(sizeof(trx_t)); diff --git a/storage/xtradb/btr/btr0btr.c b/storage/xtradb/btr/btr0btr.c index 0c429363789..fb670df2329 100644 --- a/storage/xtradb/btr/btr0btr.c +++ b/storage/xtradb/btr/btr0btr.c @@ -2974,7 +2974,6 @@ btr_level_list_remove_func( ulint prev_page_no; ulint next_page_no; - ut_ad(page && mtr); ut_ad(mtr_memo_contains_page(mtr, page, MTR_MEMO_PAGE_X_FIX)); ut_ad(space == page_get_space_id(page)); /* Get the previous and next page numbers of page */ diff --git a/storage/xtradb/btr/btr0cur.c b/storage/xtradb/btr/btr0cur.c index 0ec9367b27c..d9c1a2ef7a8 100644 --- a/storage/xtradb/btr/btr0cur.c +++ b/storage/xtradb/btr/btr0cur.c @@ -2008,7 +2008,6 @@ btr_cur_update_alloc_zip( trx_t* trx) /*!< in: NULL or transaction */ { ut_a(page_zip == buf_block_get_page_zip(block)); - ut_ad(page_zip); ut_ad(!dict_index_is_ibuf(index)); if (page_zip_available(page_zip, dict_index_is_clust(index), @@ -3008,7 +3007,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(page_is_leaf(page_align(rec))); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && thr) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr), index, "del mark "); rec_print_new(stderr, rec, offsets); } @@ -3017,7 +3016,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(dict_index_is_clust(index)); ut_ad(!rec_get_deleted_flag(rec, rec_offs_comp(offsets))); - if (UNIV_UNLIKELY(thr && thr_get_trx(thr)->fake_changes)) { + if (UNIV_UNLIKELY(thr_get_trx(thr)->fake_changes)) { /* skip LOCK, UNDO, CHANGE, LOG */ return(DB_SUCCESS); } @@ -4323,7 +4322,6 @@ btr_cur_disown_inherited_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(!rec_offs_comp(offsets) || !rec_get_node_ptr_flag(rec)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(mtr); for (i = 0; i < rec_offs_n_fields(offsets); i++) { if (rec_offs_nth_extern(offsets, i) @@ -4386,9 +4384,6 @@ btr_push_update_extern_fields( ulint n; const upd_field_t* uf; - ut_ad(tuple); - ut_ad(update); - uf = update->fields; n = upd_get_n_fields(update); @@ -4571,7 +4566,6 @@ btr_store_big_rec_extern_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(btr_mtr); ut_ad(mtr_memo_contains(btr_mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK)); ut_ad(mtr_memo_contains(btr_mtr, rec_block, MTR_MEMO_PAGE_X_FIX)); diff --git a/storage/xtradb/buf/buf0buddy.c b/storage/xtradb/buf/buf0buddy.c index 439be08b01f..493d0d2d41c 100644 --- a/storage/xtradb/buf/buf0buddy.c +++ b/storage/xtradb/buf/buf0buddy.c @@ -288,7 +288,6 @@ buf_buddy_alloc_low( { buf_block_t* block; - ut_ad(lru); //ut_ad(buf_pool_mutex_own(buf_pool)); ut_ad(mutex_own(&buf_pool->LRU_list_mutex)); ut_ad(!mutex_own(&buf_pool->zip_mutex)); diff --git a/storage/xtradb/dict/dict0crea.c b/storage/xtradb/dict/dict0crea.c index b44fdc1d2e4..1529c22bbb9 100644 --- a/storage/xtradb/dict/dict0crea.c +++ b/storage/xtradb/dict/dict0crea.c @@ -112,13 +112,13 @@ dict_create_sys_tables_tuple( dfield = dtuple_get_nth_field(entry, 3/*TYPE*/); ptr = mem_heap_alloc(heap, 4); - if (table->flags & (~DICT_TF_COMPACT & ~(~0 << DICT_TF_BITS))) { + if (table->flags & (~DICT_TF_COMPACT & ~(~0U << DICT_TF_BITS))) { ut_a(table->flags & DICT_TF_COMPACT); ut_a(dict_table_get_format(table) >= DICT_TF_FORMAT_ZIP); ut_a(((ulonglong) table->flags & DICT_TF_ZSSIZE_MASK) <= (ulonglong) (DICT_TF_ZSSIZE_MAX << DICT_TF_ZSSIZE_SHIFT)); - ut_a(!(table->flags & (~0 << DICT_TF2_BITS))); - mach_write_to_4(ptr, table->flags & ~(~0 << DICT_TF_BITS)); + ut_a(!(table->flags & (~0U << DICT_TF2_BITS))); + mach_write_to_4(ptr, table->flags & ~(~0U << DICT_TF_BITS)); } else { mach_write_to_4(ptr, DICT_TABLE_ORDINARY); } @@ -306,7 +306,7 @@ dict_build_table_def_step( ut_ad(!dict_table_zip_size(table) || dict_table_get_format(table) >= DICT_TF_FORMAT_ZIP); - flags = table->flags & ~(~0 << DICT_TF_BITS); + flags = table->flags & ~(~0U << DICT_TF_BITS); error = fil_create_new_single_table_tablespace( space, path_or_name, is_path, flags == DICT_TF_COMPACT ? 0 : flags, @@ -325,7 +325,7 @@ dict_build_table_def_step( mtr_commit(&mtr); } else { /* Create in the system tablespace: disallow new features */ - table->flags &= (~0 << DICT_TF_BITS) | DICT_TF_COMPACT; + table->flags &= (~0U << DICT_TF_BITS) | DICT_TF_COMPACT; } row = dict_create_sys_tables_tuple(table, node->heap); diff --git a/storage/xtradb/dict/dict0dict.c b/storage/xtradb/dict/dict0dict.c index 6f16fc0bce0..a6970c20831 100644 --- a/storage/xtradb/dict/dict0dict.c +++ b/storage/xtradb/dict/dict0dict.c @@ -6414,7 +6414,6 @@ dict_set_corrupted( const char* status; btr_cur_t cursor; - ut_ad(index); ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(!dict_table_is_comp(dict_sys->sys_tables)); ut_ad(!dict_table_is_comp(dict_sys->sys_indexes)); diff --git a/storage/xtradb/dict/dict0load.c b/storage/xtradb/dict/dict0load.c index 1fdbffee3c8..5184746e48e 100644 --- a/storage/xtradb/dict/dict0load.c +++ b/storage/xtradb/dict/dict0load.c @@ -716,7 +716,7 @@ dict_sys_tables_get_flags( return(ULINT_UNDEFINED); } - if (UNIV_UNLIKELY(flags & (~0 << DICT_TF_BITS))) { + if (UNIV_UNLIKELY(flags & (~0U << DICT_TF_BITS))) { /* Some unused bits are set. */ return(ULINT_UNDEFINED); } @@ -1405,7 +1405,7 @@ err_len: goto err_len; } type = mach_read_from_4(field); - if (UNIV_UNLIKELY(type & (~0 << DICT_IT_BITS))) { + if (UNIV_UNLIKELY(type & (~0U << DICT_IT_BITS))) { return("unknown SYS_INDEXES.TYPE bits"); } @@ -1770,7 +1770,7 @@ err_len: flags2 = mach_read_from_4(field); - if (flags2 & (~0 << (DICT_TF2_BITS - DICT_TF2_SHIFT))) { + if (flags2 & (~0U << (DICT_TF2_BITS - DICT_TF2_SHIFT))) { ut_print_timestamp(stderr); fputs(" InnoDB: Warning: table ", stderr); ut_print_filename(stderr, name); @@ -1779,7 +1779,7 @@ err_len: " has unknown flags %lx.\n", (ulong) flags2); - flags2 &= ~(~0 << (DICT_TF2_BITS - DICT_TF2_SHIFT)); + flags2 &= ~(~0U << (DICT_TF2_BITS - DICT_TF2_SHIFT)); } flags |= flags2 << DICT_TF2_SHIFT; @@ -1912,7 +1912,7 @@ err_exit: if (!fil_open_single_table_tablespace( TRUE, table->space, table->flags == DICT_TF_COMPACT ? 0 : - table->flags & ~(~0 << DICT_TF_BITS), name, NULL)) { + table->flags & ~(~0U << DICT_TF_BITS), name, NULL)) { /* We failed to find a sensible tablespace file */ diff --git a/storage/xtradb/dict/dict0mem.c b/storage/xtradb/dict/dict0mem.c index 40099c06823..18917a30ff6 100644 --- a/storage/xtradb/dict/dict0mem.c +++ b/storage/xtradb/dict/dict0mem.c @@ -69,7 +69,7 @@ dict_mem_table_create( DBUG_ENTER("dict_mem_table_create"); ut_ad(name); - ut_a(!(flags & (~0 << DICT_TF2_BITS))); + ut_a(!(flags & (~0U << DICT_TF2_BITS))); heap = mem_heap_create(DICT_HEAP_SIZE); diff --git a/storage/xtradb/dyn/dyn0dyn.c b/storage/xtradb/dyn/dyn0dyn.c index d0f50ad0c32..b723877d648 100644 --- a/storage/xtradb/dyn/dyn0dyn.c +++ b/storage/xtradb/dyn/dyn0dyn.c @@ -40,7 +40,6 @@ dyn_array_add_block( mem_heap_t* heap; dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/xtradb/fil/fil0fil.c b/storage/xtradb/fil/fil0fil.c index 7682b49e755..86e00dc22e4 100644 --- a/storage/xtradb/fil/fil0fil.c +++ b/storage/xtradb/fil/fil0fil.c @@ -3883,7 +3883,7 @@ skip_write: } if (space_id != id - || space_flags != (flags & ~(~0 << DICT_TF_BITS))) { + || space_flags != (flags & ~(~0U << DICT_TF_BITS))) { ut_print_timestamp(stderr); fputs(" InnoDB: Error: tablespace id and flags in file ", diff --git a/storage/xtradb/fsp/fsp0fsp.c b/storage/xtradb/fsp/fsp0fsp.c index 772e224f6f7..4477b34f7b4 100644 --- a/storage/xtradb/fsp/fsp0fsp.c +++ b/storage/xtradb/fsp/fsp0fsp.c @@ -1290,7 +1290,6 @@ fsp_fill_free_list( ulint i; mtr_t ibuf_mtr; - ut_ad(header && mtr); ut_ad(page_offset(header) == FSP_HEADER_OFFSET); /* Check if we can fill free list from above the free list limit */ @@ -1589,9 +1588,6 @@ fsp_alloc_free_page( ulint page_no; ulint space_size; - ut_ad(mtr); - ut_ad(init_mtr); - header = fsp_get_space_header(space, zip_size, mtr); /* Get the hinted descriptor */ @@ -2613,7 +2609,6 @@ fseg_alloc_free_page_low( ibool success; ulint n; - ut_ad(mtr); ut_ad((direction >= FSP_UP) && (direction <= FSP_NO_DIR)); ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index 521a9f2664c..69a28905f22 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -398,8 +398,8 @@ extern "C" void wsrep_thd_set_conflict_state( extern "C" void wsrep_thd_set_trx_to_replay(THD *thd, uint64 trx_id); -extern "C"void wsrep_thd_LOCK(THD *thd); -extern "C"void wsrep_thd_UNLOCK(THD *thd); +extern "C" void wsrep_thd_LOCK(THD *thd); +extern "C" void wsrep_thd_UNLOCK(THD *thd); extern "C" uint32 wsrep_thd_wsrep_rand(THD *thd); extern "C" time_t wsrep_thd_query_start(THD *thd); extern "C" my_thread_id wsrep_thd_thread_id(THD *thd); diff --git a/storage/xtradb/include/btr0sea.ic b/storage/xtradb/include/btr0sea.ic index 3f0dfdaa511..677cb52f458 100644 --- a/storage/xtradb/include/btr0sea.ic +++ b/storage/xtradb/include/btr0sea.ic @@ -91,7 +91,6 @@ btr_search_get_hash_table( /*======================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->search_table); return(index->search_table); @@ -103,7 +102,6 @@ btr_search_get_latch( /*=================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->search_latch >= btr_search_latch_arr && index->search_latch < btr_search_latch_arr + btr_search_index_num); @@ -130,8 +128,6 @@ btr_search_index_init( /*===============*/ dict_index_t* index) /*!< in: index */ { - ut_ad(index); - index->search_latch = &btr_search_latch_arr[btr_search_get_key(index->id)]; index->search_table = diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index 6836928ff49..3410c945a80 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -247,7 +247,6 @@ dict_index_is_clust( /*================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_CLUSTERED)); @@ -261,7 +260,6 @@ dict_index_is_unique( /*=================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_UNIQUE)); @@ -276,7 +274,6 @@ dict_index_is_ibuf( /*===============*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_IBUF)); @@ -293,7 +290,6 @@ dict_index_is_sec_or_ibuf( { ulint type; - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); type = index->type; @@ -311,7 +307,6 @@ dict_table_get_n_user_cols( /*=======================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols - DATA_N_SYS_COLS); @@ -343,7 +338,6 @@ dict_table_get_n_cols( /*==================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols); @@ -950,7 +944,6 @@ dict_index_is_corrupted( /*====================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY((index->type & DICT_CORRUPT) diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index f174103c3fe..489a30592d1 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -87,7 +87,7 @@ combination of types */ /* @{ */ #define DICT_TF_FORMAT_SHIFT 5 /* file format */ #define DICT_TF_FORMAT_MASK \ -((~(~0 << (DICT_TF_BITS - DICT_TF_FORMAT_SHIFT))) << DICT_TF_FORMAT_SHIFT) +((~(~0U << (DICT_TF_BITS - DICT_TF_FORMAT_SHIFT))) << DICT_TF_FORMAT_SHIFT) #define DICT_TF_FORMAT_51 0 /*!< InnoDB/MySQL up to 5.1 */ #define DICT_TF_FORMAT_ZIP 1 /*!< InnoDB plugin for 5.1: compressed tables, diff --git a/storage/xtradb/include/dyn0dyn.ic b/storage/xtradb/include/dyn0dyn.ic index 177877ed1fd..2565a249271 100644 --- a/storage/xtradb/include/dyn0dyn.ic +++ b/storage/xtradb/include/dyn0dyn.ic @@ -47,8 +47,6 @@ dyn_block_get_used( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((block->used) & ~DYN_BLOCK_FULL_FLAG); } @@ -61,8 +59,6 @@ dyn_block_get_data( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((byte*) block->data); } @@ -76,7 +72,6 @@ dyn_array_create( dyn_array_t* arr) /*!< in/out: memory buffer of size sizeof(dyn_array_t) */ { - ut_ad(arr); #if DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG # error "DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG" #endif @@ -119,7 +114,6 @@ dyn_array_push( dyn_block_t* block; ulint used; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -159,7 +153,6 @@ dyn_array_open( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -195,7 +188,6 @@ dyn_array_close( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); block = dyn_array_get_last_block(arr); @@ -222,7 +214,6 @@ dyn_array_get_element( { const dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); /* Get the first array block */ @@ -260,7 +251,6 @@ dyn_array_get_data_size( const dyn_block_t* block; ulint sum = 0; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index f07e3decc66..0b813182947 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -42,7 +42,7 @@ Created 12/18/1995 Heikki Tuuri #define FSP_FLAGS_POS_PAGE_SSIZE 6 /** Bit mask of the PAGE_SSIZE field */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ + ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) /** Return the value of the PAGE_SSIZE field */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ diff --git a/storage/xtradb/include/mach0data.ic b/storage/xtradb/include/mach0data.ic index 238a56577af..fdcb07ca29b 100644 --- a/storage/xtradb/include/mach0data.ic +++ b/storage/xtradb/include/mach0data.ic @@ -50,7 +50,6 @@ mach_read_from_1( /*=============*/ const byte* b) /*!< in: pointer to byte */ { - ut_ad(b); return((ulint)(b[0])); } @@ -143,7 +142,6 @@ mach_read_from_3( /*=============*/ const byte* b) /*!< in: pointer to 3 bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 16) | ((ulint)(b[1]) << 8) | (ulint)(b[2]) @@ -178,7 +176,6 @@ mach_read_from_4( /*=============*/ const byte* b) /*!< in: pointer to four bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 24) | ((ulint)(b[1]) << 16) | ((ulint)(b[2]) << 8) @@ -255,8 +252,6 @@ mach_read_compressed( { ulint flag; - ut_ad(b); - flag = mach_read_from_1(b); if (flag < 0x80UL) { @@ -333,8 +328,6 @@ mach_read_from_7( /*=============*/ const byte* b) /*!< in: pointer to 7 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_3(b), mach_read_from_4(b + 3))); } @@ -364,8 +357,6 @@ mach_read_from_6( /*=============*/ const byte* b) /*!< in: pointer to 6 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_2(b), mach_read_from_4(b + 2))); } @@ -413,8 +404,6 @@ mach_ull_read_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - n = (ib_uint64_t) mach_read_compressed(b); size = mach_get_compressed_size((ulint) n); @@ -480,8 +469,6 @@ mach_ull_read_much_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - if (*b != (byte)0xFF) { n = 0; size = 0; diff --git a/storage/xtradb/include/mtr0mtr.ic b/storage/xtradb/include/mtr0mtr.ic index 7b5d268b70f..bebe202e6ac 100644 --- a/storage/xtradb/include/mtr0mtr.ic +++ b/storage/xtradb/include/mtr0mtr.ic @@ -170,7 +170,6 @@ mtr_memo_contains( dyn_array_t* memo; ulint offset; - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE || mtr->state == MTR_COMMITTING); diff --git a/storage/xtradb/include/page0page.ic b/storage/xtradb/include/page0page.ic index 4fe93345ce5..da9fbc5f01d 100644 --- a/storage/xtradb/include/page0page.ic +++ b/storage/xtradb/include/page0page.ic @@ -154,7 +154,6 @@ page_header_get_offs( { ulint offs; - ut_ad(page); ut_ad((field == PAGE_FREE) || (field == PAGE_LAST_INSERT) || (field == PAGE_HEAP_TOP)); diff --git a/storage/xtradb/include/rem0rec.ic b/storage/xtradb/include/rem0rec.ic index b14366312e0..b99d076b500 100644 --- a/storage/xtradb/include/rem0rec.ic +++ b/storage/xtradb/include/rem0rec.ic @@ -1545,7 +1545,6 @@ rec_copy( ulint extra_len; ulint data_len; - ut_ad(rec && buf); ut_ad(rec_offs_validate((rec_t*) rec, NULL, offsets)); ut_ad(rec_validate(rec, offsets)); diff --git a/storage/xtradb/include/ut0lst.h b/storage/xtradb/include/ut0lst.h index 9bb4bc7723f..2b5f5c7a705 100644 --- a/storage/xtradb/include/ut0lst.h +++ b/storage/xtradb/include/ut0lst.h @@ -88,7 +88,6 @@ Adds the node as the first element in a two-way linked list. */ #define UT_LIST_ADD_FIRST(NAME, BASE, N)\ {\ - ut_ad(N);\ ((BASE).count)++;\ ((N)->NAME).next = (BASE).start;\ ((N)->NAME).prev = NULL;\ @@ -134,7 +133,6 @@ Inserts a NODE2 after NODE1 in a list. #define UT_LIST_INSERT_AFTER(NAME, BASE, NODE1, NODE2)\ {\ ut_ad(NODE1);\ - ut_ad(NODE2);\ ut_ad((NODE1) != (NODE2));\ ((BASE).count)++;\ ((NODE2)->NAME).prev = (NODE1);\ @@ -169,7 +167,6 @@ Removes a node from a two-way linked list. */ #define UT_LIST_REMOVE(NAME, BASE, N) \ do { \ - ut_ad(N); \ ut_a((BASE).count > 0); \ ((BASE).count)--; \ if (((N)->NAME).next != NULL) { \ diff --git a/storage/xtradb/mtr/mtr0mtr.c b/storage/xtradb/mtr/mtr0mtr.c index 083692efd77..d6aed7ee723 100644 --- a/storage/xtradb/mtr/mtr0mtr.c +++ b/storage/xtradb/mtr/mtr0mtr.c @@ -289,7 +289,6 @@ mtr_commit( /*=======*/ mtr_t* mtr) /*!< in: mini-transaction */ { - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE); ut_ad(!mtr->inside_ibuf); diff --git a/storage/xtradb/page/page0page.c b/storage/xtradb/page/page0page.c index f2ce6c9fe16..2dd5a4ba472 100644 --- a/storage/xtradb/page/page0page.c +++ b/storage/xtradb/page/page0page.c @@ -1345,7 +1345,6 @@ page_dir_split_slot( ulint i; ulint n_owned; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); @@ -1407,7 +1406,6 @@ page_dir_balance_slot( rec_t* old_rec; rec_t* new_rec; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); diff --git a/storage/xtradb/page/page0zip.c b/storage/xtradb/page/page0zip.c index 3d3605d6362..7482323c9b5 100644 --- a/storage/xtradb/page/page0zip.c +++ b/storage/xtradb/page/page0zip.c @@ -4743,7 +4743,6 @@ page_zip_parse_compress( ulint size; ulint trailer_size; - ut_ad(ptr && end_ptr); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(ptr + (2 + 2) > end_ptr)) { diff --git a/storage/xtradb/rem/rem0rec.c b/storage/xtradb/rem/rem0rec.c index a0e289e2163..cc181c7d333 100644 --- a/storage/xtradb/rem/rem0rec.c +++ b/storage/xtradb/rem/rem0rec.c @@ -794,8 +794,7 @@ rec_get_converted_size_comp_prefix_low( ulint extra_size; ulint data_size; ulint i; - ut_ad(index); - ut_ad(fields); + ut_ad(n_fields > 0); ut_ad(n_fields <= dict_index_get_n_fields(index)); ut_ad(!temp || extra); diff --git a/storage/xtradb/row/row0merge.c b/storage/xtradb/row/row0merge.c index 3a03c7c8578..7409d5a5e1a 100644 --- a/storage/xtradb/row/row0merge.c +++ b/storage/xtradb/row/row0merge.c @@ -823,14 +823,8 @@ row_merge_read_rec( ulint data_size; ulint avail_size; - ut_ad(block); - ut_ad(buf); ut_ad(b >= block[0]); ut_ad(b < block[1]); - ut_ad(index); - ut_ad(foffs); - ut_ad(mrec); - ut_ad(offsets); ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); @@ -1202,12 +1196,6 @@ row_merge_read_clustered_index( trx->op_info = "reading clustered index"; - ut_ad(trx); - ut_ad(old_table); - ut_ad(new_table); - ut_ad(index); - ut_ad(files); - /* Create and initialize memory for record buffers */ merge_buf = mem_alloc(n_index * sizeof *merge_buf); diff --git a/storage/xtradb/row/row0purge.c b/storage/xtradb/row/row0purge.c index 1e87016bc5e..8ada79a2bbc 100644 --- a/storage/xtradb/row/row0purge.c +++ b/storage/xtradb/row/row0purge.c @@ -751,9 +751,6 @@ row_purge( { ibool updated_extern; - ut_ad(node); - ut_ad(thr); - node->undo_rec = trx_purge_fetch_next_rec(&node->roll_ptr, &node->reservation, node->heap); diff --git a/storage/xtradb/row/row0upd.c b/storage/xtradb/row/row0upd.c index 33a7e45765e..191d3e5815a 100644 --- a/storage/xtradb/row/row0upd.c +++ b/storage/xtradb/row/row0upd.c @@ -1253,8 +1253,6 @@ row_upd_index_replace_new_col_vals_index_pos( ulint n_fields; const ulint zip_size = dict_table_zip_size(index->table); - ut_ad(index); - dtuple_set_info_bits(entry, update->info_bits); if (order_only) { @@ -1437,9 +1435,6 @@ row_upd_changes_ord_field_binary_func( ulint i; const dict_index_t* clust_index; - ut_ad(index); - ut_ad(update); - ut_ad(thr); ut_ad(thr->graph); ut_ad(thr->graph->trx); diff --git a/storage/xtradb/trx/trx0trx.c b/storage/xtradb/trx/trx0trx.c index d6ce4767b39..fbdf11f378d 100644 --- a/storage/xtradb/trx/trx0trx.c +++ b/storage/xtradb/trx/trx0trx.c @@ -218,7 +218,6 @@ trx_create( trx_t* trx; ut_ad(mutex_own(&kernel_mutex)); - ut_ad(sess); trx = mem_alloc(sizeof(trx_t)); From b8afa783bcd32c9a0e5b26d780f4debf859f1b8f Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 18 Apr 2017 11:50:43 +0530 Subject: [PATCH 012/203] MW-267 Fix Galera crash at startup when compiled with gcc 6 Signed-off-by: Sachin Setiya --- sql/wsrep_mysqld.cc | 2 +- sql/wsrep_mysqld.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 21c7e94096d..a0938bdfb43 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -83,7 +83,7 @@ my_bool wsrep_creating_startup_threads = 0; */ my_bool wsrep_inited = 0; // initialized ? -static const wsrep_uuid_t cluster_uuid = WSREP_UUID_UNDEFINED; +static wsrep_uuid_t cluster_uuid = WSREP_UUID_UNDEFINED; const wsrep_uuid_t* wsrep_cluster_uuid() { return &cluster_uuid; diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 85d4cea241f..bd45399a948 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -314,7 +314,6 @@ int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len); int wsrep_create_event_query(THD *thd, uchar** buf, size_t* buf_len); int wsrep_alter_event_query(THD *thd, uchar** buf, size_t* buf_len); -const wsrep_uuid_t* wsrep_cluster_uuid(); struct xid_t; void wsrep_set_SE_checkpoint(xid_t*); void wsrep_get_SE_checkpoint(wsrep_uuid_t&, wsrep_seqno_t&); From 1d4cc423886201d9d373e17bb29d667ab8a5f503 Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko Date: Fri, 12 Aug 2016 14:21:01 +0300 Subject: [PATCH 013/203] MW-267: followup to the original pull request, removed unnecessary cast. --- sql/wsrep_mysqld.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index a0938bdfb43..53d53c2c404 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -249,8 +249,7 @@ wsrep_view_handler_cb (void* app_ctx, if (memcmp(&cluster_uuid, &view->state_id.uuid, sizeof(wsrep_uuid_t))) { - memcpy((wsrep_uuid_t*)&cluster_uuid, &view->state_id.uuid, - sizeof(cluster_uuid)); + memcpy(&cluster_uuid, &view->state_id.uuid, sizeof(cluster_uuid)); wsrep_uuid_print (&cluster_uuid, cluster_uuid_str, sizeof(cluster_uuid_str)); From 33aaee8ee997dc76a9148d96ebe0d40b2830ddd9 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 18 May 2016 11:07:58 +0200 Subject: [PATCH 014/203] MW-175 Fix definitively lost memory in wsrep_get_params --- sql/wsrep_var.cc | 5 ++--- wsrep/wsrep_dummy.c | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 66b33e7697c..6385fa926a2 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -185,9 +185,8 @@ static bool refresh_provider_options() char* opts= wsrep->options_get(wsrep); if (opts) { - if (wsrep_provider_options) my_free((void *)wsrep_provider_options); - wsrep_provider_options = (char*)my_memdup(opts, strlen(opts) + 1, - MYF(MY_WME)); + wsrep_provider_options_init(opts); + free(opts); } else { diff --git a/wsrep/wsrep_dummy.c b/wsrep/wsrep_dummy.c index bab5329dc02..5f1ea63cc40 100644 --- a/wsrep/wsrep_dummy.c +++ b/wsrep/wsrep_dummy.c @@ -86,7 +86,7 @@ static wsrep_status_t dummy_options_set( static char* dummy_options_get (wsrep_t* w) { WSREP_DBUG_ENTER(w); - return WSREP_DUMMY(w)->options; + return strdup(WSREP_DUMMY(w)->options); } static wsrep_status_t dummy_connect( From d1313d605e34a0e805022b1fd207d8df849eee4c Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 18 Apr 2017 11:53:03 +0530 Subject: [PATCH 015/203] GCF-837 Check wsrep interface version before loading provider Signed-off-by: Sachin Setiya --- wsrep/wsrep_loader.c | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/wsrep/wsrep_loader.c b/wsrep/wsrep_loader.c index 0825d7e9ecf..5f98b0ace6a 100644 --- a/wsrep/wsrep_loader.c +++ b/wsrep/wsrep_loader.c @@ -37,6 +37,22 @@ static wsrep_log_cb_t logger = default_logger; * Library loader **************************************************************************/ +static int wsrep_check_iface_version(const char* found, const char* iface_ver) +{ + const size_t msg_len = 128; + char msg[128]; + + if (strcmp(found, iface_ver)) { + snprintf (msg, msg_len, + "provider interface version mismatch: need '%s', found '%s'", + iface_ver, found); + logger (WSREP_LOG_ERROR, msg); + return EINVAL; + } + + return 0; +} + static int verify(const wsrep_t *wh, const char *iface_ver) { char msg[128]; @@ -50,13 +66,8 @@ static int verify(const wsrep_t *wh, const char *iface_ver) VERIFY(wh); VERIFY(wh->version); - if (strcmp(wh->version, iface_ver)) { - snprintf (msg, sizeof(msg), - "provider interface version mismatch: need '%s', found '%s'", - iface_ver, wh->version); - logger (WSREP_LOG_ERROR, msg); + if (wsrep_check_iface_version(wh->version, iface_ver)) return EINVAL; - } VERIFY(wh->init); VERIFY(wh->options_set); @@ -107,6 +118,15 @@ static wsrep_loader_fun wsrep_dlf(void *dlh, const char *sym) return alias.dlfun; } +static int wsrep_check_version_symbol(void *dlh) +{ + char** dlversion = NULL; + dlversion = (char**) dlsym(dlh, "wsrep_interface_version"); + if (dlversion == NULL) + return 0; + return wsrep_check_iface_version(*dlversion, WSREP_INTERFACE_VERSION); +} + extern int wsrep_dummy_loader(wsrep_t *w); int wsrep_load(const char *spec, wsrep_t **hptr, wsrep_log_cb_t log_cb) @@ -151,6 +171,11 @@ int wsrep_load(const char *spec, wsrep_t **hptr, wsrep_log_cb_t log_cb) goto out; } + if (wsrep_check_version_symbol(dlh) != 0) { + ret = EINVAL; + goto out; + } + if ((ret = (*dlfun)(*hptr)) != 0) { snprintf(msg, sizeof(msg), "wsrep_load(): loader failed: %s", strerror(ret)); From fce9a0c46ab9a8a95345a8d661d2b047e64ad7d6 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 18 Apr 2017 11:55:04 +0530 Subject: [PATCH 016/203] Bump WSREP_PATCH_VERSION to 20 Signed-off-by: Sachin Setiya --- cmake/wsrep.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index 6d92537e40f..b9e79539ac6 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -18,7 +18,7 @@ # so WSREP_VERSION is produced regardless # Set the patch version -SET(WSREP_PATCH_VERSION "14") +SET(WSREP_PATCH_VERSION "20") # MariaDB addition: Revision number of the last revision merged from # codership branch visible in @@visible_comment. From 1d821bad8cc4a64a27092fa1f476fef6a5deea21 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 18 Apr 2017 14:00:21 +0530 Subject: [PATCH 017/203] Fix sys_vars.wsrep_provider_options_basic test failure. --- wsrep/wsrep_dummy.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wsrep/wsrep_dummy.c b/wsrep/wsrep_dummy.c index 5f1ea63cc40..7c3e3d7f9e2 100644 --- a/wsrep/wsrep_dummy.c +++ b/wsrep/wsrep_dummy.c @@ -86,7 +86,8 @@ static wsrep_status_t dummy_options_set( static char* dummy_options_get (wsrep_t* w) { WSREP_DBUG_ENTER(w); - return strdup(WSREP_DUMMY(w)->options); + char * opt= WSREP_DUMMY(w)->options; + return opt ? strdup(opt) : NULL; } static wsrep_status_t dummy_connect( From b0bae66b8ef95844b5d94a72c35d2a5fe6460275 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Thu, 20 Apr 2017 18:33:20 +0530 Subject: [PATCH 018/203] FIX test failures --- mysql-test/suite/galera/disabled.def | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 mysql-test/suite/galera/disabled.def diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def new file mode 100644 index 00000000000..c91654c2023 --- /dev/null +++ b/mysql-test/suite/galera/disabled.def @@ -0,0 +1,2 @@ +galera_var_dirty_reads : MDEV-12539 +query_cache : MDEV-12539 From 85f53e46f049725431ded4718abae07d4d779e66 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Fri, 21 Apr 2017 09:27:23 +0530 Subject: [PATCH 019/203] Fix BuildBot Failure. In some slave like p8-xenial-bintar-debug we are using C90 standard which does not allow declaration and code to be mixed. --- wsrep/wsrep_dummy.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wsrep/wsrep_dummy.c b/wsrep/wsrep_dummy.c index 7c3e3d7f9e2..ff9b0b0ea92 100644 --- a/wsrep/wsrep_dummy.c +++ b/wsrep/wsrep_dummy.c @@ -85,8 +85,9 @@ static wsrep_status_t dummy_options_set( static char* dummy_options_get (wsrep_t* w) { + char * opt; WSREP_DBUG_ENTER(w); - char * opt= WSREP_DUMMY(w)->options; + opt= WSREP_DUMMY(w)->options; return opt ? strdup(opt) : NULL; } From 2cd585ad34a64b250a17e29301ffd72d8ddb4d9c Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Sat, 22 Apr 2017 11:19:48 -0400 Subject: [PATCH 020/203] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bc0b19220b6..ade2e129c54 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=55 +MYSQL_VERSION_PATCH=56 MYSQL_VERSION_EXTRA= From 92316dcdeb01c1da208f92445d68411ab3ecefa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Dor=C5=88=C3=A1k?= Date: Sun, 30 Apr 2017 14:40:38 +1000 Subject: [PATCH 021/203] wsrep.cnf - wsrep_on=1 by default From: https://github.com/devexp-db/mariadb/blob/f27/mariadb-galera.cnf.patch --- support-files/wsrep.cnf.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/support-files/wsrep.cnf.sh b/support-files/wsrep.cnf.sh index a5390855ca1..51ce3dca2dd 100644 --- a/support-files/wsrep.cnf.sh +++ b/support-files/wsrep.cnf.sh @@ -30,6 +30,9 @@ bind-address=0.0.0.0 ## WSREP options ## +# Enable wsrep +wsrep_on=1 + # Full path to wsrep provider library or 'none' wsrep_provider=none From 4f1a3dd115ec2050265f0a4cf0e8f3a3791c3e96 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 3 May 2017 11:11:33 +0530 Subject: [PATCH 022/203] Add tokudb_bugs test to disabled. --- storage/tokudb/mysql-test/tokudb_bugs/disabled.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/tokudb/mysql-test/tokudb_bugs/disabled.def b/storage/tokudb/mysql-test/tokudb_bugs/disabled.def index b6fa575522b..81f9c69f9bc 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/disabled.def +++ b/storage/tokudb/mysql-test/tokudb_bugs/disabled.def @@ -12,4 +12,4 @@ xa-2: Not for 5.5 6053: N/A checkpoint_lock_2: test can not work when the checkpoint_safe_lock is a fair rwlock - +xa-3 : failed in 5.5.56-galera From 9df17790f2b0199864443d8aa99e30cb10e71b20 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 3 May 2017 13:10:17 -0400 Subject: [PATCH 023/203] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ade2e129c54..68358acf754 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=56 +MYSQL_VERSION_PATCH=57 MYSQL_VERSION_EXTRA= From e8a2a751212a04e835b2b03408132ecbbab52410 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 20 Jun 2017 14:29:25 +0530 Subject: [PATCH 024/203] MDEV-11036 Add link wsrep_sst_rsync_wan -> wsrep_sst_rsync Currently galera has capability of using delta transfer algorithm (rsync) for SST. But for using delta transfer we have change/copy wsrep_sst_rsync wsrep_sst_rsync_wan. This patch creates a symbolic link of wsrep_sst_rsync_wan to wsrep_sst_rsync. --- .gitignore | 1 + .../Ubuntu/mariadb-galera-server-5.5.files.in | 1 + scripts/CMakeLists.txt | 16 ++++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/.gitignore b/.gitignore index 4e3d2a1e66e..b00f7dd5643 100644 --- a/.gitignore +++ b/.gitignore @@ -116,6 +116,7 @@ scripts/mytop scripts/wsrep_sst_common scripts/wsrep_sst_mysqldump scripts/wsrep_sst_rsync +scripts/wsrep_sst_rsync_wan scripts/wsrep_sst_xtrabackup scripts/wsrep_sst_xtrabackup-v2 sql-bench/bench-count-distinct diff --git a/debian/dist/Ubuntu/mariadb-galera-server-5.5.files.in b/debian/dist/Ubuntu/mariadb-galera-server-5.5.files.in index f921759fa84..ccfc7350ed0 100644 --- a/debian/dist/Ubuntu/mariadb-galera-server-5.5.files.in +++ b/debian/dist/Ubuntu/mariadb-galera-server-5.5.files.in @@ -44,6 +44,7 @@ usr/bin/resolveip usr/bin/wsrep_sst_common usr/bin/wsrep_sst_mysqldump usr/bin/wsrep_sst_rsync +usr/bin/wsrep_sst_rsync_wan usr/bin/wsrep_sst_xtrabackup usr/bin/wsrep_sst_xtrabackup-v2 usr/share/doc/mariadb-galera-server-5.5/mysqld.sym.gz diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index eacb516d618..da4f0d9b3bb 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -374,6 +374,22 @@ ELSE() COMPONENT ${${file}_COMPONENT} ) ENDFOREACH() + SET (wsrep_sst_rsync_wan ${CMAKE_CURRENT_BINARY_DIR}/wsrep_sst_rsync_wan) + ADD_CUSTOM_COMMAND( + OUTPUT ${wsrep_sst_rsync_wan} + COMMAND ${CMAKE_COMMAND} ARGS -E create_symlink + wsrep_sst_rsync + wsrep_sst_rsync_wan + ) + ADD_CUSTOM_TARGET(symlink_wsrep_sst_rsync + ALL + DEPENDS ${wsrep_sst_rsync_wan} + ) + INSTALL( + FILES ${wsrep_sst_rsync_wan} + DESTINATION ${INSTALL_BINDIR} + COMPONENT Server + ) ENDIF() # Install libgcc as mylibgcc.a From a4bc8db216b4dd61ca0b1cb5a8b7806437416dc7 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Thu, 27 Apr 2017 19:51:18 +0300 Subject: [PATCH 025/203] MW-322 - CTAS fix merged to 5.5-v25 branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Lindström --- sql/handler.cc | 5 +++ sql/sql_base.cc | 13 +++++- sql/sql_insert.cc | 29 +++++++++++++ sql/wsrep_mysqld.cc | 24 +++-------- sql/wsrep_mysqld.h | 11 +++++ storage/innobase/handler/ha_innodb.cc | 62 +++++++++++++++------------ 6 files changed, 99 insertions(+), 45 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index 5917ba505a3..5cff15821c7 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -5426,6 +5426,11 @@ void ha_wsrep_fake_trx_id(THD *thd) DBUG_VOID_RETURN; } + if (thd->wsrep_ws_handle.trx_id != WSREP_UNDEFINED_TRX_ID) + { + WSREP_DEBUG("fake trx id skipped: %lu", thd->wsrep_ws_handle.trx_id); + DBUG_VOID_RETURN; + } handlerton *hton= installed_htons[DB_TYPE_INNODB]; if (hton && hton->wsrep_fake_trx_id) { diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 345c7acb508..3832a4ab471 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -9506,11 +9506,22 @@ void tdc_remove_table(THD *thd, enum_tdc_remove_table_type remove_type, I_P_List_iterator it(share->free_tables); #ifndef DBUG_OFF if (remove_type == TDC_RT_REMOVE_ALL) +#ifdef WITH_WSREP { - DBUG_ASSERT(share->used_tables.is_empty()); + /* following assert may cause false posivive fire for CTAS */ + if (thd->lex->sql_command != SQLCOM_CREATE_TABLE) + { +#endif /* WITH_WSREP */ + { + DBUG_ASSERT(share->used_tables.is_empty()); + } +#ifdef WITH_WSREP + } } else if (remove_type == TDC_RT_REMOVE_NOT_OWN || remove_type == TDC_RT_REMOVE_NOT_OWN_AND_MARK_NOT_USABLE) +#endif /* WITH_WSREP */ + { I_P_List_iterator it2(share->used_tables); while ((table= it2++)) diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 4f0129c6e70..e8f00b76d5a 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -4313,9 +4313,38 @@ bool select_create::send_eof() */ if (!table->s->tmp_table) { + #ifdef WITH_WSREP + /* + append table level exclusive key for CTAS + */ + wsrep_key_arr_t key_arr= {0, 0}; + wsrep_prepare_keys_for_isolation(thd, + create_table->db, + create_table->table_name, + table_list, + &key_arr); + int rcode = wsrep->append_key( + wsrep, + &thd->wsrep_ws_handle, + key_arr.keys, //&wkey, + key_arr.keys_len, + WSREP_KEY_EXCLUSIVE, + false); + wsrep_keys_free(&key_arr); + if (rcode) { + DBUG_PRINT("wsrep", ("row key failed: %d", rcode)); + WSREP_ERROR("Appending table key for CTAS failed: %s, %d", + (wsrep_thd_query(thd)) ? + wsrep_thd_query(thd) : "void", rcode); + return true; + } + /* If commit fails, we should be able to reset the OK status. */ + thd->stmt_da->can_overwrite_status= TRUE; +#endif /* WITH_WSREP */ trans_commit_stmt(thd); trans_commit_implicit(thd); #ifdef WITH_WSREP + thd->stmt_da->can_overwrite_status= FALSE; mysql_mutex_lock(&thd->LOCK_wsrep_thd); if (thd->wsrep_conflict_state != NO_CONFLICT) { diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 53d53c2c404..4ac346ca0a3 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -944,17 +944,7 @@ bool wsrep_sync_wait (THD* thd, uint mask) return false; } -/* - * Helpers to deal with TOI key arrays - */ -typedef struct wsrep_key_arr -{ - wsrep_key_t* keys; - size_t keys_len; -} wsrep_key_arr_t; - - -static void wsrep_keys_free(wsrep_key_arr_t* key_arr) +void wsrep_keys_free(wsrep_key_arr_t* key_arr) { for (size_t i= 0; i < key_arr->keys_len; ++i) { @@ -1019,11 +1009,11 @@ static bool wsrep_prepare_key_for_isolation(const char* db, } /* Prepare key list from db/table and table_list */ -static bool wsrep_prepare_keys_for_isolation(THD* thd, - const char* db, - const char* table, - const TABLE_LIST* table_list, - wsrep_key_arr_t* ka) +bool wsrep_prepare_keys_for_isolation(THD* thd, + const char* db, + const char* table, + const TABLE_LIST* table_list, + wsrep_key_arr_t* ka) { ka->keys= 0; ka->keys_len= 0; @@ -1554,7 +1544,7 @@ wsrep_grant_mdl_exception(MDL_context *requestor_ctx, } else if (request_thd->lex->sql_command == SQLCOM_DROP_TABLE) { - WSREP_DEBUG("DROP caused BF abort"); + WSREP_DEBUG("DROP caused BF abort, conf %d", granted_thd->wsrep_conflict_state); ticket->wsrep_report(wsrep_debug); mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd); wsrep_abort_thd((void*)request_thd, (void*)granted_thd, 1); diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index bd45399a948..23e9718e533 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -322,4 +322,15 @@ const wsrep_uuid_t* wsrep_xid_uuid(const xid_t*); wsrep_seqno_t wsrep_xid_seqno(const xid_t*); extern "C" int wsrep_is_wsrep_xid(const void* xid); +typedef struct wsrep_key_arr +{ + wsrep_key_t* keys; + size_t keys_len; +} wsrep_key_arr_t; +bool wsrep_prepare_keys_for_isolation(THD* thd, + const char* db, + const char* table, + const TABLE_LIST* table_list, + wsrep_key_arr_t* ka); +void wsrep_keys_free(wsrep_key_arr_t* key_arr); #endif /* WSREP_MYSQLD_H */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5261a50fdd6..ad31598384c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6000,10 +6000,14 @@ report_error: prebuilt->table->flags, user_thd); #ifdef WITH_WSREP - if (!error_result && - wsrep_thd_exec_mode(user_thd) == LOCAL_STATE && - wsrep_on(user_thd) && - !wsrep_consistency_check(user_thd)) + if (!error_result + && wsrep_on(user_thd) + && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE + && !wsrep_consistency_check(user_thd)) + && (sql_command != SQLCOM_CREATE_TABLE) + && (sql_command != SQLCOM_LOAD || + thd_binlog_format(user_thd) == BINLOG_FORMAT_ROW)) { + { if (wsrep_append_keys(user_thd, false, record, NULL)) { @@ -12788,8 +12792,10 @@ wsrep_innobase_kill_one_trx( wsrep_thd_thread_id(thd), victim_trx->id); - WSREP_DEBUG("Aborting query: %s", - (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void"); + WSREP_DEBUG("Aborting query: %s conf %d trx: %lu", + (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void", + wsrep_thd_conflict_state(thd), + wsrep_thd_ws_handle(thd)->trx_id); wsrep_thd_LOCK(thd); @@ -12841,9 +12847,8 @@ wsrep_innobase_kill_one_trx( } else { rcode = wsrep->abort_pre_commit( wsrep, bf_seqno, - (wsrep_trx_id_t)victim_trx->id + (wsrep_trx_id_t)wsrep_thd_ws_handle(thd)->trx_id ); - switch (rcode) { case WSREP_WARNING: WSREP_DEBUG("cancel commit warning: %llu", @@ -12973,15 +12978,17 @@ wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd, my_bool signal) { DBUG_ENTER("wsrep_innobase_abort_thd"); - trx_t* victim_trx = thd_to_trx(victim_thd); - trx_t* bf_trx = (bf_thd) ? thd_to_trx(bf_thd) : NULL; + + trx_t* victim_trx = thd_to_trx(victim_thd); + trx_t* bf_trx = (bf_thd) ? thd_to_trx(bf_thd) : NULL; + + WSREP_DEBUG("abort transaction: BF: %s victim: %s victim conf: %d", + wsrep_thd_query(bf_thd), + wsrep_thd_query(victim_thd), + wsrep_thd_conflict_state(victim_thd)); ut_ad(!mutex_own(&kernel_mutex)); - WSREP_DEBUG("abort transaction: BF: %s victim: %s", - wsrep_thd_query(bf_thd), - wsrep_thd_query(victim_thd)); - if (victim_trx) { int rcode = wsrep_innobase_kill_one_trx( @@ -13001,24 +13008,24 @@ wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd, static int innobase_wsrep_set_checkpoint(handlerton* hton, const XID* xid) { DBUG_ASSERT(hton == innodb_hton_ptr); - if (wsrep_is_wsrep_xid(xid)) { - mtr_t mtr; - mtr_start(&mtr); - trx_sysf_t* sys_header = trx_sysf_get(&mtr); - trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr); - mtr_commit(&mtr); - innobase_flush_logs(hton); - return 0; - } else { - return 1; - } + if (wsrep_is_wsrep_xid(xid)) { + mtr_t mtr; + mtr_start(&mtr); + trx_sysf_t* sys_header = trx_sysf_get(&mtr); + trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr); + mtr_commit(&mtr); + innobase_flush_logs(hton); + return 0; + } else { + return 1; + } } static int innobase_wsrep_get_checkpoint(handlerton* hton, XID* xid) { DBUG_ASSERT(hton == innodb_hton_ptr); - trx_sys_read_wsrep_checkpoint(xid); - return 0; + trx_sys_read_wsrep_checkpoint(xid); + return 0; } static void @@ -13030,6 +13037,7 @@ wsrep_fake_trx_id( mutex_enter(&kernel_mutex); trx_id_t trx_id = trx_sys_get_new_trx_id(); mutex_exit(&kernel_mutex); + WSREP_DEBUG("innodb fake trx id: %lu thd: %s", trx_id, wsrep_thd_query(thd)); (void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); } From 48b7245bf285b757aa46bd37c86b9c202705e3bb Mon Sep 17 00:00:00 2001 From: sjaakola Date: Thu, 27 Apr 2017 20:28:22 +0300 Subject: [PATCH 026/203] MW-369 - merged fix for FK issue from 5.6-v25 branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Lindström --- storage/innobase/row/row0ins.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/storage/innobase/row/row0ins.c b/storage/innobase/row/row0ins.c index c380390d62a..3726c517ff0 100644 --- a/storage/innobase/row/row0ins.c +++ b/storage/innobase/row/row0ins.c @@ -1082,11 +1082,12 @@ row_ins_foreign_check_on_constraint( #ifdef WITH_WSREP err = wsrep_append_foreign_key( - thr_get_trx(thr), - foreign, - clust_rec, - clust_index, - FALSE, FALSE); + thr_get_trx(thr), + foreign, + clust_rec, + clust_index, + FALSE, + (node) ? TRUE : FALSE); if (err != DB_SUCCESS) { fprintf(stderr, "WSREP: foreign key append failed: %lu\n", err); @@ -1247,6 +1248,9 @@ row_ins_check_foreign_constraint( ulint* offsets = offsets_; rec_offs_init(offsets_); +#ifdef WITH_WSREP + upd_node= NULL; +#endif /* WITH_WSREP */ run_again: #ifdef UNIV_SYNC_DEBUG ut_ad(rw_lock_own(&dict_operation_lock, RW_LOCK_SHARED)); @@ -1429,9 +1433,10 @@ run_again: err = wsrep_append_foreign_key( thr_get_trx(thr), foreign, - rec, - check_index, - check_ref, TRUE); + rec, + check_index, + check_ref, + (upd_node) ? TRUE : FALSE); #endif /* WITH_WSREP */ goto end_scan; } else if (foreign->type != 0) { From dbda504275af5eded33f62f509bff0b46e1560e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 20 Jul 2017 13:52:22 +0300 Subject: [PATCH 027/203] Fix merge error and compiler warning. --- storage/innobase/handler/ha_innodb.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index ad31598384c..2f1e3b41c47 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6003,14 +6003,12 @@ report_error: if (!error_result && wsrep_on(user_thd) && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE - && !wsrep_consistency_check(user_thd)) + && !wsrep_consistency_check(user_thd) && (sql_command != SQLCOM_CREATE_TABLE) && (sql_command != SQLCOM_LOAD || thd_binlog_format(user_thd) == BINLOG_FORMAT_ROW)) { - { - if (wsrep_append_keys(user_thd, false, record, NULL)) - { + if (wsrep_append_keys(user_thd, false, record, NULL)) { DBUG_PRINT("wsrep", ("row key failed")); error_result = HA_ERR_INTERNAL_ERROR; goto wsrep_error; @@ -13028,16 +13026,15 @@ static int innobase_wsrep_get_checkpoint(handlerton* hton, XID* xid) return 0; } -static void -wsrep_fake_trx_id( -/*==================*/ +static void wsrep_fake_trx_id( handlerton *hton, THD *thd) /*!< in: user thread handle */ { mutex_enter(&kernel_mutex); trx_id_t trx_id = trx_sys_get_new_trx_id(); mutex_exit(&kernel_mutex); - WSREP_DEBUG("innodb fake trx id: %lu thd: %s", trx_id, wsrep_thd_query(thd)); + WSREP_DEBUG("innodb fake trx id: %llu thd: %s", + trx_id, wsrep_thd_query(thd)); (void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); } From 20ab1665af07080441ecea0734eb78b3aa95e58e Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Fri, 5 May 2017 10:55:45 +0300 Subject: [PATCH 028/203] MW-322 Fix compilation error with debug build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Lindström --- storage/innobase/handler/ha_innodb.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 2f1e3b41c47..97c6cf84910 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -13033,8 +13033,7 @@ static void wsrep_fake_trx_id( mutex_enter(&kernel_mutex); trx_id_t trx_id = trx_sys_get_new_trx_id(); mutex_exit(&kernel_mutex); - WSREP_DEBUG("innodb fake trx id: %llu thd: %s", - trx_id, wsrep_thd_query(thd)); + WSREP_DEBUG("innodb fake trx id: %llu thd: %s", trx_id, wsrep_thd_query(thd)); (void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); } From 6326f0eac6f5a7b8132676af1b37bd7ec453009e Mon Sep 17 00:00:00 2001 From: sjaakola Date: Fri, 5 May 2017 11:09:01 +0300 Subject: [PATCH 029/203] MW-322 CTAS fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pushed fix for a typo Signed-off-by: Jan Lindström --- sql/sql_insert.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index e8f00b76d5a..09e3d2c3c61 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -4313,7 +4313,7 @@ bool select_create::send_eof() */ if (!table->s->tmp_table) { - #ifdef WITH_WSREP +#ifdef WITH_WSREP /* append table level exclusive key for CTAS */ From 519e4322e1d5ae928d6939d88d0395d60f224da2 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Mon, 8 May 2017 23:12:51 +0300 Subject: [PATCH 030/203] MW-369 FK fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skipping wsrep extra FK check for applier and replayer Signed-off-by: Jan Lindström --- storage/innobase/row/row0upd.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/innobase/row/row0upd.c b/storage/innobase/row/row0upd.c index 40e9fbfde23..dd9c7e8defd 100644 --- a/storage/innobase/row/row0upd.c +++ b/storage/innobase/row/row0upd.c @@ -1822,7 +1822,9 @@ row_upd_sec_index_entry( index, offsets, thr, &mtr); } #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && + if (wsrep_on(trx->mysql_thd) && + !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && + err == DB_SUCCESS && !referenced && !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && ((upd_node_t*)parent)->cascade_node == node) && From a82611771b58fcfee024ce59c13adc2a928c5428 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 8 Mar 2017 13:08:21 +0100 Subject: [PATCH 031/203] MW-86 Add separate wsrep_sync_wait bitmask value for SHOW commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, setting `wsrep_sync_wait = 1` would have an effect on both SELECT and SHOW statements. This patch changes wsrep_sync_wait so that bitmask value 1 is used for SELECT statements, while bitmask value 8 is reserved for SHOW statements. It is still possible to achieve sync wait on both SELECT and SHOW statements by setting `wsrep_sync_wait = 9`. Signed-off-by: Jan Lindström --- sql/sql_parse.cc | 72 ++++++++++++++++------------------------------ sql/sql_parse.h | 5 ++++ sql/wsrep_mysqld.h | 5 ++-- 3 files changed, 32 insertions(+), 50 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 745ea171030..3862de0c135 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2473,9 +2473,7 @@ mysql_execute_command(THD *thd) #endif case SQLCOM_SHOW_STATUS_PROC: case SQLCOM_SHOW_STATUS_FUNC: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if ((res= check_table_access(thd, SELECT_ACL, all_tables, FALSE, UINT_MAX, FALSE))) goto error; @@ -2483,7 +2481,9 @@ mysql_execute_command(THD *thd) break; case SQLCOM_SHOW_STATUS: { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); execute_show_status(thd, all_tables); + #ifdef WITH_WSREP if (lex->sql_command == SQLCOM_SHOW_STATUS) wsrep_free_status(thd); #endif /* WITH_WSREP */ @@ -2497,27 +2497,22 @@ mysql_execute_command(THD *thd) case SQLCOM_SHOW_PLUGINS: case SQLCOM_SHOW_FIELDS: case SQLCOM_SHOW_KEYS: -#ifndef WITH_WSREP case SQLCOM_SHOW_VARIABLES: case SQLCOM_SHOW_CHARSETS: case SQLCOM_SHOW_COLLATIONS: case SQLCOM_SHOW_STORAGE_ENGINES: case SQLCOM_SHOW_PROFILE: -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); case SQLCOM_SHOW_CLIENT_STATS: case SQLCOM_SHOW_USER_STATS: case SQLCOM_SHOW_TABLE_STATS: case SQLCOM_SHOW_INDEX_STATS: case SQLCOM_SELECT: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; - case SQLCOM_SHOW_VARIABLES: - case SQLCOM_SHOW_CHARSETS: - case SQLCOM_SHOW_COLLATIONS: - case SQLCOM_SHOW_STORAGE_ENGINES: - case SQLCOM_SHOW_PROFILE: -#endif /* WITH_WSREP */ { +#ifdef WITH_WSREP + if (lex->sql_command == SQLCOM_SELECT) + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_READ); +#endif /* WITH_WSREP */ thd->status_var.last_query_cost= 0.0; /* @@ -3079,9 +3074,7 @@ end_with_restore_list: goto error; #else { -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); /* Access check: @@ -3138,9 +3131,7 @@ end_with_restore_list: case SQLCOM_CHECKSUM: { DBUG_ASSERT(first_table == all_tables && first_table != 0); -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_READ); if (check_table_access(thd, SELECT_ACL, all_tables, FALSE, UINT_MAX, FALSE)) @@ -3150,11 +3141,8 @@ end_with_restore_list: break; } case SQLCOM_UPDATE: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE)) goto error; -#endif /* WITH_WSREP */ { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE); ha_rows found= 0, updated= 0; DBUG_ASSERT(first_table == all_tables && first_table != 0); if (update_precheck(thd, all_tables)) @@ -3193,10 +3181,7 @@ end_with_restore_list: /* if we switched from normal update, rights are checked */ if (up_result != 2) { -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE); if ((res= multi_update_precheck(thd, all_tables))) break; } @@ -3266,10 +3251,7 @@ end_with_restore_list: break; } case SQLCOM_REPLACE: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); #ifndef DBUG_OFF if (mysql_bin_log.is_open()) { @@ -3305,11 +3287,8 @@ end_with_restore_list: } #endif case SQLCOM_INSERT: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE)) goto error; -#endif /* WITH_WSREP */ { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); DBUG_ASSERT(first_table == all_tables && first_table != 0); if ((res= insert_precheck(thd, all_tables))) break; @@ -3350,12 +3329,9 @@ end_with_restore_list: } case SQLCOM_REPLACE_SELECT: case SQLCOM_INSERT_SELECT: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE)) goto error; -#endif /* WITH_WSREP */ { - select_result *sel_result; + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); + select_insert *sel_result; DBUG_ASSERT(first_table == all_tables && first_table != 0); if ((res= insert_precheck(thd, all_tables))) break; @@ -3450,11 +3426,8 @@ end_with_restore_list: break; } case SQLCOM_DELETE: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE)) goto error; -#endif /* WITH_WSREP */ { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE); DBUG_ASSERT(first_table == all_tables && first_table != 0); if ((res= delete_precheck(thd, all_tables))) break; @@ -3469,11 +3442,8 @@ end_with_restore_list: break; } case SQLCOM_DELETE_MULTI: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE)) goto error; -#endif /* WITH_WSREP */ { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE); DBUG_ASSERT(first_table == all_tables && first_table != 0); TABLE_LIST *aux_tables= thd->lex->auxiliary_table_list.first; multi_delete *del_result; @@ -3840,6 +3810,7 @@ end_with_restore_list: my_error(ER_WRONG_DB_NAME, MYF(0), lex->name.str); break; } + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); res= mysqld_show_create_db(thd, lex->name.str, &lex->create_info); break; } @@ -3891,6 +3862,7 @@ end_with_restore_list: /* lex->unit.cleanup() is called outside, no need to call it here */ break; case SQLCOM_SHOW_CREATE_EVENT: + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); res= Events::show_create_event(thd, lex->spname->m_db, lex->spname->m_name); break; @@ -4751,12 +4723,14 @@ create_sp_error: } case SQLCOM_SHOW_CREATE_PROC: { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (sp_show_create_routine(thd, TYPE_ENUM_PROCEDURE, lex->spname)) goto error; break; } case SQLCOM_SHOW_CREATE_FUNC: { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (sp_show_create_routine(thd, TYPE_ENUM_FUNCTION, lex->spname)) goto error; break; @@ -4769,6 +4743,7 @@ create_sp_error: stored_procedure_type type= (lex->sql_command == SQLCOM_SHOW_PROC_CODE ? TYPE_ENUM_PROCEDURE : TYPE_ENUM_FUNCTION); + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (sp_cache_routine(thd, type, lex->spname, FALSE, &sp)) goto error; if (!sp || sp->show_routine_code(thd)) @@ -4790,6 +4765,7 @@ create_sp_error: if (check_ident_length(&lex->spname->m_name)) goto error; + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (show_create_trigger(thd, lex->spname)) goto error; /* Error has been already logged. */ diff --git a/sql/sql_parse.h b/sql/sql_parse.h index 58885594a5d..e176e4eadf8 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -218,10 +218,15 @@ inline bool is_supported_parser_charset(CHARSET_INFO *cs) if (WSREP(thd) || (thd && thd->wsrep_exec_mode==TOTAL_ORDER)) \ wsrep_to_isolation_end(thd); +#define WSREP_SYNC_WAIT(thd_, before_) \ + if (WSREP_CLIENT(thd_) && \ + wsrep_sync_wait(thd_, before_)) goto error; + #else #define WSREP_TO_ISOLATION_BEGIN(db_, table_, table_list_) #define WSREP_TO_ISOLATION_END +#define WSREP_SYNC_WAIT(thd_, before_) #endif /* WITH_WSREP */ diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 23e9718e533..bb33d2f1069 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -105,11 +105,12 @@ extern my_bool wsrep_creating_startup_threads; enum enum_wsrep_OSU_method { WSREP_OSU_TOI, WSREP_OSU_RSU }; enum enum_wsrep_sync_wait { WSREP_SYNC_WAIT_NONE = 0x0, - // show, select, begin + // select, begin WSREP_SYNC_WAIT_BEFORE_READ = 0x1, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE = 0x2, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE = 0x4, - WSREP_SYNC_WAIT_MAX = 0x7 + WSREP_SYNC_WAIT_BEFORE_SHOW = 0x8, + WSREP_SYNC_WAIT_MAX = 0xF }; // MySQL status variables From 34853fa7937b8081516eddb0226e8031f06ba96f Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Mon, 13 Mar 2017 15:35:04 +0100 Subject: [PATCH 032/203] MW-86 Additional wsrep_sync_wait coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following commands are now subject to wsrep_sync_wait with bitmask value 8: SHOW BINARY LOGS SHOW BINLOG EVENTS SHOW GRANTS Signed-off-by: Jan Lindström --- sql/sql_parse.cc | 10 +++++++--- sql/sql_parse.h | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 3862de0c135..c222adbbac2 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2502,7 +2502,6 @@ mysql_execute_command(THD *thd) case SQLCOM_SHOW_COLLATIONS: case SQLCOM_SHOW_STORAGE_ENGINES: case SQLCOM_SHOW_PROFILE: - WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); case SQLCOM_SHOW_CLIENT_STATS: case SQLCOM_SHOW_USER_STATS: case SQLCOM_SHOW_TABLE_STATS: @@ -2510,8 +2509,10 @@ mysql_execute_command(THD *thd) case SQLCOM_SELECT: { #ifdef WITH_WSREP - if (lex->sql_command == SQLCOM_SELECT) - WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_READ); + if (lex->sql_command == SQLCOM_SELECT) + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_READ) + else + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW) #endif /* WITH_WSREP */ thd->status_var.last_query_cost= 0.0; @@ -2635,6 +2636,7 @@ case SQLCOM_PREPARE: case SQLCOM_SHOW_RELAYLOG_EVENTS: /* fall through */ case SQLCOM_SHOW_BINLOG_EVENTS: { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (check_global_access(thd, REPL_SLAVE_ACL)) goto error; res = mysql_show_binlog_events(thd); @@ -3061,6 +3063,7 @@ end_with_restore_list: { if (check_global_access(thd, SUPER_ACL | REPL_CLIENT_ACL)) goto error; + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); res = show_binlogs(thd); break; } @@ -4163,6 +4166,7 @@ end_with_restore_list: !strcmp(thd->security_ctx->priv_user, grant_user->user.str)) || !check_access(thd, SELECT_ACL, "mysql", NULL, NULL, 1, 0)) { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); res = mysql_show_grants(thd, grant_user); } break; diff --git a/sql/sql_parse.h b/sql/sql_parse.h index e176e4eadf8..2093ecc85ee 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -219,8 +219,8 @@ inline bool is_supported_parser_charset(CHARSET_INFO *cs) wsrep_to_isolation_end(thd); #define WSREP_SYNC_WAIT(thd_, before_) \ - if (WSREP_CLIENT(thd_) && \ - wsrep_sync_wait(thd_, before_)) goto error; + { if (WSREP_CLIENT(thd_) && \ + wsrep_sync_wait(thd_, before_)) goto error; } #else From be416cfa3b6ec6faad3e0ac273fc8698de61094d Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Mon, 13 Mar 2017 22:45:42 +0100 Subject: [PATCH 033/203] MW-86 Removed unnecessary wsrep_sync_wait before processing SQLCOM_REPLACE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync waiting before processing SQLCOM_REPLACE was not necessary given that this case falls through to processing of SQLCOM_INSERT. In case of SQLCOM_REPLACE, wsrep_sync_wait would be called twice. Signed-off-by: Jan Lindström --- sql/sql_parse.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index c222adbbac2..52b18318849 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -3254,7 +3254,6 @@ end_with_restore_list: break; } case SQLCOM_REPLACE: - WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); #ifndef DBUG_OFF if (mysql_bin_log.is_open()) { From 970859a599714d469c15b5fd49703d22b35317ad Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Wed, 24 May 2017 14:46:25 +0300 Subject: [PATCH 034/203] MW-383 Bumped wsrep patch version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Lindström --- cmake/wsrep.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index b9e79539ac6..6a9dd720f77 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -18,7 +18,7 @@ # so WSREP_VERSION is produced regardless # Set the patch version -SET(WSREP_PATCH_VERSION "20") +SET(WSREP_PATCH_VERSION "21") # MariaDB addition: Revision number of the last revision merged from # codership branch visible in @@visible_comment. From e5c488a49ba54aee596f4ff0080f3a855461b911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 21 Jul 2017 08:29:52 +0300 Subject: [PATCH 035/203] Fix failing test case. --- mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result b/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result index 1e7b9364570..0df3dff8990 100644 --- a/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result +++ b/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result @@ -34,11 +34,9 @@ SELECT @@session.wsrep_sync_wait; @@session.wsrep_sync_wait 7 SET @@session.wsrep_sync_wait=8; -Warnings: -Warning 1292 Truncated incorrect wsrep_sync_wait value: '8' SELECT @@session.wsrep_sync_wait; @@session.wsrep_sync_wait -7 +8 # invalid values SET @@global.wsrep_sync_wait=NULL; From 07f8360f173c824df592c3ff130bfc256aba2bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Jul 2017 11:06:42 +0300 Subject: [PATCH 036/203] MDEV-10379: Failing assertion: xid_seqno > trx_sys_cur_xid_seqno Remove too strict debug assertion. --- storage/innobase/trx/trx0sys.c | 20 -------------------- storage/xtradb/trx/trx0sys.c | 20 -------------------- 2 files changed, 40 deletions(-) diff --git a/storage/innobase/trx/trx0sys.c b/storage/innobase/trx/trx0sys.c index a0fcc2b2370..f8bd87034e8 100644 --- a/storage/innobase/trx/trx0sys.c +++ b/storage/innobase/trx/trx0sys.c @@ -825,26 +825,6 @@ trx_sys_update_wsrep_checkpoint( trx_sysf_t* sys_header, /*!< in: sys_header */ mtr_t* mtr) /*!< in: mtr */ { - -#ifdef UNIV_DEBUG - { - /* Check that seqno is monotonically increasing */ - unsigned char xid_uuid[16]; - long long xid_seqno = read_wsrep_xid_seqno(xid); - read_wsrep_xid_uuid(xid, xid_uuid); - if (!memcmp(xid_uuid, trx_sys_cur_xid_uuid, 8)) - { - ut_ad(xid_seqno > trx_sys_cur_xid_seqno); - trx_sys_cur_xid_seqno = xid_seqno; - } - else - { - memcpy(trx_sys_cur_xid_uuid, xid_uuid, 16); - } - trx_sys_cur_xid_seqno = xid_seqno; - } -#endif /* UNIV_DEBUG */ - ut_ad(xid && mtr && sys_header); ut_a(xid->formatID == -1 || wsrep_is_wsrep_xid(xid)); diff --git a/storage/xtradb/trx/trx0sys.c b/storage/xtradb/trx/trx0sys.c index 7d9397b8e2c..198430f9a01 100644 --- a/storage/xtradb/trx/trx0sys.c +++ b/storage/xtradb/trx/trx0sys.c @@ -990,26 +990,6 @@ trx_sys_update_wsrep_checkpoint( trx_sysf_t* sys_header, /*!< in: sys_header */ mtr_t* mtr) /*!< in: mtr */ { - -#ifdef UNIV_DEBUG - { - /* Check that seqno is monotonically increasing */ - unsigned char xid_uuid[16]; - long long xid_seqno = read_wsrep_xid_seqno(xid); - read_wsrep_xid_uuid(xid, xid_uuid); - if (!memcmp(xid_uuid, trx_sys_cur_xid_uuid, 8)) - { - ut_ad(xid_seqno > trx_sys_cur_xid_seqno); - trx_sys_cur_xid_seqno = xid_seqno; - } - else - { - memcpy(trx_sys_cur_xid_uuid, xid_uuid, 16); - } - trx_sys_cur_xid_seqno = xid_seqno; - } -#endif /* UNIV_DEBUG */ - ut_ad(xid && mtr && sys_header); ut_a(xid->formatID == -1 || wsrep_is_wsrep_xid(xid)); From 2aaed4489f92ede9fd1b28a848a2d8434dea17f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Jul 2017 15:43:45 +0300 Subject: [PATCH 037/203] Fix regression on galera.partition test case by commenting the problematic condition. --- storage/innobase/handler/ha_innodb.cc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 97c6cf84910..094d6f76d17 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6000,13 +6000,22 @@ report_error: prebuilt->table->flags, user_thd); #ifdef WITH_WSREP + fprintf(stderr, "JAN: split %d load %d\n", wsrep_load_data_splitting, + sql_command == SQLCOM_LOAD); if (!error_result && wsrep_on(user_thd) && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE && !wsrep_consistency_check(user_thd) - && (sql_command != SQLCOM_CREATE_TABLE) - && (sql_command != SQLCOM_LOAD || - thd_binlog_format(user_thd) == BINLOG_FORMAT_ROW)) { + && (sql_command != SQLCOM_CREATE_TABLE)) { + /* This change part of commit + a4bc8db216b4dd61ca0b1cb5a8b7806437416dc7 + MW-322 - CTAS fix will cause regression on galera.partition + test case. Commented until galeracluster developers have + looked the issue. + && (sql_command != SQLCOM_LOAD || + thd_binlog_format(user_thd) == + BINLOG_FORMAT_ROW)) { + */ if (wsrep_append_keys(user_thd, false, record, NULL)) { DBUG_PRINT("wsrep", ("row key failed")); From eec6417e05f6b5f76758e619a44fbdb00860258a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Jul 2017 16:14:27 +0300 Subject: [PATCH 038/203] Apply galera patches to XtraDB storage engine and remove one debug output. --- storage/innobase/handler/ha_innodb.cc | 2 -- storage/xtradb/handler/ha_innodb.cc | 22 +++++++++++++++------- storage/xtradb/row/row0ins.c | 25 +++++++++++++++---------- storage/xtradb/row/row0upd.c | 4 +++- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 094d6f76d17..1991850e6cd 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6000,8 +6000,6 @@ report_error: prebuilt->table->flags, user_thd); #ifdef WITH_WSREP - fprintf(stderr, "JAN: split %d load %d\n", wsrep_load_data_splitting, - sql_command == SQLCOM_LOAD); if (!error_result && wsrep_on(user_thd) && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 805b28d9d21..d1f40905184 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -7025,13 +7025,21 @@ report_error: prebuilt->table->flags, user_thd); #ifdef WITH_WSREP - if (!error_result && - wsrep_thd_exec_mode(user_thd) == LOCAL_STATE && - wsrep_on(user_thd) && - !wsrep_consistency_check(user_thd)) - { - if (wsrep_append_keys(user_thd, false, record, NULL)) - { + if (!error_result + && wsrep_on(user_thd) + && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE + && !wsrep_consistency_check(user_thd) + && (sql_command != SQLCOM_CREATE_TABLE)) { + /* This change part of commit + a4bc8db216b4dd61ca0b1cb5a8b7806437416dc7 + MW-322 - CTAS fix will cause regression on galera.partition + test case. Commented until galeracluster developers have + looked the issue. + && (sql_command != SQLCOM_LOAD || + thd_binlog_format(user_thd) == + BINLOG_FORMAT_ROW)) { + */ + if (wsrep_append_keys(user_thd, false, record, NULL)) { DBUG_PRINT("wsrep", ("row key failed")); error_result = HA_ERR_INTERNAL_ERROR; goto wsrep_error; diff --git a/storage/xtradb/row/row0ins.c b/storage/xtradb/row/row0ins.c index a4309612377..52495c7ec60 100644 --- a/storage/xtradb/row/row0ins.c +++ b/storage/xtradb/row/row0ins.c @@ -761,7 +761,7 @@ row_ins_invalidate_query_cache( mem_free(buf); } #ifdef WITH_WSREP -ulint wsrep_append_foreign_key(trx_t *trx, +ulint wsrep_append_foreign_key(trx_t *trx, dict_foreign_t* foreign, const rec_t* clust_rec, dict_index_t* clust_index, @@ -1083,13 +1083,14 @@ row_ins_foreign_check_on_constraint( #ifdef WITH_WSREP err = wsrep_append_foreign_key( - thr_get_trx(thr), - foreign, - clust_rec, - clust_index, - FALSE, FALSE); + thr_get_trx(thr), + foreign, + clust_rec, + clust_index, + FALSE, + (node) ? TRUE : FALSE); if (err != DB_SUCCESS) { - fprintf(stderr, + fprintf(stderr, "WSREP: foreign key append failed: %lu\n", err); } else #endif @@ -1248,6 +1249,9 @@ row_ins_check_foreign_constraint( ulint* offsets = offsets_; rec_offs_init(offsets_); +#ifdef WITH_WSREP + upd_node= NULL; +#endif /* WITH_WSREP */ run_again: #ifdef UNIV_SYNC_DEBUG ut_ad(rw_lock_own(&dict_operation_lock, RW_LOCK_SHARED)); @@ -1436,9 +1440,10 @@ run_again: err = wsrep_append_foreign_key( thr_get_trx(thr), foreign, - rec, - check_index, - check_ref, TRUE); + rec, + check_index, + check_ref, + (upd_node) ? TRUE : FALSE); #endif /* WITH_WSREP */ goto end_scan; } else if (foreign->type != 0) { diff --git a/storage/xtradb/row/row0upd.c b/storage/xtradb/row/row0upd.c index a2eabc347d6..d75100c91e8 100644 --- a/storage/xtradb/row/row0upd.c +++ b/storage/xtradb/row/row0upd.c @@ -1842,7 +1842,9 @@ row_upd_sec_index_entry( index, offsets, thr, &mtr); } #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && + if (wsrep_on(trx->mysql_thd) && + !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && + err == DB_SUCCESS && !referenced && !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && ((upd_node_t*)parent)->cascade_node == node) && From d9675a10d5b1494458730367e1b790efc3b03bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Jul 2017 20:24:03 +0300 Subject: [PATCH 039/203] Remove unneeded code. --- storage/innobase/trx/trx0sys.c | 18 ------------------ storage/xtradb/trx/trx0sys.c | 18 ------------------ 2 files changed, 36 deletions(-) diff --git a/storage/innobase/trx/trx0sys.c b/storage/innobase/trx/trx0sys.c index f8bd87034e8..51729e05c9d 100644 --- a/storage/innobase/trx/trx0sys.c +++ b/storage/innobase/trx/trx0sys.c @@ -801,24 +801,6 @@ trx_sys_print_mysql_binlog_offset(void) #ifdef WITH_WSREP -#ifdef UNIV_DEBUG -static long long trx_sys_cur_xid_seqno = -1; -static unsigned char trx_sys_cur_xid_uuid[16]; - -long long read_wsrep_xid_seqno(const XID* xid) -{ - long long seqno; - memcpy(&seqno, xid->data + 24, sizeof(long long)); - return seqno; -} - -void read_wsrep_xid_uuid(const XID* xid, unsigned char* buf) -{ - memcpy(buf, xid->data + 8, 16); -} - -#endif /* UNIV_DEBUG */ - void trx_sys_update_wsrep_checkpoint( const XID* xid, /*!< in: transaction XID */ diff --git a/storage/xtradb/trx/trx0sys.c b/storage/xtradb/trx/trx0sys.c index 198430f9a01..0d13de23e0e 100644 --- a/storage/xtradb/trx/trx0sys.c +++ b/storage/xtradb/trx/trx0sys.c @@ -966,24 +966,6 @@ trx_sys_print_mysql_binlog_offset(void) #ifdef WITH_WSREP -#ifdef UNIV_DEBUG -static long long trx_sys_cur_xid_seqno = -1; -static unsigned char trx_sys_cur_xid_uuid[16]; - -long long read_wsrep_xid_seqno(const XID* xid) -{ - long long seqno; - memcpy(&seqno, xid->data + 24, sizeof(long long)); - return seqno; -} - -void read_wsrep_xid_uuid(const XID* xid, unsigned char* buf) -{ - memcpy(buf, xid->data + 8, 16); -} - -#endif /* UNIV_DEBUG */ - void trx_sys_update_wsrep_checkpoint( const XID* xid, /*!< in: transaction XID */ From 2b811f0624c46469185ff4d93bd8f184e1f7ac92 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 26 Jul 2017 11:36:57 -0400 Subject: [PATCH 040/203] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 68358acf754..2905c37cc09 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=57 +MYSQL_VERSION_PATCH=58 MYSQL_VERSION_EXTRA= From 86d31ce9f141d34fffcf7f355d4c0e6e54077eb3 Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko Date: Mon, 19 Jun 2017 17:23:02 +0700 Subject: [PATCH 041/203] MW-384 protect access to wsrep_ready variable with mutex --- scripts/mysql_config.pl | 6 +++--- sql/mysqld.cc | 2 +- sql/slave.cc | 2 +- sql/sql_parse.cc | 4 ++-- sql/wsrep_mysqld.cc | 36 +++++++++++++++++++++++------------- sql/wsrep_mysqld.h | 3 ++- sql/wsrep_priv.h | 2 +- sql/wsrep_sst.cc | 1 - 8 files changed, 33 insertions(+), 23 deletions(-) diff --git a/scripts/mysql_config.pl b/scripts/mysql_config.pl index 5490853c091..a9254bb966e 100644 --- a/scripts/mysql_config.pl +++ b/scripts/mysql_config.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl # -*- cperl -*- # -# Copyright (c) 2007, 2014, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -53,7 +53,7 @@ my $cwd = cwd(); my $basedir; my $socket = '/tmp/mysql.sock'; -my $version = '5.5.38'; +my $version = '5.5.58'; sub which { @@ -204,7 +204,7 @@ $flags->{embedded_libs} = [@ldflags,@lib_e_opts,'','','','','']; $flags->{include} = ["-I$pkgincludedir"]; -$flags->{cflags} = [@{$flags->{include}},split(" ",'-Wall -DWITH_WSREP -DWSREP_PROC_INFO -DMYSQL_MAX_VARIABLE_VALUE_LEN=2048 -DWITH_INNODB_DISALLOW_WRITES -O2 -g -DNDEBUG -DDBUG_OFF')]; +$flags->{cflags} = [@{$flags->{include}},split(" ",'')]; # ---------------------------------------------------------------------- # Remove some options that a client doesn't have to care about diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 9198f616d15..f76cbe5e5b2 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -7929,7 +7929,7 @@ SHOW_VAR status_vars[]= { #endif #ifdef WITH_WSREP {"wsrep_connected", (char*) &wsrep_connected, SHOW_BOOL}, - {"wsrep_ready", (char*) &wsrep_ready, SHOW_BOOL}, + {"wsrep_ready", (char*) &wsrep_show_ready, SHOW_FUNC}, {"wsrep_cluster_state_uuid", (char*) &wsrep_cluster_state_uuid,SHOW_CHAR_PTR}, {"wsrep_cluster_conf_id", (char*) &wsrep_cluster_conf_id, SHOW_LONGLONG}, {"wsrep_cluster_status", (char*) &wsrep_cluster_status, SHOW_CHAR_PTR}, diff --git a/sql/slave.cc b/sql/slave.cc index 5f7c5ffdb84..f370e3dd27f 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3794,7 +3794,7 @@ err_during_init: */ if (wsrep_node_dropped && wsrep_restart_slave) { - if (wsrep_ready) + if (wsrep_ready_get()) { WSREP_INFO("Slave error due to node temporarily non-primary" "SQL slave will continue"); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index f0f021d4047..d102c6162d0 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -851,7 +851,7 @@ bool do_command(THD *thd) * bail out if DB snapshot has not been installed. We however, * allow queries "SET" and "SHOW", they are trapped later in execute_command */ - if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && + if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready_get() && command != COM_QUERY && command != COM_PING && command != COM_QUIT && @@ -2382,7 +2382,7 @@ mysql_execute_command(THD *thd) We additionally allow all other commands that do not change data in case wsrep_dirty_reads is enabled. */ - if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && + if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready_get() && lex->sql_command != SQLCOM_SET_OPTION && !(thd->variables.wsrep_dirty_reads && !is_update_query(lex->sql_command)) && diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 4ac346ca0a3..54fdf430f86 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -13,6 +13,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include // SHOW_MY_BOOL #include #include #include @@ -295,7 +296,7 @@ wsrep_view_handler_cb (void* app_ctx, // version change if (view->proto_ver != wsrep_protocol_version) { - my_bool wsrep_ready_saved= wsrep_ready; + my_bool wsrep_ready_saved= wsrep_ready_get(); wsrep_ready_set(FALSE); WSREP_INFO("closing client connections for " "protocol change %ld -> %d", @@ -409,16 +410,34 @@ out: return WSREP_CB_SUCCESS; } -void wsrep_ready_set (my_bool x) +my_bool wsrep_ready_set (my_bool x) { WSREP_DEBUG("Setting wsrep_ready to %d", x); if (mysql_mutex_lock (&LOCK_wsrep_ready)) abort(); - if (wsrep_ready != x) + my_bool ret= (wsrep_ready != x); + if (ret) { wsrep_ready= x; mysql_cond_signal (&COND_wsrep_ready); } mysql_mutex_unlock (&LOCK_wsrep_ready); + return ret; +} + +my_bool wsrep_ready_get (void) +{ + if (mysql_mutex_lock (&LOCK_wsrep_ready)) abort(); + my_bool ret= wsrep_ready; + mysql_mutex_unlock (&LOCK_wsrep_ready); + return ret; +} + +int wsrep_show_ready(THD *thd, SHOW_VAR *var, char *buff) +{ + var->type= SHOW_MY_BOOL; + var->value= buff; + *((my_bool *)buff)= wsrep_ready_get(); + return 0; } // Wait until wsrep has reached ready state @@ -437,17 +456,8 @@ void wsrep_ready_wait () static void wsrep_synced_cb(void* app_ctx) { WSREP_INFO("Synchronized with group, ready for connections"); - bool signal_main= false; - if (mysql_mutex_lock (&LOCK_wsrep_ready)) abort(); - if (!wsrep_ready) - { - wsrep_ready= TRUE; - mysql_cond_signal (&COND_wsrep_ready); - signal_main= true; - - } + my_bool signal_main= wsrep_ready_set(TRUE); local_status.set(WSREP_MEMBER_SYNCED); - mysql_mutex_unlock (&LOCK_wsrep_ready); if (signal_main) { diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index bb33d2f1069..56e3baae7cc 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -115,7 +115,6 @@ enum enum_wsrep_sync_wait { // MySQL status variables extern my_bool wsrep_connected; -extern my_bool wsrep_ready; extern const char* wsrep_cluster_state_uuid; extern long long wsrep_cluster_conf_id; extern const char* wsrep_cluster_status; @@ -130,6 +129,7 @@ extern const char* wsrep_provider_vendor; extern my_bool wsrep_inited; // whether wsrep is initialized ? int wsrep_show_status(THD *thd, SHOW_VAR *var, char *buff); +int wsrep_show_ready(THD *thd, SHOW_VAR *var, char *buff); void wsrep_free_status(THD *thd); /* Filters out --wsrep-new-cluster oprtion from argv[] @@ -245,6 +245,7 @@ extern wsrep_seqno_t wsrep_locked_seqno; #define WSREP_PROVIDER_EXISTS \ (wsrep_provider && strncasecmp(wsrep_provider, WSREP_NONE, FN_REFLEN)) +extern my_bool wsrep_ready_get(); extern void wsrep_ready_wait(); enum wsrep_trx_status { diff --git a/sql/wsrep_priv.h b/sql/wsrep_priv.h index 93640fbcc03..ce97ac65182 100644 --- a/sql/wsrep_priv.h +++ b/sql/wsrep_priv.h @@ -26,7 +26,7 @@ #include #include -void wsrep_ready_set (my_bool x); +my_bool wsrep_ready_set (my_bool x); ssize_t wsrep_sst_prepare (void** msg); wsrep_cb_status wsrep_sst_donate_cb (void* app_ctx, diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index a5fe8a1e2d8..d870824ed9d 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -1057,7 +1057,6 @@ wsrep_cb_status_t wsrep_sst_donate_cb (void* app_ctx, void* recv_ctx, { /* This will be reset when sync callback is called. * Should we set wsrep_ready to FALSE here too? */ -// wsrep_notify_status(WSREP_MEMBER_DONOR); local_status.set(WSREP_MEMBER_DONOR); const char* method = (char*)msg; From 0332acc77cc7e22ed2f98537477d3e23fb3efa36 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Fri, 4 Aug 2017 11:22:35 +0300 Subject: [PATCH 042/203] MW-394 Merged fix for LOAD DATA splitting with log_bin==OFF, from mysql-wsrep-bugs/MW-394 --- sql/sql_class.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 43d7ffd94d6..d540abb276c 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -4297,14 +4297,16 @@ extern "C" int thd_non_transactional_update(const MYSQL_THD thd) extern "C" int thd_binlog_format(const MYSQL_THD thd) { #ifdef WITH_WSREP - if (((WSREP(thd) && wsrep_emulate_bin_log) || mysql_bin_log.is_open()) && - (thd->variables.option_bits & OPTION_BIN_LOG)) -#else + if (WSREP(thd)) + { + /* for wsrep binlog format is meaningful also when binlogging is off */ + return (int) WSREP_BINLOG_FORMAT(thd->variables.binlog_format); + } +#endif /* WITH_WSREP */ if (mysql_bin_log.is_open() && (thd->variables.option_bits & OPTION_BIN_LOG)) -#endif - return (int) WSREP_BINLOG_FORMAT(thd->variables.binlog_format); + return (int) thd->variables.binlog_format; else - return BINLOG_FORMAT_UNSPEC; + return BINLOG_FORMAT_UNSPEC; } extern "C" void thd_mark_transaction_to_rollback(MYSQL_THD thd, bool all) From e90249bf472eda519a1485e5f3381ef2a80ce59d Mon Sep 17 00:00:00 2001 From: sjaakola Date: Fri, 25 Aug 2017 12:41:56 +0300 Subject: [PATCH 043/203] MW-402 cascading FK issues (5.5 version) copied the fix for MW-402 from mysql-wsrep-bugs/MW-402-5.6 (note, this is not git merge) --- storage/innobase/row/row0ins.c | 3 +-- storage/innobase/row/row0upd.c | 10 +++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/storage/innobase/row/row0ins.c b/storage/innobase/row/row0ins.c index 3726c517ff0..1cd3e9e7c74 100644 --- a/storage/innobase/row/row0ins.c +++ b/storage/innobase/row/row0ins.c @@ -1086,8 +1086,7 @@ row_ins_foreign_check_on_constraint( foreign, clust_rec, clust_index, - FALSE, - (node) ? TRUE : FALSE); + FALSE, FALSE); if (err != DB_SUCCESS) { fprintf(stderr, "WSREP: foreign key append failed: %lu\n", err); diff --git a/storage/innobase/row/row0upd.c b/storage/innobase/row/row0upd.c index dd9c7e8defd..0a503f01f40 100644 --- a/storage/innobase/row/row0upd.c +++ b/storage/innobase/row/row0upd.c @@ -2107,7 +2107,7 @@ err_exit: } } #ifdef WITH_WSREP - if (!referenced && + if (wsrep_on(trx->mysql_thd) && !referenced && !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && ((upd_node_t*)parent)->cascade_node == node) && foreign @@ -2313,6 +2313,7 @@ row_upd_del_mark_clust_rec( #ifdef WITH_WSREP rec_t* rec; que_node_t *parent = que_node_get_parent(node); + trx_t* trx = thr_get_trx(thr); #endif /* WITH_WSREP */ ut_ad(node); @@ -2348,11 +2349,10 @@ row_upd_del_mark_clust_rec( node, pcur, index->table, index, offsets, thr, mtr); } #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && + if (trx && wsrep_on(trx->mysql_thd) && + err == DB_SUCCESS && !referenced && foreign && !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - thr_get_trx(thr) && - foreign + ((upd_node_t*)parent)->cascade_node == node) ) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); From 2864c37d6c5d4c13472438863d513d0ea8b9063c Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Mon, 18 Sep 2017 16:12:13 +0300 Subject: [PATCH 044/203] MW-406 Bumped up the wsrep patch version (5.5.57-25.22) --- cmake/wsrep.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index 6a9dd720f77..afab9105d8d 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -18,7 +18,7 @@ # so WSREP_VERSION is produced regardless # Set the patch version -SET(WSREP_PATCH_VERSION "21") +SET(WSREP_PATCH_VERSION "22") # MariaDB addition: Revision number of the last revision merged from # codership branch visible in @@visible_comment. From 38530c86aa36e2cbd315447a5362bcc751fb9205 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Thu, 5 Oct 2017 11:41:02 +0200 Subject: [PATCH 045/203] MW-415 THD::COND_wsrep_thd is never destroyed THD::COND_wsrep_thd is never destroyed. This patch adds missing call to mysql_cond_destroy(&COND_wsrep_thd) in THD::release_resources(). --- sql/sql_class.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index d540abb276c..1398fdff94f 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1693,6 +1693,7 @@ THD::~THD() mysql_mutex_lock(&LOCK_wsrep_thd); mysql_mutex_unlock(&LOCK_wsrep_thd); mysql_mutex_destroy(&LOCK_wsrep_thd); + mysql_cond_destroy(&COND_wsrep_thd); if (wsrep_rli) delete wsrep_rli; if (wsrep_status_vars) wsrep->stats_free(wsrep, wsrep_status_vars); #endif From 8822b30f1e258a5f0efc124043f42970e600c5d4 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Tue, 10 Oct 2017 23:39:48 +0300 Subject: [PATCH 046/203] MW-416 Replicating DDL after ACL check, 5.6 version Re-implemented the fix for MW-416 according to 5.7 version --- mysql-test/suite/galera/r/MW-416.result | 114 ++++++++++++++++++++ mysql-test/suite/galera/t/MW-416.test | 132 ++++++++++++++++++++++++ sql/events.cc | 15 +++ sql/sql_class.h | 1 + sql/sql_parse.cc | 7 -- sql/sql_plugin.cc | 19 +++- sql/sql_trigger.cc | 5 + sql/sql_view.cc | 5 + 8 files changed, 287 insertions(+), 11 deletions(-) create mode 100644 mysql-test/suite/galera/r/MW-416.result create mode 100644 mysql-test/suite/galera/t/MW-416.test diff --git a/mysql-test/suite/galera/r/MW-416.result b/mysql-test/suite/galera/r/MW-416.result new file mode 100644 index 00000000000..05399b213a8 --- /dev/null +++ b/mysql-test/suite/galera/r/MW-416.result @@ -0,0 +1,114 @@ +CREATE USER 'userMW416'@'localhost'; +GRANT SELECT, INSERT, UPDATE ON test.* TO 'userMW416'@'localhost'; +SHOW GLOBAL STATUS LIKE 'wsrep_replicated'; +Variable_name Value +wsrep_replicated 2 +ALTER DATABASE db CHARACTER SET = utf8; +ERROR 42000: Access denied for user 'userMW416'@'localhost' to database 'db' +ALTER EVENT ev1 RENAME TO ev2; +ERROR 42000: Access denied for user 'userMW416'@'localhost' to database 'test' +ALTER FUNCTION fun1 COMMENT 'foo'; +ERROR 42000: alter routine command denied to user 'userMW416'@'localhost' for routine 'test.fun1' +ALTER LOGFILE GROUP lfg ADD UNDOFILE 'file' ENGINE=InnoDB; +Got one of the listed errors +ALTER PROCEDURE proc1 COMMENT 'foo'; +Got one of the listed errors +ALTER SERVER srv OPTIONS (USER 'sally'); +Got one of the listed errors +ALTER TABLE tbl DROP COLUMN col; +Got one of the listed errors +ALTER TABLESPACE tblspc DROP DATAFILE 'file' ENGINE=innodb; +Got one of the listed errors +ALTER VIEW vw AS SELECT 1; +Got one of the listed errors +CREATE DATABASE db; +Got one of the listed errors +CREATE EVENT ev1 ON SCHEDULE AT CURRENT_TIMESTAMP DO SELECT 1; +Got one of the listed errors +CREATE FUNCTION fun1() RETURNS int RETURN(1); +Got one of the listed errors +CREATE FUNCTION fun1 RETURNS STRING SONAME 'funlib.so'; +Got one of the listed errors +CREATE PROCEDURE proc1() BEGIN END; +Got one of the listed errors +CREATE INDEX idx ON tbl(id); +Got one of the listed errors +CREATE LOGFILE GROUP lfg ADD UNDOFILE 'undofile' ENGINE innodb; +Got one of the listed errors +CREATE SERVER srv FOREIGN DATA WRAPPER 'fdw' OPTIONS (USER 'user'); +Got one of the listed errors +CREATE TABLE t (i int); +Got one of the listed errors +CREATE TABLESPACE tblspc ADD DATAFILE 'file' ENGINE=innodb; +Got one of the listed errors +CREATE TRIGGER trg BEFORE UPDATE ON t FOR EACH ROW BEGIN END; +Got one of the listed errors +CREATE VIEW vw AS SELECT 1; +Got one of the listed errors +DROP DATABASE db; +Got one of the listed errors +DROP EVENT ev; +Got one of the listed errors +DROP FUNCTION fun1; +Got one of the listed errors +DROP INDEX idx ON t0; +Got one of the listed errors +DROP LOGFILE GROUP lfg; +Got one of the listed errors +DROP PROCEDURE proc1; +Got one of the listed errors +DROP SERVEr srv; +Got one of the listed errors +DROP TABLE t0; +Got one of the listed errors +DROP TABLESPACE tblspc; +Got one of the listed errors +DROP TRIGGER trg; +Got one of the listed errors +DROP VIEW vw; +Got one of the listed errors +RENAME TABLE t0 TO t1; +Got one of the listed errors +TRUNCATE TABLE t0; +Got one of the listed errors +ALTER USER myuser PASSWORD EXPIRE; +Got one of the listed errors +CREATE USER myuser IDENTIFIED BY 'pass'; +Got one of the listed errors +DROP USER myuser; +Got one of the listed errors +GRANT ALL ON *.* TO 'myuser'; +Got one of the listed errors +RENAME USER myuser TO mariauser; +Got one of the listed errors +REVOKE SELECT ON test FROM myuser; +Got one of the listed errors +REVOKE ALL, GRANT OPTION FROM myuser; +Got one of the listed errors +REVOKE PROXY ON myuser FROM myuser; +Got one of the listed errors +ANALYZE TABLE db.tbl; +Got one of the listed errors +CHECK TABLE db.tbl; +Got one of the listed errors +CHECKSUM TABLE db.tbl; +Got one of the listed errors +OPTIMIZE TABLE db.tbl; +Got one of the listed errors +REPAIR TABLE db.tbl; +Got one of the listed errors +INSTALL PLUGIN plg SONAME 'plg.so'; +Got one of the listed errors +UNINSTALL PLUGIN plg; +Got one of the listed errors +DROP USER 'userMW416'@'localhost'; +SHOW DATABASES; +Database +information_schema +mtr +mysql +performance_schema +test +SHOW GLOBAL STATUS LIKE 'wsrep_replicated'; +Variable_name Value +wsrep_replicated 3 diff --git a/mysql-test/suite/galera/t/MW-416.test b/mysql-test/suite/galera/t/MW-416.test new file mode 100644 index 00000000000..ebc15438adc --- /dev/null +++ b/mysql-test/suite/galera/t/MW-416.test @@ -0,0 +1,132 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +CREATE USER 'userMW416'@'localhost'; +GRANT SELECT, INSERT, UPDATE ON test.* TO 'userMW416'@'localhost'; + +SHOW GLOBAL STATUS LIKE 'wsrep_replicated'; + +--connect userMW416, localhost, userMW416,, test, $NODE_MYPORT_1 +--connection userMW416 + +# DDL + +--error 1044 +ALTER DATABASE db CHARACTER SET = utf8; +--error 1044 +ALTER EVENT ev1 RENAME TO ev2; +--error 1370 +ALTER FUNCTION fun1 COMMENT 'foo'; +#--error 1044,1227 +#ALTER INSTANCE ROTATE INNODB MASTER KEY; +--error 1044,1227 +ALTER LOGFILE GROUP lfg ADD UNDOFILE 'file' ENGINE=InnoDB; +--error 1044,1227,1370 +ALTER PROCEDURE proc1 COMMENT 'foo'; +--error 1044,1227,1370 +ALTER SERVER srv OPTIONS (USER 'sally'); +--error 1044,1142,1227,1370 +ALTER TABLE tbl DROP COLUMN col; +--error 1044,1227,1370 +ALTER TABLESPACE tblspc DROP DATAFILE 'file' ENGINE=innodb; +--error 1044,1142,1227,1370 +ALTER VIEW vw AS SELECT 1; + +--error 1044,1227,1370 +CREATE DATABASE db; +--error 1044,1227,1370 +CREATE EVENT ev1 ON SCHEDULE AT CURRENT_TIMESTAMP DO SELECT 1; +--error 1044,1227,1370 +CREATE FUNCTION fun1() RETURNS int RETURN(1); +--error 1044,1227,1370 +CREATE FUNCTION fun1 RETURNS STRING SONAME 'funlib.so'; +--error 1044,1227,1370 +CREATE PROCEDURE proc1() BEGIN END; +--error 1044,1142,1227,1370 +CREATE INDEX idx ON tbl(id); +--error 1044,1142,1227,1370 +CREATE LOGFILE GROUP lfg ADD UNDOFILE 'undofile' ENGINE innodb; +--error 1044,1142,1227,1370 +CREATE SERVER srv FOREIGN DATA WRAPPER 'fdw' OPTIONS (USER 'user'); +--error 1044,1142,1227,1370 +CREATE TABLE t (i int); +--error 1044,1142,1227,1370 +CREATE TABLESPACE tblspc ADD DATAFILE 'file' ENGINE=innodb; +--error 1044,1142,1227,1370 +CREATE TRIGGER trg BEFORE UPDATE ON t FOR EACH ROW BEGIN END; +--error 1044,1142,1227,1370 +CREATE VIEW vw AS SELECT 1; + + + +--error 1044,1142,1227,1370 +DROP DATABASE db; +--error 1044,1142,1227,1370 +DROP EVENT ev; +--error 1044,1142,1227,1370 +DROP FUNCTION fun1; +--error 1044,1142,1227,1370 +DROP INDEX idx ON t0; +--error 1044,1142,1227,1370 +DROP LOGFILE GROUP lfg; +--error 1044,1142,1227,1370 +DROP PROCEDURE proc1; +--error 1044,1142,1227,1370 +DROP SERVEr srv; +--error 1044,1142,1227,1370 +DROP TABLE t0; +--error 1044,1142,1227,1370 +DROP TABLESPACE tblspc; +--error 1044,1142,1227,1360,1370 +DROP TRIGGER trg; +--error 1044,1142,1227,1370 +DROP VIEW vw; + +--error 1044,1142,1227,1370 +RENAME TABLE t0 TO t1; + +--error 1044,1142,1227,1370 +TRUNCATE TABLE t0; + +# DCL + +# account management +--error 1044,1142,1227,1370 +ALTER USER myuser PASSWORD EXPIRE; +--error 1044,1142,1227,1370 +CREATE USER myuser IDENTIFIED BY 'pass'; +--error 1044,1142,1227,1370 +DROP USER myuser; +--error 1044,1045,1142,1227,1370 +GRANT ALL ON *.* TO 'myuser'; +--error 1044,1142,1227,1370 +RENAME USER myuser TO mariauser; +--error 1044,1142,1227,1370 +REVOKE SELECT ON test FROM myuser; +--error 1044,1142,1227,1370,1698 +REVOKE ALL, GRANT OPTION FROM myuser; +--error 1044,1142,1227,1370,1698 +REVOKE PROXY ON myuser FROM myuser; + +# table maintenance +--error 1044,1142,1227,1370 +ANALYZE TABLE db.tbl; +--error 1044,1142,1227,1370 +CHECK TABLE db.tbl; +--error 1044,1142,1227,1370 +CHECKSUM TABLE db.tbl; +--error 1044,1142,1227,1370 +OPTIMIZE TABLE db.tbl; +--error 1044,1142,1227,1370 +REPAIR TABLE db.tbl; + +# plugin and user defined functions +--error 1044,1142,1227,1370 +INSTALL PLUGIN plg SONAME 'plg.so'; +--error 1044,1142,1227,1370 +UNINSTALL PLUGIN plg; + +--connection node_1 +DROP USER 'userMW416'@'localhost'; +SHOW DATABASES; +SHOW GLOBAL STATUS LIKE 'wsrep_replicated'; diff --git a/sql/events.cc b/sql/events.cc index 7aa7dae4d7f..a6379ec5a46 100644 --- a/sql/events.cc +++ b/sql/events.cc @@ -327,6 +327,7 @@ Events::create_event(THD *thd, Event_parse_data *parse_data, if (check_access(thd, EVENT_ACL, parse_data->dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); if (check_db_dir_existence(parse_data->dbname.str)) { @@ -406,6 +407,10 @@ Events::create_event(THD *thd, Event_parse_data *parse_data, thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(ret); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -446,6 +451,7 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, if (check_access(thd, EVENT_ACL, parse_data->dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); if (new_dbname) /* It's a rename */ { @@ -521,6 +527,10 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(ret); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -560,6 +570,7 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) if (check_access(thd, EVENT_ACL, dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); /* Turn off row binlogging of this statement and use statement-based so @@ -585,6 +596,10 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) if (save_binlog_row_based) thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(ret); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } diff --git a/sql/sql_class.h b/sql/sql_class.h index 8cfb3ced4b6..70bf6f7783d 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -2383,6 +2383,7 @@ public: enum wsrep_query_state wsrep_query_state; enum wsrep_conflict_state wsrep_conflict_state; mysql_mutex_t LOCK_wsrep_thd; + mysql_cond_t COND_wsrep_thd; // changed from wsrep_seqno_t to wsrep_trx_meta_t in wsrep API rev 75 // wsrep_seqno_t wsrep_trx_seqno; wsrep_trx_meta_t wsrep_trx_meta; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index d102c6162d0..553a6e7539d 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -3825,7 +3825,6 @@ end_with_restore_list: if (res) break; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) switch (lex->sql_command) { case SQLCOM_CREATE_EVENT: { @@ -3861,7 +3860,6 @@ end_with_restore_list: lex->spname->m_name); break; case SQLCOM_DROP_EVENT: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (!(res= Events::drop_event(thd, lex->spname->m_db, lex->spname->m_name, lex->drop_if_exists))) @@ -4773,7 +4771,6 @@ create_sp_error: Note: SQLCOM_CREATE_VIEW also handles 'ALTER VIEW' commands as specified through the thd->lex->create_view_mode flag. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_view(thd, first_table, thd->lex->create_view_mode); break; } @@ -4789,7 +4786,6 @@ create_sp_error: case SQLCOM_CREATE_TRIGGER: { /* Conditionally writes to binlog. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_or_drop_trigger(thd, all_tables, 1); break; @@ -4797,7 +4793,6 @@ create_sp_error: case SQLCOM_DROP_TRIGGER: { /* Conditionally writes to binlog. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_or_drop_trigger(thd, all_tables, 0); break; } @@ -4860,13 +4855,11 @@ create_sp_error: my_ok(thd); break; case SQLCOM_INSTALL_PLUGIN: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (! (res= mysql_install_plugin(thd, &thd->lex->comment, &thd->lex->ident))) my_ok(thd); break; case SQLCOM_UNINSTALL_PLUGIN: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (! (res= mysql_uninstall_plugin(thd, &thd->lex->comment, &thd->lex->ident))) my_ok(thd); diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index d4f497743b7..7e5c7c9c852 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2079,6 +2079,8 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, bool error; int argc=orig_argc; char **argv=orig_argv; + unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = + { MYSQL_AUDIT_GENERAL_CLASSMASK }; DBUG_ENTER("mysql_install_plugin"); if (opt_noacl) @@ -2090,6 +2092,7 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, tables.init_one_table("mysql", 5, "plugin", 6, "plugin", TL_WRITE); if (check_table_access(thd, INSERT_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table = open_ltable(thd, &tables, TL_WRITE, @@ -2123,8 +2126,7 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, See also mysql_uninstall_plugin() and initialize_audit_plugin() */ - unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = - { MYSQL_AUDIT_GENERAL_CLASSMASK }; + mysql_audit_acquire_plugins(thd, event_class_mask); mysql_mutex_lock(&LOCK_plugin); @@ -2155,6 +2157,10 @@ err: if (argv) free_defaults(argv); DBUG_RETURN(error); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -2221,6 +2227,8 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, TABLE_LIST tables; LEX_STRING dl= *dl_arg; bool error= false; + unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = + { MYSQL_AUDIT_GENERAL_CLASSMASK }; DBUG_ENTER("mysql_uninstall_plugin"); if (opt_noacl) @@ -2233,6 +2241,7 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, if (check_table_access(thd, DELETE_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table= open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT))) @@ -2259,8 +2268,6 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, See also mysql_install_plugin() and initialize_audit_plugin() */ - unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = - { MYSQL_AUDIT_GENERAL_CLASSMASK }; mysql_audit_acquire_plugins(thd, event_class_mask); mysql_mutex_lock(&LOCK_plugin); @@ -2290,6 +2297,10 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, mysql_mutex_unlock(&LOCK_plugin); DBUG_RETURN(error); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index c562ee9762c..70ed6f0e600 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -501,6 +501,7 @@ bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create) if (err_status) goto end; } + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); /* We should have only one table in table list. */ DBUG_ASSERT(tables->next_global == 0); @@ -605,6 +606,10 @@ end: my_ok(thd); DBUG_RETURN(result); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 5bd82fdd842..f6948a76f6a 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -429,6 +429,7 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, if ((res= create_view_precheck(thd, tables, view, mode))) goto err; + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; @@ -721,6 +722,10 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, lex->link_first_table_back(view, link_to_local); DBUG_RETURN(0); +#ifdef WITH_WSREP + error: + res= TRUE; +#endif /* WITH_WSREP */ err: thd_proc_info(thd, "end"); lex->link_first_table_back(view, link_to_local); From 12d7ee03efa36e9eb293fefe8313d3848fc41113 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Tue, 10 Oct 2017 23:39:48 +0300 Subject: [PATCH 047/203] MW-416 Replicating DDL after ACL check --- sql/event_data_objects.cc | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/sql/event_data_objects.cc b/sql/event_data_objects.cc index 656881b9977..0cb123451df 100644 --- a/sql/event_data_objects.cc +++ b/sql/event_data_objects.cc @@ -1468,23 +1468,35 @@ end: thd->security_ctx->master_access |= SUPER_ACL; #ifdef WITH_WSREP - if (WSREP(thd)) { - // sql_print_information("sizeof(LEX) = %d", sizeof(struct LEX)); - // sizeof(LEX) = 4512, so it's relatively safe to allocate it on stack. - LEX lex; - lex.sql_command = SQLCOM_DROP_EVENT; - LEX* saved = thd->lex; - thd->lex = &lex; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); - thd->lex = saved; + /* + This code is processing event execution and does not have client + connection. Here, event execution will now execute a prepared + DROP EVENT statement, but thd->lex->sql_command is set to + SQLCOM_CREATE_PROCEDURE + DROP EVENT will be logged in binlog, and we have to + replicate it to make all nodes have consistent event definitions + Wsrep DDL replication is triggered inside Events::drop_event(), + and here we need to prepare the THD so that DDL replication is + possible, essentially it requires setting sql_command to + SQLCOMM_DROP_EVENT, we will switch sql_command for the duration + of DDL replication only. + */ + const enum_sql_command sql_command_save= thd->lex->sql_command; + const bool sql_command_set= WSREP(thd); + if (WSREP(thd)) + { + thd->lex->sql_command = SQLCOM_DROP_EVENT; } #endif ret= Events::drop_event(thd, dbname, name, FALSE); #ifdef WITH_WSREP - WSREP_TO_ISOLATION_END; - error: + if (sql_command_set) + { + WSREP_TO_ISOLATION_END; + thd->lex->sql_command = sql_command_save; + } #endif thd->security_ctx->master_access= saved_master_access; } From 241a2687d7ce07572adb2fb4629bbd319a732e5a Mon Sep 17 00:00:00 2001 From: sjaakola Date: Wed, 11 Oct 2017 15:35:17 +0300 Subject: [PATCH 048/203] MW-416 Replicate DDL after ACL check CREATE VIEW handling had an issue, because CREATE VIEW replication happened when view was unlinked from table list. This caused debug assert in wsrep_mysqld.cc:wsrep_can_run_in_toi() MySQL 5.5 version does not have this assert, but it is still better to run replication when table list is not manipulated. --- sql/sql_view.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_view.cc b/sql/sql_view.cc index f6948a76f6a..bbc5f002461 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -429,10 +429,10 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, if ((res= create_view_precheck(thd, tables, view, mode))) goto err; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); if (open_and_lock_tables(thd, lex->query_tables, TRUE, 0)) { From 181f3015bf62bccc108aacd4df55065e0158b060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 19 Oct 2017 11:06:55 +0300 Subject: [PATCH 049/203] MariaDB adjustments. Add missing COND variable and disable test that fail. --- mysql-test/suite/galera/disabled.def | 2 ++ mysql-test/suite/galera/t/MW-416.test | 8 +++++--- sql/mysqld.cc | 3 ++- sql/mysqld.h | 1 + sql/sql_class.cc | 1 + 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index c91654c2023..ad966ebab0d 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -1,2 +1,4 @@ galera_var_dirty_reads : MDEV-12539 query_cache : MDEV-12539 +MW-421 : MDEV-12539 +galera_concurrent_ctas : MDEV-12539 diff --git a/mysql-test/suite/galera/t/MW-416.test b/mysql-test/suite/galera/t/MW-416.test index ebc15438adc..df4fa35abc7 100644 --- a/mysql-test/suite/galera/t/MW-416.test +++ b/mysql-test/suite/galera/t/MW-416.test @@ -1,6 +1,8 @@ --source include/galera_cluster.inc --source include/have_innodb.inc +--source include/wait_until_ready.inc + CREATE USER 'userMW416'@'localhost'; GRANT SELECT, INSERT, UPDATE ON test.* TO 'userMW416'@'localhost'; @@ -67,7 +69,7 @@ DROP EVENT ev; DROP FUNCTION fun1; --error 1044,1142,1227,1370 DROP INDEX idx ON t0; ---error 1044,1142,1227,1370 +--error 1044,1142,1227,1370,1064 DROP LOGFILE GROUP lfg; --error 1044,1142,1227,1370 DROP PROCEDURE proc1; @@ -75,7 +77,7 @@ DROP PROCEDURE proc1; DROP SERVEr srv; --error 1044,1142,1227,1370 DROP TABLE t0; ---error 1044,1142,1227,1370 +--error 1044,1142,1227,1370,1064 DROP TABLESPACE tblspc; --error 1044,1142,1227,1360,1370 DROP TRIGGER trg; @@ -91,7 +93,7 @@ TRUNCATE TABLE t0; # DCL # account management ---error 1044,1142,1227,1370 +--error 1044,1142,1227,1370,1064 ALTER USER myuser PASSWORD EXPIRE; --error 1044,1142,1227,1370 CREATE USER myuser IDENTIFIED BY 'pass'; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index f76cbe5e5b2..3edbeafe224 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -911,7 +911,7 @@ PSI_cond_key key_BINLOG_COND_prep_xids, key_BINLOG_update_cond, key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache, key_BINLOG_COND_queue_busy; #ifdef WITH_WSREP -PSI_cond_key key_COND_wsrep_rollback, +PSI_cond_key key_COND_wsrep_rollback, key_COND_wsrep_thd, key_COND_wsrep_replaying, key_COND_wsrep_ready, key_COND_wsrep_sst, key_COND_wsrep_sst_init, key_COND_wsrep_sst_thread; #endif /* WITH_WSREP */ @@ -962,6 +962,7 @@ static PSI_cond_info all_server_conds[]= { &key_COND_wsrep_sst_init, "COND_wsrep_sst_init", PSI_FLAG_GLOBAL}, { &key_COND_wsrep_sst_thread, "wsrep_sst_thread", 0}, { &key_COND_wsrep_rollback, "COND_wsrep_rollback", PSI_FLAG_GLOBAL}, + { &key_COND_wsrep_thd, "THD::COND_wsrep_thd", 0}, { &key_COND_wsrep_replaying, "COND_wsrep_replaying", PSI_FLAG_GLOBAL}, #endif { &key_COND_flush_thread_cache, "COND_flush_thread_cache", PSI_FLAG_GLOBAL} diff --git a/sql/mysqld.h b/sql/mysqld.h index 14997455f2f..91fa2eda7fd 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -223,6 +223,7 @@ extern PSI_mutex_key key_PAGE_lock, key_LOCK_sync, key_LOCK_active, #endif /* HAVE_MMAP */ #ifdef WITH_WSREP extern PSI_mutex_key key_LOCK_wsrep_thd; +extern PSI_cond_key key_COND_wsrep_thd; #endif /* HAVE_WSREP */ #ifdef HAVE_OPENSSL diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 1398fdff94f..fcba6661410 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1105,6 +1105,7 @@ THD::THD() #ifdef WITH_WSREP mysql_mutex_init(key_LOCK_wsrep_thd, &LOCK_wsrep_thd, MY_MUTEX_INIT_FAST); + mysql_cond_init(key_COND_wsrep_thd, &COND_wsrep_thd, NULL); wsrep_ws_handle.trx_id = WSREP_UNDEFINED_TRX_ID; wsrep_ws_handle.opaque = NULL; wsrep_retry_counter = 0; From 016785f6aa9c56b731b612363e7d52488735ee98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 19 Oct 2017 15:02:14 +0300 Subject: [PATCH 050/203] Fix test failure on perfschema.all_instances. --- mysql-test/suite/perfschema/r/all_instances.result | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/perfschema/r/all_instances.result b/mysql-test/suite/perfschema/r/all_instances.result index 3b0b4db4541..bfe50b881da 100644 --- a/mysql-test/suite/perfschema/r/all_instances.result +++ b/mysql-test/suite/perfschema/r/all_instances.result @@ -152,6 +152,7 @@ wait/synch/cond/sql/Relay_log_info::sleep_cond wait/synch/cond/sql/Relay_log_info::start_cond wait/synch/cond/sql/Relay_log_info::stop_cond wait/synch/cond/sql/THD::COND_wakeup_ready +wait/synch/cond/sql/THD::COND_wsrep_thd select event_name from file_instances group by event_name; event_name wait/io/file/aria/control From 22936df631633be2b136e6bf9a88b66e94fa1402 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 25 Oct 2017 11:42:04 -0400 Subject: [PATCH 051/203] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2905c37cc09..87b72051a84 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=58 +MYSQL_VERSION_PATCH=59 MYSQL_VERSION_EXTRA= From e6e026ae51a77969749de201d491a176483bbc69 Mon Sep 17 00:00:00 2001 From: Ian Gilfillan Date: Thu, 23 Nov 2017 14:02:36 +0200 Subject: [PATCH 052/203] Update wsrep_sync_wait documentation as per MW-86 --- mysql-test/r/mysqld--help.result | 4 ++-- sql/sys_vars.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 34d4bae0f28..91131870752 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -942,8 +942,8 @@ The following options may be given as the first argument: --wsrep-sync-wait[=#] Ensure "synchronous" read view before executing an operation of the type specified by bitmask: 1 - - READ(includes SELECT, SHOW and BEGIN/START TRANSACTION); - 2 - UPDATE and DELETE; 4 - INSERT and REPLACE + READ(includes SELECT and BEGIN/START TRANSACTION); 2 - + UPDATE and DELETE; 4 - INSERT and REPLACE; 8 - SHOW Variables (--variable-name=value) allow-suspicious-udfs FALSE diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index a0475c136ae..53b606c0336 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3897,8 +3897,8 @@ static Sys_var_mybool Sys_wsrep_causal_reads( static Sys_var_uint Sys_wsrep_sync_wait( "wsrep_sync_wait", "Ensure \"synchronous\" read view before executing " "an operation of the type specified by bitmask: 1 - READ(includes " - "SELECT, SHOW and BEGIN/START TRANSACTION); 2 - UPDATE and DELETE; 4 - " - "INSERT and REPLACE", + "SELECT and BEGIN/START TRANSACTION); 2 - UPDATE and DELETE; 4 - " + "INSERT and REPLACE; 8 - SHOW", SESSION_VAR(wsrep_sync_wait), CMD_LINE(OPT_ARG), VALID_RANGE(WSREP_SYNC_WAIT_NONE, WSREP_SYNC_WAIT_MAX), DEFAULT(WSREP_SYNC_WAIT_NONE), BLOCK_SIZE(1), From 9007ca68738cb11b24854aa1bfa12dcd204a459d Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 20 Dec 2017 00:06:02 +0530 Subject: [PATCH 053/203] MDEV-13478 Full SST sync fails because of the error in the cleaning part Problem: The command was: find $paths -mindepth 1 -regex $cpat -prune -o -exec rm -rf {} \+ Which was supposed to work as * skipping $paths directories themselves (-mindepth 1) * see if the dir/file name matches $cpat (-regex) * if yes - don't dive into the directory, skip it (-prune) * otherwise (-o) * remove it and everything inside (-exec) Now -exec ... \+ works like this: every new found path is appended to the end of the command line. when accumulated command line length reaches `getconf ARG_MAX` (~2Gb) it's executed, and find continues, appending to a new command line. What happens here, find appends some directory to the command line, then dives into it, and starts appending files from that directory. At some point command line overflows, rm -rf gets executed and removes the whole directory. Now find tries to continue scanning the directory that was already removed. Fix: don't dive into directories that will be recursively removed anyway, use -prune for them. Basically, we should be pruning both paths that have matched $cpat and paths that have not matched it. This is achived by pruning unconditionally, before the regex is tested: find $paths -mindepth 1 -prune -regex $cpat -o -exec rm -rf {} \+ Patch Credit:- Serg --- scripts/wsrep_sst_xtrabackup-v2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 327c92fb6cf..00d8fe21113 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -863,7 +863,7 @@ then wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" - find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ tempdir=$(parse_cnf mysqld log-bin "") if [[ -n ${tempdir:-} ]];then From 42ccfd82110292a4693b760295699cd6c57bbd94 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 16 Jan 2018 22:45:48 +1100 Subject: [PATCH 054/203] wsrep_sst_mysqldump: safer test of version != 5 --- scripts/wsrep_sst_mysqldump.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index f6f2187006d..56174e914dc 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -124,7 +124,7 @@ SET_GTID_BINLOG_STATE="" SQL_LOG_BIN_OFF="" # Safety check -if echo $SERVER_VERSION | grep '^10.0' > /dev/null +if [ ${SERVER_VERSION%%.*} != '5' ] then # If binary logging is enabled on the joiner node, we need to copy donor's # gtid_binlog_state to joiner. In order to do that, a RESET MASTER must be From 843503e90f902a0c4691467b294a5edc34cf9f21 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 24 Oct 2017 16:48:08 +0300 Subject: [PATCH 055/203] Set wsrep_rli to NULL after deleting it --- sql/sql_class.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 883f0c9cc88..7aae3ed91ed 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1695,7 +1695,10 @@ THD::~THD() mysql_mutex_unlock(&LOCK_wsrep_thd); mysql_mutex_destroy(&LOCK_wsrep_thd); mysql_cond_destroy(&COND_wsrep_thd); - if (wsrep_rli) delete wsrep_rli; + if (wsrep_rli != NULL) { + delete wsrep_rli; + wsrep_rli = NULL; + } if (wsrep_status_vars) wsrep->stats_free(wsrep, wsrep_status_vars); #endif /* Close connection */ From 2400b769c641adaaed3badc1f8eeae1de4065c05 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 21 Nov 2016 16:20:10 -0500 Subject: [PATCH 056/203] MDEV-10442: "Address already in use" on restart SST processes should inherit mysqld's process group. --- sql/wsrep_utils.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/wsrep_utils.cc b/sql/wsrep_utils.cc index 951007c2660..14bf4f29794 100644 --- a/sql/wsrep_utils.cc +++ b/sql/wsrep_utils.cc @@ -264,7 +264,6 @@ process::process (const char* cmd, const char* type, char** env) err_ = posix_spawnattr_setflags (&attr, POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK | - /* start a new process group */ POSIX_SPAWN_SETPGROUP | POSIX_SPAWN_USEVFORK); if (err_) { From 4132b1785a51234b4204682419308da0c967e66e Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 23 Jan 2018 12:05:10 -0500 Subject: [PATCH 057/203] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 87b72051a84..44f719ca097 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=59 +MYSQL_VERSION_PATCH=60 MYSQL_VERSION_EXTRA= From 7cec685758c9ae222e30eb799a673b819c91fa0f Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Tue, 30 Jan 2018 05:37:22 -0800 Subject: [PATCH 058/203] Bump wsrep patch version to 25.23 --- cmake/wsrep.cmake | 2 +- scripts/mysql_config.pl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index afab9105d8d..5020a2669b4 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -18,7 +18,7 @@ # so WSREP_VERSION is produced regardless # Set the patch version -SET(WSREP_PATCH_VERSION "22") +SET(WSREP_PATCH_VERSION "23") # MariaDB addition: Revision number of the last revision merged from # codership branch visible in @@visible_comment. diff --git a/scripts/mysql_config.pl b/scripts/mysql_config.pl index a9254bb966e..0a022034a6d 100644 --- a/scripts/mysql_config.pl +++ b/scripts/mysql_config.pl @@ -53,7 +53,7 @@ my $cwd = cwd(); my $basedir; my $socket = '/tmp/mysql.sock'; -my $version = '5.5.58'; +my $version = '5.5.59'; sub which { From 09b25f85966f44aae933e86b84b4ebe59ded47c3 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 1 Mar 2018 16:35:46 +0100 Subject: [PATCH 059/203] only allow SUPER user to modify wsrep_on --- mysql-test/suite/wsrep/r/variables.result | 7 +++++++ mysql-test/suite/wsrep/t/variables.test | 14 ++++++++++++++ sql/sys_vars.cc | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/wsrep/r/variables.result b/mysql-test/suite/wsrep/r/variables.result index 5117dd1b048..72ea8eacfbf 100644 --- a/mysql-test/suite/wsrep/r/variables.result +++ b/mysql-test/suite/wsrep/r/variables.result @@ -73,6 +73,13 @@ Threads_connected 1 SHOW STATUS LIKE 'wsrep_thread_count'; Variable_name Value wsrep_thread_count 11 +set wsrep_on=0; +set wsrep_on=1; +create user test@localhost; +set auto_increment_increment=10; +set wsrep_on=0; +ERROR 42000: Access denied; you need (at least one of) the SUPER privilege(s) for this operation +drop user test@localhost; # # MDEV#6411: Setting set @@global.wsrep_sst_auth=NULL causes crash # diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test index 3093d8ba942..e148152fc22 100644 --- a/mysql-test/suite/wsrep/t/variables.test +++ b/mysql-test/suite/wsrep/t/variables.test @@ -68,6 +68,20 @@ sleep 3; SHOW STATUS LIKE 'threads_connected'; SHOW STATUS LIKE 'wsrep_thread_count'; +# +# privileges for wsrep_on +# +set wsrep_on=0; +set wsrep_on=1; +create user test@localhost; +connect con1,localhost,test; +set auto_increment_increment=10; +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +set wsrep_on=0; +disconnect con1; +connection default; +drop user test@localhost; + --echo # --echo # MDEV#6411: Setting set @@global.wsrep_sst_auth=NULL causes crash --echo # diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 53b606c0336..8a9120abdd0 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3854,7 +3854,7 @@ static Sys_var_mybool Sys_wsrep_on ( "wsrep_on", "To enable wsrep replication ", SESSION_VAR(wsrep_on), CMD_LINE(OPT_ARG), DEFAULT(TRUE), - NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_has_super), ON_UPDATE(wsrep_on_update)); static Sys_var_charptr Sys_wsrep_start_position ( From 9629bca1f0be1bf160c2e8ba205e7bf7fa4ba780 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 10:54:00 +1100 Subject: [PATCH 060/203] MDEV-8743: use O_CLOEXEC (innodb/xtradb) --- storage/innobase/os/os0file.c | 6 +++--- storage/xtradb/os/os0file.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/storage/innobase/os/os0file.c b/storage/innobase/os/os0file.c index a6156e555b5..a410d4644ac 100644 --- a/storage/innobase/os/os0file.c +++ b/storage/innobase/os/os0file.c @@ -1184,10 +1184,10 @@ try_again: } if (create_mode == OS_FILE_CREATE) { - file = open(name, create_flag, S_IRUSR | S_IWUSR + file = open(name, create_flag | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); } else { - file = open(name, create_flag); + file = open(name, create_flag | O_CLOEXEC); } if (file == -1) { @@ -1631,7 +1631,7 @@ try_again: } #endif /* O_SYNC */ - file = open(name, create_flag, os_innodb_umask); + file = open(name, create_flag | O_CLOEXEC, os_innodb_umask); if (file == -1) { *success = FALSE; diff --git a/storage/xtradb/os/os0file.c b/storage/xtradb/os/os0file.c index 8e5cc9a6ba6..d0f014e8f08 100644 --- a/storage/xtradb/os/os0file.c +++ b/storage/xtradb/os/os0file.c @@ -1286,10 +1286,10 @@ try_again: } if (create_mode == OS_FILE_CREATE) { - file = open(name, create_flag, S_IRUSR | S_IWUSR + file = open(name, create_flag | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); } else { - file = open(name, create_flag); + file = open(name, create_flag | O_CLOEXEC); } if (file == -1) { @@ -1752,7 +1752,7 @@ try_again: } #endif /* O_SYNC */ - file = open(name, create_flag, os_innodb_umask); + file = open(name, create_flag | O_CLOEXEC, os_innodb_umask); if (file == -1) { *success = FALSE; From 4ec7b840776e498456b9a22a51178099e4d66aa3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 10:54:34 +1100 Subject: [PATCH 061/203] MDEV-8743: use O_CLOEXEC MYSQL_LOG::open / TC_LOG_MMAP::open For galera compatibility, the main thing is to ensure the FD 1, 2 are not opened with O_CLOEXEC otherwise galera sst errors don't appear in the error.log Files without O_CLOEXEC from the test below: 0 -> /dev/pts/9 1 -> /tmp/error.log (intended) 2 -> /tmp/error.log (intended) 5 -> /tmp/datadir 6 -> /tmp/datadir/aria_log.00000001 (Innodb temp files) 8 -> /tmp/ibIIrhFL (deleted) 9 -> /tmp/ibfx1vai (deleted) 10 -> /tmp/ibAQUKFO (deleted) 11 -> /tmp/ibWBQSHR (deleted) 15 -> /tmp/ibEXEcfo (deleted) 20 -> /tmp/datadir/mysql/host.MYD 22 -> /tmp/datadir/mysql/user.MYD ... (rest of MYD files) Test for this and the previous commit. sql/mysqld --skip-networking --datadir=/tmp/datadir --log-bin=/tmp/datadir/mysqlbin --socket /tmp/s.sock --lc-messages-dir=${PWD}/sql/share --verbose --log-error=/tmp/error.log --general-log-file=/tmp/general.log --general-log=1 --slow-query-log-file=/tmp/slow.log --slow-query-log=1 180302 10:56:41 [Note] sql/mysqld (mysqld 5.5.60-MariaDB-wsrep) starting as process 26056 ... $ cd /proc/26056 $ ls -la --sort=none fd total 0 dr-x------. 2 dan dan 0 Mar 2 10:57 . dr-xr-xr-x. 9 dan dan 0 Mar 2 10:56 .. lrwx------. 1 dan dan 64 Mar 2 10:57 0 -> /dev/pts/9 l-wx------. 1 dan dan 64 Mar 2 10:57 1 -> /tmp/error.log l-wx------. 1 dan dan 64 Mar 2 10:57 2 -> /tmp/error.log lrwx------. 1 dan dan 64 Mar 2 10:57 3 -> /tmp/datadir/mysqlbin.index lrwx------. 1 dan dan 64 Mar 2 10:57 4 -> /tmp/datadir/aria_log_control lr-x------. 1 dan dan 64 Mar 2 10:57 5 -> /tmp/datadir lrwx------. 1 dan dan 64 Mar 2 10:57 6 -> /tmp/datadir/aria_log.00000001 lrwx------. 1 dan dan 64 Mar 2 10:57 7 -> /tmp/datadir/ibdata1 lrwx------. 1 dan dan 64 Mar 2 10:57 8 -> /tmp/ibIIrhFL (deleted) lrwx------. 1 dan dan 64 Mar 2 10:57 9 -> /tmp/ibfx1vai (deleted) lrwx------. 1 dan dan 64 Mar 2 10:57 10 -> /tmp/ibAQUKFO (deleted) lrwx------. 1 dan dan 64 Mar 2 10:57 11 -> /tmp/ibWBQSHR (deleted) lrwx------. 1 dan dan 64 Mar 2 10:57 12 -> /tmp/datadir/ib_logfile0 lrwx------. 1 dan dan 64 Mar 2 10:57 13 -> /tmp/datadir/ib_logfile1 l-wx------. 1 dan dan 64 Mar 2 10:57 14 -> /tmp/slow.log lrwx------. 1 dan dan 64 Mar 2 10:57 15 -> /tmp/ibEXEcfo (deleted) l-wx------. 1 dan dan 64 Mar 2 10:57 16 -> /tmp/general.log lrwx------. 1 dan dan 64 Mar 2 10:57 17 -> socket:[1897356] lrwx------. 1 dan dan 64 Mar 2 10:57 18 -> socket:[45335] l-wx------. 1 dan dan 64 Mar 2 10:57 19 -> /tmp/datadir/mysqlbin.000004 lrwx------. 1 dan dan 64 Mar 2 10:57 20 -> /tmp/datadir/mysql/host.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 21 -> /tmp/datadir/mysql/host.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 22 -> /tmp/datadir/mysql/user.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 23 -> /tmp/datadir/mysql/user.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 24 -> /tmp/datadir/mysql/db.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 25 -> /tmp/datadir/mysql/db.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 26 -> /tmp/datadir/mysql/proxies_priv.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 27 -> /tmp/datadir/mysql/proxies_priv.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 28 -> /tmp/datadir/mysql/tables_priv.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 29 -> /tmp/datadir/mysql/tables_priv.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 30 -> /tmp/datadir/mysql/columns_priv.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 31 -> /tmp/datadir/mysql/columns_priv.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 32 -> /tmp/datadir/mysql/procs_priv.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 33 -> /tmp/datadir/mysql/procs_priv.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 34 -> /tmp/datadir/mysql/servers.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 35 -> /tmp/datadir/mysql/servers.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 36 -> /tmp/datadir/mysql/event.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 37 -> /tmp/datadir/mysql/event.MYI O_CLOEXEC files are those with flags 02000000 /usr/include/bits/fcntl-linux.h:# define __O_CLOEXEC 02000000 /usr/include/bits/fcntl-linux.h:# define O_CLOEXEC __O_CLOEXEC /* Set close_on_exec. */ $ find fdinfo/ -type f -ls -exec cat {} \; | more 1924720 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/0 pos: 0 flags: 0100002 mnt_id: 25 1924721 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/1 pos: 9954 flags: 0102001 mnt_id: 82 1924722 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/2 pos: 10951 flags: 0102001 mnt_id: 82 1924723 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/3 pos: 116 flags: 02100002 mnt_id: 82 1924724 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/4 pos: 52 flags: 0100002 mnt_id: 82 lock: 1: POSIX ADVISORY WRITE 26056 00:2c:1866365 0 EOF 1924725 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/5 pos: 0 flags: 0100000 mnt_id: 82 1924726 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/6 pos: 16384 flags: 0100002 mnt_id: 82 1924727 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/7 pos: 0 flags: 02100002 mnt_id: 82 lock: 1: POSIX ADVISORY WRITE 26056 00:2c:1866491 0 EOF 1924728 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/8 pos: 0 flags: 0100002 mnt_id: 82 1924729 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/9 pos: 0 flags: 0100002 mnt_id: 82 1924730 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/10 pos: 0 flags: 0100002 mnt_id: 82 1924731 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/11 pos: 0 flags: 0100002 mnt_id: 82 1924732 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/12 pos: 0 flags: 02100002 mnt_id: 82 lock: 1: POSIX ADVISORY WRITE 26056 00:2c:1866492 0 EOF 1924733 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/13 pos: 0 flags: 02100002 mnt_id: 82 lock: 1: POSIX ADVISORY WRITE 26056 00:2c:1866493 0 EOF 1924734 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/14 pos: 763 flags: 02102001 mnt_id: 82 1924735 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/15 pos: 0 flags: 0100002 mnt_id: 82 1924736 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/16 pos: 473 flags: 02102001 mnt_id: 82 1924737 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/17 pos: 0 flags: 02000002 mnt_id: 9 1924738 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/18 pos: 0 flags: 02 mnt_id: 9 1924739 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/19 pos: 245 flags: 02100001 mnt_id: 82 1924740 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/20 pos: 0 flags: 0100002 mnt_id: 82 1924741 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/21 pos: 503 flags: 0500002 mnt_id: 82 1924742 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/22 pos: 324 flags: 0100002 mnt_id: 82 1924743 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/23 pos: 642 flags: 0500002 mnt_id: 82 1924744 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/24 pos: 880 flags: 0100002 mnt_id: 82 1924745 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/25 pos: 581 flags: 0500002 mnt_id: 82 1924746 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/26 pos: 1386 flags: 0100002 mnt_id: 82 1924747 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/27 pos: 498 flags: 0500002 mnt_id: 82 1924748 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/28 pos: 0 flags: 0100002 mnt_id: 82 1924749 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/29 pos: 513 flags: 0500002 mnt_id: 82 1924750 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/30 pos: 0 flags: 0100002 mnt_id: 82 1924751 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/31 pos: 494 flags: 0500002 mnt_id: 82 1924752 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/32 pos: 0 flags: 0100002 mnt_id: 82 1924753 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/33 pos: 535 flags: 0500002 mnt_id: 82 1924754 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/34 pos: 0 flags: 0100002 mnt_id: 82 1924755 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/35 pos: 396 flags: 0500002 mnt_id: 82 1924756 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/36 pos: 0 flags: 0100002 mnt_id: 82 1924757 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/37 pos: 517 flags: 0500002 mnt_id: 82 --- mysys/my_symlink.c | 2 +- sql/log.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mysys/my_symlink.c b/mysys/my_symlink.c index 12959f1f24a..f2dd2e596ff 100644 --- a/mysys/my_symlink.c +++ b/mysys/my_symlink.c @@ -236,7 +236,7 @@ const char *my_open_parent_dir_nosymlinks(const char *pathname, int *pdfd) return pathname + (s - buf); } - fd = openat(dfd, s, O_NOFOLLOW | O_PATH); + fd = openat(dfd, s, O_NOFOLLOW | O_PATH | O_CLOEXEC); if (fd < 0) goto err; diff --git a/sql/log.cc b/sql/log.cc index 4760e668aeb..ca7833a0460 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -2466,7 +2466,7 @@ bool MYSQL_LOG::open( File file= -1; my_off_t seek_offset; bool is_fifo = false; - int open_flags= O_CREAT | O_BINARY; + int open_flags= O_CREAT | O_BINARY | O_CLOEXEC; DBUG_ENTER("MYSQL_LOG::open"); DBUG_PRINT("enter", ("log_type: %d", (int) log_type_arg)); @@ -3088,7 +3088,7 @@ bool MYSQL_BIN_LOG::open_index_file(const char *index_file_name_arg, ".index", opt); if ((index_file_nr= mysql_file_open(m_key_file_log_index, index_file_name, - O_RDWR | O_CREAT | O_BINARY, + O_RDWR | O_CREAT | O_BINARY | O_CLOEXEC, MYF(MY_WME))) < 0 || mysql_file_sync(index_file_nr, MYF(MY_WME)) || init_io_cache(&index_file, index_file_nr, @@ -7045,14 +7045,14 @@ int TC_LOG_MMAP::open(const char *opt_name) DBUG_ASSERT(TC_LOG_PAGE_SIZE % tc_log_page_size == 0); fn_format(logname,opt_name,mysql_data_home,"",MY_UNPACK_FILENAME); - if ((fd= mysql_file_open(key_file_tclog, logname, O_RDWR, MYF(0))) < 0) + if ((fd= mysql_file_open(key_file_tclog, logname, O_RDWR | O_CLOEXEC, MYF(0))) < 0) { if (my_errno != ENOENT) goto err; if (using_heuristic_recover()) return 1; if ((fd= mysql_file_create(key_file_tclog, logname, CREATE_MODE, - O_RDWR, MYF(MY_WME))) < 0) + O_RDWR | O_CLOEXEC, MYF(MY_WME))) < 0) goto err; inited=1; file_length= opt_tc_log_size; From c54c490c59e2d6aed21934cfd5948974ee0ee774 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Fri, 2 Mar 2018 11:17:35 +1100 Subject: [PATCH 062/203] MDEV-8743: O_CLOEXEC/SOCK_CLOEXEC defines for non-unix compatibility From 0f4478105f5027ce589a0f08c6019d906d632029 --- include/my_global.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/my_global.h b/include/my_global.h index cf140cf54ce..6e8fb33b137 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -578,6 +578,12 @@ typedef SOCKET_SIZE_TYPE size_socket; #ifndef O_NOFOLLOW #define O_NOFOLLOW 0 #endif +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif +#ifndef SOCK_CLOEXEC +#define SOCK_CLOEXEC 0 +#endif /* additional file share flags for win32 */ #ifdef __WIN__ From 88fb8b2e367158dce7a4c3e14f2517c5deb9b51d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 11:52:20 +1100 Subject: [PATCH 063/203] MDEV-8743: protect myisam/aria MYD files and aria log files --- storage/maria/ma_control_file.c | 2 +- storage/maria/ma_loghandler.c | 6 +++--- storage/maria/ma_open.c | 4 ++-- storage/myisam/mi_open.c | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/storage/maria/ma_control_file.c b/storage/maria/ma_control_file.c index 2fa3e6ed18e..11cb4c5fd24 100644 --- a/storage/maria/ma_control_file.c +++ b/storage/maria/ma_control_file.c @@ -273,7 +273,7 @@ CONTROL_FILE_ERROR ma_control_file_open(my_bool create_if_missing, " file is probably in use by another process"; uint new_cf_create_time_size, new_cf_changeable_size, new_block_size; my_off_t file_size; - int open_flags= O_BINARY | /*O_DIRECT |*/ O_RDWR; + int open_flags= O_BINARY | /*O_DIRECT |*/ O_RDWR | O_CLOEXEC; int error= CONTROL_FILE_UNKNOWN_ERROR; DBUG_ENTER("ma_control_file_open"); diff --git a/storage/maria/ma_loghandler.c b/storage/maria/ma_loghandler.c index 2118b3b6ce6..f31662ce003 100644 --- a/storage/maria/ma_loghandler.c +++ b/storage/maria/ma_loghandler.c @@ -942,7 +942,7 @@ static File create_logfile_by_number_no_cache(uint32 file_no) /* TODO: add O_DIRECT to open flags (when buffer is aligned) */ if ((file= mysql_file_create(key_file_translog, translog_filename_by_fileno(file_no, path), - 0, O_BINARY | O_RDWR, MYF(MY_WME))) < 0) + 0, O_BINARY | O_RDWR | O_CLOEXEC, MYF(MY_WME))) < 0) { DBUG_PRINT("error", ("Error %d during creating file '%s'", errno, path)); translog_stop_writing(); @@ -979,7 +979,7 @@ static File open_logfile_by_number_no_cache(uint32 file_no) /* TODO: use mysql_file_create() */ if ((file= mysql_file_open(key_file_translog, translog_filename_by_fileno(file_no, path), - log_descriptor.open_flags, + log_descriptor.open_flags | O_CLOEXEC, MYF(MY_WME))) < 0) { DBUG_PRINT("error", ("Error %d during opening file '%s'", errno, path)); @@ -3263,7 +3263,7 @@ static my_bool translog_get_last_page_addr(TRANSLOG_ADDRESS *addr, File fd; if ((fd= mysql_file_open(key_file_translog, translog_filename_by_fileno(file_no, path), - O_RDONLY, (no_errors ? MYF(0) : MYF(MY_WME)))) < 0) + O_RDONLY | O_CLOEXEC, (no_errors ? MYF(0) : MYF(MY_WME)))) < 0) { my_errno= errno; DBUG_PRINT("error", ("Error %d during opening file #%d", diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c index a382c30e025..d20fa06e069 100644 --- a/storage/maria/ma_open.c +++ b/storage/maria/ma_open.c @@ -1885,7 +1885,7 @@ int _ma_open_datafile(MARIA_HA *info, MARIA_SHARE *share) DEBUG_SYNC_C("mi_open_datafile"); info->dfile.file= share->bitmap.file.file= mysql_file_open(key_file_dfile, share->data_file_name.str, - share->mode | O_SHARE, MYF(flags)); + share->mode | O_SHARE | O_CLOEXEC, MYF(flags)); return info->dfile.file >= 0 ? 0 : 1; } @@ -1899,7 +1899,7 @@ int _ma_open_keyfile(MARIA_SHARE *share) mysql_mutex_lock(&share->intern_lock); share->kfile.file= mysql_file_open(key_file_kfile, share->unique_file_name.str, - share->mode | O_SHARE | O_NOFOLLOW, + share->mode | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_WME | MY_NOSYMLINKS)); mysql_mutex_unlock(&share->intern_lock); return (share->kfile.file < 0); diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c index 86b70129203..8e739109c0d 100644 --- a/storage/myisam/mi_open.c +++ b/storage/myisam/mi_open.c @@ -1249,7 +1249,7 @@ int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share) myf flags= MY_WME | (share->mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0); DEBUG_SYNC_C("mi_open_datafile"); info->dfile= mysql_file_open(mi_key_file_dfile, share->data_file_name, - share->mode | O_SHARE, MYF(flags)); + share->mode | O_SHARE | O_CLOEXEC, MYF(flags)); return info->dfile >= 0 ? 0 : 1; } @@ -1258,7 +1258,7 @@ int mi_open_keyfile(MYISAM_SHARE *share) { if ((share->kfile= mysql_file_open(mi_key_file_kfile, share->unique_file_name, - share->mode | O_SHARE | O_NOFOLLOW, + share->mode | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS | MY_WME))) < 0) return 1; return 0; From bbee025370d885103cba55716c9c2ab44e533283 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 12:45:36 +1100 Subject: [PATCH 064/203] MDEV-8743: O_CLOEXEC on innodb/xtradb temp files Thread 1 "mysqld" hit Breakpoint 1, innobase_mysql_tmpfile () at /home/dan/repos/mariadb-server-5.5/storage/xtradb/handler/ha_innodb.cc:1639 1639 os_event_wait(srv_allow_writes_event); (gdb) p fd $2 = 7 (gdb) n 1682 fd2 = fcntl(fd, F_DUPFD_CLOEXEC, 0); (gdb) 1687 if (fd2 < 0) { (gdb) p fd2 $3 = 8 cat /proc/20448/fdinfo/{7,8} 2051972 0 -r-------- 1 dan dan 0 Mar 2 12:04 fdinfo/7 pos: 0 flags: 0100002 mnt_id: 82 2051973 0 -r-------- 1 dan dan 0 Mar 2 12:04 fdinfo/8 pos: 0 flags: 02100002 mnt_id: 82 So fd 8 has 02000000 hence CLOEXEC --- storage/innobase/handler/ha_innodb.cc | 4 ++++ storage/xtradb/handler/ha_innodb.cc | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3a96763d6c2..c9eee5b4dcd 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1410,8 +1410,12 @@ innobase_mysql_tmpfile(void) fd2 = -1; } } +#else +#ifdef F_DUPFD_CLOEXEC + fd2 = fcntl(fd, F_DUPFD_CLOEXEC, 0); #else fd2 = dup(fd); +#endif #endif if (fd2 < 0) { DBUG_PRINT("error",("Got error %d on dup",fd2)); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index f99ae7c7de9..04fbcd1a9fd 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1677,8 +1677,12 @@ innobase_mysql_tmpfile(void) fd2 = -1; } } +#else +#ifdef F_DUPFD_CLOEXEC + fd2 = fcntl(fd, F_DUPFD_CLOEXEC, 0); #else fd2 = dup(fd); +#endif #endif if (fd2 < 0) { DBUG_PRINT("error",("Got error %d on dup",fd2)); From d3c0e34bdc99ee57fd5d63237d29531d1afdef2b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 16:19:14 +1100 Subject: [PATCH 065/203] MDEV-8743: protect myisam/aria MYI with O_CLOEXEC --- storage/maria/ma_open.c | 4 ++-- storage/myisam/mi_open.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c index d20fa06e069..d9ff2179691 100644 --- a/storage/maria/ma_open.c +++ b/storage/maria/ma_open.c @@ -314,13 +314,13 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) }); DEBUG_SYNC_C("mi_open_kfile"); if ((kfile=mysql_file_open(key_file_kfile, name_buff, - (open_mode=O_RDWR) | O_SHARE | O_NOFOLLOW, + (open_mode=O_RDWR) | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) { if ((errno != EROFS && errno != EACCES) || mode != O_RDONLY || (kfile=mysql_file_open(key_file_kfile, name_buff, - (open_mode=O_RDONLY) | O_SHARE | O_NOFOLLOW, + (open_mode=O_RDONLY) | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) goto err; } diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c index 8e739109c0d..9775aade9c1 100644 --- a/storage/myisam/mi_open.c +++ b/storage/myisam/mi_open.c @@ -117,13 +117,13 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) DEBUG_SYNC_C("mi_open_kfile"); if ((kfile= mysql_file_open(mi_key_file_kfile, name_buff, - (open_mode= O_RDWR) | O_SHARE | O_NOFOLLOW, + (open_mode= O_RDWR) | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) { if ((errno != EROFS && errno != EACCES) || mode != O_RDONLY || (kfile= mysql_file_open(mi_key_file_kfile, name_buff, - (open_mode= O_RDONLY) | O_SHARE| O_NOFOLLOW, + (open_mode= O_RDONLY) | O_SHARE| O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) goto err; } From ccd566af20dc47a5708e2e707c9624796ea79bef Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 19 Apr 2018 07:38:57 +1000 Subject: [PATCH 066/203] MDEV-8743: mysqld port/socket - FD_CLOEXEC if no SOCK_CLOEXEC In MDEV-8743, the port/socket of mysqld was changed to set FD_CLOEXEC. The existing mysql_socket_socket function already set that with SOCK_CLOEXEC when the socket was created. So here we move the fcntl functionality to the mysql_socket_socket as port/socket are the only callers. Preprocessor checks of SOCK_CLOEXEC cannot be done as its a 0 if not there and SOCK_CLOEXEC (being the value of the enum in bits/socket_type.h) Preprocesssor logic for arithmetic and non-arithmetic defines are hard/nonportable/ugly to read. As such we just check in my_global.h and define HAVE_SOCK_CLOEXEC if we have it. There was a disparity in behaviour between defined(WITH_WSREP) and not depending on the OS, so the WITH_WSREP condition was removed from setting calling fcntl. All sockets are now maked SOCK_CLOEXEC/FD_CLOEXEC. strace of mysqld with SOCK_CLOEXEC: socket(AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 10 write(2, "180419 14:52:40 [Note] Server socket created on IP: '127.0.0.1'.\n", 65180419 14:52:40 [Note] Server socket created on IP: '127.0.0.1'. ) = 65 setsockopt(10, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 bind(10, {sa_family=AF_INET, sin_port=htons(16020), sin_addr=inet_addr("127.0.0.1")}, 16) = 0 listen(10, 150) = 0 socket(AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 11 write(2, "180419 14:52:40 [Note] Server socket created on IP: '127.0.0.1'.\n", 65180419 14:52:40 [Note] Server socket created on IP: '127.0.0.1'. ) = 65 setsockopt(11, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 bind(11, {sa_family=AF_INET, sin_port=htons(16021), sin_addr=inet_addr("127.0.0.1")}, 16) = 0 listen(11, 150) = 0 socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0) = 12 unlink("/home/dan/repos/build-mariadb-server-10.0/mysql-test/var/tmp/mysqld.1.sock") = -1 ENOENT (No such file or directory) setsockopt(12, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 umask(000) = 006 bind(12, {sa_family=AF_UNIX, sun_path="/home/dan/repos/build-mariadb-server-10.0/mysql-test/var/tmp/mysqld.1.sock"}, 110) = 0 umask(006) = 000 listen(12, 150) = 0 --- include/my_global.h | 2 ++ include/mysql/psi/mysql_socket.h | 6 ++++++ sql/mysqld.cc | 7 ------- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/include/my_global.h b/include/my_global.h index ad1527d7b37..057c613d64c 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -608,6 +608,8 @@ typedef SOCKET_SIZE_TYPE size_socket; #endif #ifndef SOCK_CLOEXEC #define SOCK_CLOEXEC 0 +#else +#define HAVE_SOCK_CLOEXEC #endif /* additional file share flags for win32 */ diff --git a/include/mysql/psi/mysql_socket.h b/include/mysql/psi/mysql_socket.h index bd05ef9b8a9..70700a733d8 100644 --- a/include/mysql/psi/mysql_socket.h +++ b/include/mysql/psi/mysql_socket.h @@ -562,6 +562,12 @@ inline_mysql_socket_socket (key, (const my_socket*)&mysql_socket.fd, NULL, 0); } #endif + + /* SOCK_CLOEXEC isn't always a number - can't preprocessor compare */ +#if defined(HAVE_FCNTL) && defined(FD_CLOEXEC) && !defined(HAVE_SOCK_CLOEXEC) + (void) fcntl(mysql_socket.fd, F_SETFD, FD_CLOEXEC); +#endif + return mysql_socket; } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 8575709203c..15f4af5edbb 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2593,10 +2593,6 @@ static MYSQL_SOCKET activate_tcp_port(uint port) socket_errno); unireg_abort(1); } -#if defined(WITH_WSREP) && defined(HAVE_FCNTL) && defined(FD_CLOEXEC) - (void) fcntl(mysql_socket_getfd(ip_sock), F_SETFD, FD_CLOEXEC); -#endif /* WITH_WSREP */ - DBUG_RETURN(ip_sock); } @@ -2714,9 +2710,6 @@ static void network_init(void) if (mysql_socket_listen(unix_sock,(int) back_log) < 0) sql_print_warning("listen() on Unix socket failed with error %d", socket_errno); -#if defined(WITH_WSREP) && defined(HAVE_FCNTL) - (void) fcntl(mysql_socket_getfd(unix_sock), F_SETFD, FD_CLOEXEC); -#endif /* WITH_WSREP */ } #endif DBUG_PRINT("info",("server started")); From c2c61bbccea3525e12d8897c009898138a335392 Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Sun, 17 Dec 2017 14:41:55 +0200 Subject: [PATCH 067/203] Provider rollback for ineffective trx codership/mysql-wsrep#318 Adapt MTR tests to new Galera status variables and fix exposed leaks In certain cases it is possible that transaction has populated keys in the provider but during commit time the IO cache is empty, so the provider commit does not happen (for example early ROLLBACK TO SAVEPOINT followed by COMMIT). Run provider post_rollback() to clean up the transaction object. --- sql/wsrep_hton.cc | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/sql/wsrep_hton.cc b/sql/wsrep_hton.cc index 0572230b18b..c18c179a4bd 100644 --- a/sql/wsrep_hton.cc +++ b/sql/wsrep_hton.cc @@ -99,16 +99,45 @@ void wsrep_register_hton(THD* thd, bool all) */ void wsrep_post_commit(THD* thd, bool all) { - if (thd->wsrep_exec_mode == LOCAL_COMMIT) + if (!WSREP(thd)) return; + + switch (thd->wsrep_exec_mode) { - DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED); - if (wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) + case LOCAL_COMMIT: { + DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED); + if (wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) + { DBUG_PRINT("wsrep", ("set committed fail")); - WSREP_WARN("set committed fail: %llu %d", + WSREP_WARN("set committed fail: %llu %d", (long long)thd->real_id, thd->stmt_da->status()); + } + wsrep_cleanup_transaction(thd); + break; } - wsrep_cleanup_transaction(thd); + case LOCAL_STATE: + { + /* non-InnoDB statements may have populated events in stmt cache + => cleanup + */ + WSREP_DEBUG("cleanup transaction for LOCAL_STATE"); + /* + Run post-rollback hook to clean up in the case if + some keys were populated for the transaction in provider + but during commit time there was no write set to replicate. + This may happen when client sets the SAVEPOINT and immediately + rolls back to savepoint after first operation. + */ + if (all && thd->wsrep_conflict_state != MUST_REPLAY && + wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) + { + WSREP_WARN("post_rollback fail: %llu %d", + (long long)thd->thread_id, thd->stmt_da->status()); + } + wsrep_cleanup_transaction(thd); + break; + } + default: break; } } From 231c02f7b9fc05ffa07dacd5f31bbd2ff14e7cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 24 Apr 2018 13:58:42 +0300 Subject: [PATCH 068/203] MariaDB adjustments. --- scripts/mysql_config.pl | 2 +- sql/wsrep_hton.cc | 48 ++++++++++++++++++++--------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/scripts/mysql_config.pl b/scripts/mysql_config.pl index 0a022034a6d..113d8fc10be 100644 --- a/scripts/mysql_config.pl +++ b/scripts/mysql_config.pl @@ -53,7 +53,7 @@ my $cwd = cwd(); my $basedir; my $socket = '/tmp/mysql.sock'; -my $version = '5.5.59'; +my $version = '5.5.60'; sub which { diff --git a/sql/wsrep_hton.cc b/sql/wsrep_hton.cc index c18c179a4bd..1676daab5fe 100644 --- a/sql/wsrep_hton.cc +++ b/sql/wsrep_hton.cc @@ -103,10 +103,10 @@ void wsrep_post_commit(THD* thd, bool all) switch (thd->wsrep_exec_mode) { - case LOCAL_COMMIT: + case LOCAL_COMMIT: { DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED); - if (wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) + if (wsrep && wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) { DBUG_PRINT("wsrep", ("set committed fail")); WSREP_WARN("set committed fail: %llu %d", @@ -115,29 +115,29 @@ void wsrep_post_commit(THD* thd, bool all) wsrep_cleanup_transaction(thd); break; } - case LOCAL_STATE: - { - /* non-InnoDB statements may have populated events in stmt cache - => cleanup - */ - WSREP_DEBUG("cleanup transaction for LOCAL_STATE"); - /* - Run post-rollback hook to clean up in the case if - some keys were populated for the transaction in provider - but during commit time there was no write set to replicate. - This may happen when client sets the SAVEPOINT and immediately - rolls back to savepoint after first operation. + case LOCAL_STATE: + { + /* non-InnoDB statements may have populated events in stmt cache + => cleanup */ - if (all && thd->wsrep_conflict_state != MUST_REPLAY && - wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) - { - WSREP_WARN("post_rollback fail: %llu %d", - (long long)thd->thread_id, thd->stmt_da->status()); - } - wsrep_cleanup_transaction(thd); - break; - } - default: break; + WSREP_DEBUG("cleanup transaction for LOCAL_STATE"); + /* + Run post-rollback hook to clean up in the case if + some keys were populated for the transaction in provider + but during commit time there was no write set to replicate. + This may happen when client sets the SAVEPOINT and immediately + rolls back to savepoint after first operation. + */ + if (all && thd->wsrep_conflict_state != MUST_REPLAY && + wsrep && wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) + { + WSREP_WARN("post_rollback fail: %llu %d", + (long long)thd->thread_id, thd->stmt_da->status()); + } + wsrep_cleanup_transaction(thd); + break; + } + default: break; } } From 75dec85c2ef7383cb452fcd4e5b0fe06c0016bc0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 27 Apr 2018 11:21:55 +0200 Subject: [PATCH 069/203] Bug#25471090: MYSQL USE AFTER FREE fix another similar line followup for 7828ba0df488 --- sql-common/client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql-common/client.c b/sql-common/client.c index bb7bdb1ff7d..9cb3311d2e1 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -1636,7 +1636,7 @@ MYSQL_DATA *cli_read_rows(MYSQL *mysql,MYSQL_FIELD *mysql_fields, else { cur->data[field] = to; - if (len > (ulong) (end_to - to)) + if (to + len > end_to) { free_rows(result); set_mysql_error(mysql, CR_MALFORMED_PACKET, unknown_sqlstate); From ccad629d7e453b06b16193deb4685fc10908f92f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 30 Apr 2018 13:50:59 +0200 Subject: [PATCH 070/203] Bug#25471090: MYSQL USE AFTER FREE a better fix --- sql-common/client.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql-common/client.c b/sql-common/client.c index 9cb3311d2e1..b485ebf4f60 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -1636,7 +1636,7 @@ MYSQL_DATA *cli_read_rows(MYSQL *mysql,MYSQL_FIELD *mysql_fields, else { cur->data[field] = to; - if (to + len > end_to) + if (unlikely(len > (ulong)(end_to-to) || to > end_to)) { free_rows(result); set_mysql_error(mysql, CR_MALFORMED_PACKET, unknown_sqlstate); @@ -1708,7 +1708,7 @@ read_one_row(MYSQL *mysql,uint fields,MYSQL_ROW row, ulong *lengths) } else { - if (pos + len > end_pos) + if (unlikely(len > (ulong)(end_pos - pos) || pos > end_pos)) { set_mysql_error(mysql, CR_UNKNOWN_ERROR, unknown_sqlstate); return -1; From 1ecd68d867ced1d00ebffdcedbf6bc97493f5067 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 30 Apr 2018 23:06:09 +0200 Subject: [PATCH 071/203] Use after free in authentication --- mysql-test/r/connect_debug.result | 5 +++++ mysql-test/t/connect_debug.test | 12 ++++++++++++ sql-common/client.c | 4 ++-- sql/sql_acl.cc | 1 + 4 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 mysql-test/r/connect_debug.result create mode 100644 mysql-test/t/connect_debug.test diff --git a/mysql-test/r/connect_debug.result b/mysql-test/r/connect_debug.result new file mode 100644 index 00000000000..0452b238db9 --- /dev/null +++ b/mysql-test/r/connect_debug.result @@ -0,0 +1,5 @@ +set @old_dbug=@@global.debug_dbug; +set global debug_dbug='+d,auth_disconnect'; +create user 'bad' identified by 'worse'; +set global debug_dbug=@old_dbug; +drop user bad; diff --git a/mysql-test/t/connect_debug.test b/mysql-test/t/connect_debug.test new file mode 100644 index 00000000000..299b605b2cd --- /dev/null +++ b/mysql-test/t/connect_debug.test @@ -0,0 +1,12 @@ +source include/have_debug.inc; +set @old_dbug=@@global.debug_dbug; + +# +# use after free if need plugin change and auth aborted +# +set global debug_dbug='+d,auth_disconnect'; +create user 'bad' identified by 'worse'; +--error 1 +--exec $MYSQL --default-auth=mysql_old_password --user=bad --password=worse +set global debug_dbug=@old_dbug; +drop user bad; diff --git a/sql-common/client.c b/sql-common/client.c index b485ebf4f60..00e2877bedb 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -2742,7 +2742,7 @@ static int client_mpvio_read_packet(struct st_plugin_vio *mpv, uchar **buf) *buf= mysql->net.read_pos; /* was it a request to change plugins ? */ - if (**buf == 254) + if (pkt_len == packet_error || **buf == 254) return (int)packet_error; /* if yes, this plugin shan't continue */ /* @@ -2927,7 +2927,7 @@ int run_plugin_auth(MYSQL *mysql, char *data, uint data_len, compile_time_assert(CR_OK == -1); compile_time_assert(CR_ERROR == 0); - if (res > CR_OK && mysql->net.read_pos[0] != 254) + if (res > CR_OK && (mysql->net.last_errno || mysql->net.read_pos[0] != 254)) { /* the plugin returned an error. write it down in mysql, diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index c3740f8ab29..15a238193dd 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -8253,6 +8253,7 @@ static bool send_plugin_request_packet(MPVIO_EXT *mpvio, const char *client_auth_plugin= ((st_mysql_auth *) (plugin_decl(mpvio->plugin)->info))->client_auth_plugin; + DBUG_EXECUTE_IF("auth_disconnect", { vio_close(net->vio); DBUG_RETURN(1); }); DBUG_ASSERT(client_auth_plugin); /* From 0088fb91f36574ea79a575282951e956ab7f15f8 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Fri, 2 Feb 2018 14:18:39 +0100 Subject: [PATCH 072/203] Fix sporadic failure of MTR test galera.galera_many_tables_pk The test relies on the false assumption that node_1 is always "assigned" autoincrement value 1 for a given key. However that is no necessarily the case when wsrep autoincrement control is enabled. This patch changed the test so that it does not make use of a autoincrement key, which is not fundamental for the test itself. codership/mysql-wsrep#322 --- mysql-test/suite/galera/t/galera_many_tables_pk.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_many_tables_pk.test b/mysql-test/suite/galera/t/galera_many_tables_pk.test index 72c7e08bcc3..0af7160d4d8 100644 --- a/mysql-test/suite/galera/t/galera_many_tables_pk.test +++ b/mysql-test/suite/galera/t/galera_many_tables_pk.test @@ -16,7 +16,7 @@ while ($count) { --disable_query_log - --let $ddl_var = `SELECT CONCAT("CREATE TABLE t", $count, " (f1 INTEGER AUTO_INCREMENT PRIMARY KEY) ENGINE=InnoDB")` + --let $ddl_var = `SELECT CONCAT("CREATE TABLE t", $count, " (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB")` --eval $ddl_var --enable_query_log --dec $count @@ -37,7 +37,7 @@ START TRANSACTION; while ($count) { --disable_query_log - --let $ddl_var = `SELECT CONCAT("INSERT INTO t", $count, " VALUES (DEFAULT)")` + --let $ddl_var = `SELECT CONCAT("INSERT INTO t", $count, " VALUES (1)")` --eval $ddl_var --enable_query_log --dec $count From b1fabb2c0aecc4518a04341325abdbf26e341754 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 7 Feb 2018 14:14:21 +0100 Subject: [PATCH 073/203] Galera MTR Tests: missing wsrep_sync_wait in test galera_evs_suspect_timeout --- .../suite/galera_3nodes/r/galera_evs_suspect_timeout.result | 1 + mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result b/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result index 1464222a079..7e0d282ec7f 100644 --- a/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result +++ b/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result @@ -11,6 +11,7 @@ SET SESSION wsrep_sync_wait = 0; SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; VARIABLE_VALUE = 2 1 +SET SESSION wsrep_sync_wait = DEFAULT; SELECT COUNT(*) = 1 FROM t1; COUNT(*) = 1 1 diff --git a/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test b/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test index a87f19ac94e..03236a3cb93 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test +++ b/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test @@ -50,6 +50,7 @@ SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_N --enable_query_log --source include/wait_until_connected_again.inc +SET SESSION wsrep_sync_wait = DEFAULT; SELECT COUNT(*) = 1 FROM t1; DROP TABLE t1; From ba07581c811a3991116605f51143a5f3e3dd8b7f Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 7 Feb 2018 14:16:50 +0100 Subject: [PATCH 074/203] Galera MTR tests: remove unused config files in galera suite --- .../galera/galera_2nodes_as_master_slave.cnf | 83 ------------------ ...lera_2nodes_as_master_with_repl_filter.cnf | 87 ------------------- 2 files changed, 170 deletions(-) delete mode 100644 mysql-test/suite/galera/galera_2nodes_as_master_slave.cnf delete mode 100644 mysql-test/suite/galera/galera_2nodes_as_master_with_repl_filter.cnf diff --git a/mysql-test/suite/galera/galera_2nodes_as_master_slave.cnf b/mysql-test/suite/galera/galera_2nodes_as_master_slave.cnf deleted file mode 100644 index f68fe524904..00000000000 --- a/mysql-test/suite/galera/galera_2nodes_as_master_slave.cnf +++ /dev/null @@ -1,83 +0,0 @@ - -# -# Let's understand the topology. -# * Independent Master with server-id = 1 -# * Galera cluster with 2 nodes: node#1 and node#2 with server-id = 2, 3 -# node#1 act as slave to Independent Master with server-id = 1 -# * Independent Slave with server-id = 4 replicating from galera node#2 -# - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf - -[mysqld] -log-slave-updates -log-bin=mysqld-bin -binlog-format=row -gtid-mode=on -enforce-gtid-consistency=true - -[mysqld.1] -server-id=1 - -[mysqld.2] -server-id=2 - -wsrep_provider=@ENV.WSREP_PROVIDER -wsrep_cluster_address='gcomm://' -wsrep_provider_options='base_port=@mysqld.2.#galera_port;evs.install_timeout = PT15S; evs.max_install_timeouts=1;' - -# enforce read-committed characteristics across the cluster -wsrep_causal_reads=ON -wsrep_sync_wait = 15 - -wsrep_node_address=127.0.0.1 -wsrep_sst_receive_address=127.0.0.2:@mysqld.2.#sst_port -wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port - -# Required for Galera -innodb_autoinc_lock_mode=2 - -innodb_flush_log_at_trx_commit=2 - -[mysqld.3] -server-id=3 - -wsrep_provider=@ENV.WSREP_PROVIDER -wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.2.#galera_port' -wsrep_provider_options='base_port=@mysqld.3.#galera_port;evs.install_timeout = PT15S; evs.max_install_timeouts = 1;' - -# enforce read-committed characteristics across the cluster -wsrep_causal_reads=ON -wsrep_sync_wait = 15 - -wsrep_node_address=127.0.0.1 -wsrep_sst_receive_address=127.0.0.2:@mysqld.3.#sst_port -wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port - -# Required for Galera -innodb_autoinc_lock_mode=2 - -innodb_flush_log_at_trx_commit=2 - -[mysqld.4] -server-id=4 - -[ENV] -NODE_MYPORT_1= @mysqld.1.port -NODE_MYSOCK_1= @mysqld.1.socket - -NODE_MYPORT_2= @mysqld.2.port -NODE_MYSOCK_2= @mysqld.2.socket - -NODE_MYPORT_3= @mysqld.3.port -NODE_MYSOCK_3= @mysqld.3.socket - -NODE_MYPORT_4= @mysqld.4.port -NODE_MYSOCK_4= @mysqld.4.socket - -NODE_GALERAPORT_2= @mysqld.2.#galera_port -NODE_GALERAPORT_3= @mysqld.3.#galera_port - -NODE_SSTPORT_2= @mysqld.2.#sst_port -NODE_SSTPORT_3= @mysqld.3.#sst_port diff --git a/mysql-test/suite/galera/galera_2nodes_as_master_with_repl_filter.cnf b/mysql-test/suite/galera/galera_2nodes_as_master_with_repl_filter.cnf deleted file mode 100644 index d5490280ab2..00000000000 --- a/mysql-test/suite/galera/galera_2nodes_as_master_with_repl_filter.cnf +++ /dev/null @@ -1,87 +0,0 @@ -# -# This .cnf file creates a setup with a 2-node Galera cluster and one stand-alone MySQL server, to be used as a slave -# - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf - -[mysqld] -default-storage-engine=InnoDB - -[mysqld.1] -server-id=1 -binlog-format=row -log-bin=mysqld-bin -log_slave_updates -gtid-mode=on -enforce-gtid-consistency=true -event-scheduler=1 - -wsrep_provider=@ENV.WSREP_PROVIDER -wsrep_cluster_address='gcomm://' -wsrep_provider_options='base_port=@mysqld.1.#galera_port' - -# enforce read-committed characteristics across the cluster -wsrep_causal_reads=ON -wsrep_sync_wait = 15 - -wsrep_node_address=127.0.0.1 -wsrep_sst_receive_address=127.0.0.2:@mysqld.1.#sst_port -wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port - -# Required for Galera -innodb_autoinc_lock_mode=2 - -innodb_flush_log_at_trx_commit=2 - -[mysqld.2] -server-id=2 -binlog-format=row -log-bin=mysqld-bin -log_slave_updates -gtid-mode=on -enforce-gtid-consistency=true -event-scheduler=1 - -wsrep_provider=@ENV.WSREP_PROVIDER -wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port' -wsrep_provider_options='base_port=@mysqld.2.#galera_port' - -# enforce read-committed characteristics across the cluster -wsrep_causal_reads=ON -wsrep_sync_wait = 15 - -wsrep_node_address=127.0.0.1 -wsrep_sst_receive_address=127.0.0.2:@mysqld.2.#sst_port -wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port - -# Required for Galera -innodb_autoinc_lock_mode=2 - -innodb_flush_log_at_trx_commit=2 - -[mysqld.3] -server-id=3 -replicate-ignore-db=test -replicate-wild-ignore-table=test.% -log-bin=mysqld-bin -log_slave_updates -gtid-mode=on -enforce-gtid-consistency=true -event-scheduler=1 - -[ENV] -NODE_MYPORT_1= @mysqld.1.port -NODE_MYSOCK_1= @mysqld.1.socket - -NODE_MYPORT_2= @mysqld.2.port -NODE_MYSOCK_2= @mysqld.2.socket - -NODE_MYPORT_3= @mysqld.3.port -NODE_MYSOCK_3= @mysqld.3.socket - -NODE_GALERAPORT_1= @mysqld.1.#galera_port -NODE_GALERAPORT_2= @mysqld.2.#galera_port - -NODE_SSTPORT_1= @mysqld.1.#sst_port -NODE_SSTPORT_2= @mysqld.2.#sst_port From 7274bed257420d85fe07c986b57f864fc6052e24 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Thu, 22 Feb 2018 11:14:16 +0100 Subject: [PATCH 075/203] Fix MTR test galera.galera_gcache_recover_manytrx The kills the two nodes that compose the cluster and then recovers them. The first node that is recovered is expected to recover its gcache, so that when the second node is recovered, it is expected that it rejoins the cluster via IST. However, it is possible that if the second node is killed while it is applying in TOI mode, its local state is marked unsafe. In which case, SST is the only option to rejoin the cluster. codership/mysql-wsrep#323 --- .../galera/t/galera_gcache_recover_manytrx.test | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test b/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test index 3bfcdc9f117..08165f30f7d 100644 --- a/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test +++ b/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test @@ -5,6 +5,7 @@ --source include/galera_cluster.inc --source include/big_test.inc +--source include/have_log_bin.inc SET SESSION wsrep_sync_wait = 0; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 LONGBLOB) ENGINE=InnoDB; @@ -93,6 +94,8 @@ END| DELIMITER ;| +--let $wsrep_last_committed_before = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'` + --connect node_1_insert_simple, 127.0.0.1, root, , test, $NODE_MYPORT_1 --connect node_1_insert_multi, 127.0.0.1, root, , test, $NODE_MYPORT_1 --connect node_1_insert_transaction, 127.0.0.1, root, , test, $NODE_MYPORT_1 @@ -124,6 +127,13 @@ DELIMITER ;| --connection node_2 SET SESSION wsrep_sync_wait = 0; + +# Make sure that node_2 is not killed while TOIs are applied. +# Otherwhise we risk that grastate file is marked unsafe, and +# as a consequence the node cannot rejoin with IST. +--let $wait_condition = SELECT VARIABLE_VALUE > $wsrep_last_committed_before FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed' +--source include/wait_condition.inc + --source include/kill_galera.inc --sleep 10 @@ -172,9 +182,8 @@ SET SESSION wsrep_sync_wait = 0; --source include/start_mysqld.inc --connection node_1 ---source include/wait_until_connected_again.inc ---source include/galera_wait_ready.inc --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc --let $diff_servers = 1 2 --source include/diff_servers.inc From f8ea96b80c3eb79c45b646b61ff4472e1326ec18 Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Tue, 27 Feb 2018 04:52:55 +0200 Subject: [PATCH 076/203] codership/galera#500 MTR test for proper Galera provider tear down Added a test which verifies that if the gcomm background thread is forced to quit via exception, the error is propagated all the way up the stack and wsrep_ready becomes 0. --- mysql-test/suite/galera/r/galera#500.result | 10 ++++++ mysql-test/suite/galera/t/galera#500.test | 37 +++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 mysql-test/suite/galera/r/galera#500.result create mode 100644 mysql-test/suite/galera/t/galera#500.test diff --git a/mysql-test/suite/galera/r/galera#500.result b/mysql-test/suite/galera/r/galera#500.result new file mode 100644 index 00000000000..6a07d0359a4 --- /dev/null +++ b/mysql-test/suite/galera/r/galera#500.result @@ -0,0 +1,10 @@ +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options="gmcast.isolate=2"; +SET SESSION wsrep_sync_wait = 0; +SHOW STATUS LIKE 'wsrep_cluster_status'; +Variable_name Value +wsrep_cluster_status non-Primary +SET SESSION wsrep_sync_wait = default; +SET GLOBAL wsrep_provider_options="pc.bootstrap=1"; +SET SESSION wsrep_on=0; +CALL mtr.add_suppression("WSREP: exception from gcomm, backend must be restarted: Gcomm backend termination was requested by setting gmcast.isolate=2."); diff --git a/mysql-test/suite/galera/t/galera#500.test b/mysql-test/suite/galera/t/galera#500.test new file mode 100644 index 00000000000..a313d5778d3 --- /dev/null +++ b/mysql-test/suite/galera/t/galera#500.test @@ -0,0 +1,37 @@ +# +# The purpose of this test is to verify that if an exception is +# thrown from gcomm background thread, the provider terminates properly +# and wsrep_ready becomes 0. +# + +--source include/have_innodb.inc +--source include/galera_cluster.inc + +# Force node_2 gcomm background thread to terminate via exception. +--connection node_2 +--let $wsrep_cluster_address = `SELECT @@wsrep_cluster_address` +# Setting gmcast.isolate=2 will force gcomm background thread to +# throw exception. +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options="gmcast.isolate=2"; + +# Wait until wsrep_ready becomes 0. +--let $wait_condition = SELECT VARIABLE_VALUE = 0 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME ='wsrep_ready' +--source include/wait_condition.inc + +# Wait until node_1 ends up in non-prim and rebootstrap the cluster. +--connection node_1 +SET SESSION wsrep_sync_wait = 0; +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME ='wsrep_cluster_size' +--source include/wait_condition.inc +SHOW STATUS LIKE 'wsrep_cluster_status'; +SET SESSION wsrep_sync_wait = default; +SET GLOBAL wsrep_provider_options="pc.bootstrap=1"; + +# Restart node_2 +--connection node_2 +SET SESSION wsrep_on=0; +--source include/restart_mysqld.inc + +--connection node_2 +CALL mtr.add_suppression("WSREP: exception from gcomm, backend must be restarted: Gcomm backend termination was requested by setting gmcast.isolate=2."); From 2e5681a450627709512138a7d15b8ee035a29c6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 8 May 2018 09:33:48 +0300 Subject: [PATCH 077/203] Test requires galera debug library. --- mysql-test/suite/galera/t/galera#500.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/galera/t/galera#500.test b/mysql-test/suite/galera/t/galera#500.test index a313d5778d3..3c8490b6907 100644 --- a/mysql-test/suite/galera/t/galera#500.test +++ b/mysql-test/suite/galera/t/galera#500.test @@ -6,6 +6,7 @@ --source include/have_innodb.inc --source include/galera_cluster.inc +--source include/galera_have_debug_sync.inc # Force node_2 gcomm background thread to terminate via exception. --connection node_2 From 0f80c014c1dd1523293aa4420f7e7be9d69afa72 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 9 May 2018 11:22:26 -0400 Subject: [PATCH 078/203] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 75daccdd9d4..59a439d4152 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=0 -MYSQL_VERSION_PATCH=35 +MYSQL_VERSION_PATCH=36 From 2b35db5ac4e9f8e1ba20a1f733f094028d92b969 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 7 Mar 2018 13:10:01 +1100 Subject: [PATCH 079/203] MDEV-15496: wsrep_sst_common - parse IPv6 correct Fix to e78e308e818a4a763ebb981302851f58f4c42bba, IPv6 got split at : Also got rid of awk (yuck!) to parse port (commit: b4fbb4183b35f921efe079a312b0c1be87866b40) Tested with dash to avoid bashisms: $ dash -vx './wsrep_sst_common.sh' --address '128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more + readonly WSREP_SST_OPT_ADDR=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_HOST=128.0.0.5 + readonly WSREP_SST_OPT_HOST_UNESCAPED=128.0.0.5 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver $ dash -vx './wsrep_sst_common.sh' --address '[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more + readonly WSREP_SST_OPT_ADDR=[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + addr_no_bracket=2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_HOST_UNESCAPED=2001:db8::1 + readonly WSREP_SST_OPT_HOST=[2001:db8::1] + remain=:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver And bash as well: $ bash -vx './wsrep_sst_common.sh' --address '127.0.0.3:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more + readonly WSREP_SST_OPT_ADDR=127.0.0.3:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + WSREP_SST_OPT_ADDR=127.0.0.3:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + case "${WSREP_SST_OPT_ADDR}" in + readonly WSREP_SST_OPT_HOST=127.0.0.3 + WSREP_SST_OPT_HOST=127.0.0.3 + readonly WSREP_SST_OPT_HOST_UNESCAPED=127.0.0.3 + WSREP_SST_OPT_HOST_UNESCAPED=127.0.0.3 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_PORT=3333 + WSREP_SST_OPT_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver + WSREP_SST_OPT_SST_VER=sst_ver $ bash -vx './wsrep_sst_common.sh' --address '[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more + readonly 'WSREP_SST_OPT_ADDR=[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' + WSREP_SST_OPT_ADDR='[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' + case "${WSREP_SST_OPT_ADDR}" in + addr_no_bracket='2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' + readonly WSREP_SST_OPT_HOST_UNESCAPED=2001:db8::1 + WSREP_SST_OPT_HOST_UNESCAPED=2001:db8::1 + readonly 'WSREP_SST_OPT_HOST=[2001:db8::1]' + WSREP_SST_OPT_HOST='[2001:db8::1]' + remain=:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_PORT=3333 + WSREP_SST_OPT_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver + WSREP_SST_OPT_SST_VER=sst_ver --- scripts/wsrep_sst_common.sh | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index b9d9d04fef6..9926b4ac9d9 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -35,18 +35,25 @@ case "$1" in # # Break address string into host:port/path parts # - readonly WSREP_SST_OPT_HOST=${WSREP_SST_OPT_ADDR%%[:/]*} - if [ ${WSREP_SST_OPT_HOST:0:1} = '[' ] - then - # IPv6 notation - readonly WSREP_SST_OPT_HOST_UNESCAPED=${WSREP_SST_OPT_HOST:1:-1} - else - readonly WSREP_SST_OPT_HOST_UNESCAPED=${WSREP_SST_OPT_HOST} - fi - readonly WSREP_SST_OPT_PORT=$(echo $WSREP_SST_OPT_ADDR | \ - cut -d ']' -f 2 | cut -s -d ':' -f 2 | cut -d '/' -f 1) + case "${WSREP_SST_OPT_ADDR}" in + \[*) + # IPv6 + addr_no_bracket=${WSREP_SST_OPT_ADDR#\[} + readonly WSREP_SST_OPT_HOST_UNESCAPED=${addr_no_bracket%%\]*} + readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" + remain=${WSREP_SST_OPT_ADDR#*\]} + remain=${remain#*:} + ;; + *) + readonly WSREP_SST_OPT_HOST=${WSREP_SST_OPT_ADDR%%[:/]*} + readonly WSREP_SST_OPT_HOST_UNESCAPED=$WSREP_SST_OPT_HOST + remain=${WSREP_SST_OPT_ADDR#*:} + ;; + esac + readonly WSREP_SST_OPT_PORT=${remain%%/*} + remain=${remain#*/} + readonly WSREP_SST_OPT_MODULE=${remain%%/*} readonly WSREP_SST_OPT_PATH=${WSREP_SST_OPT_ADDR#*/} - readonly WSREP_SST_OPT_MODULE=${WSREP_SST_OPT_PATH%%/*} remain=${WSREP_SST_OPT_PATH#*/} readonly WSREP_SST_OPT_LSN=${remain%%/*} remain=${remain#*/} From 24ab82e67560f840f0dddda066c2612e7fb3a20e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 12 Mar 2018 15:14:15 +1100 Subject: [PATCH 080/203] MDEV-15541: wsrep_sst_common - WSREP_SST_OPT_PORT set twice (--address and --port) --- scripts/wsrep_sst_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 9926b4ac9d9..c69dbf92448 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -50,7 +50,7 @@ case "$1" in remain=${WSREP_SST_OPT_ADDR#*:} ;; esac - readonly WSREP_SST_OPT_PORT=${remain%%/*} + readonly WSREP_SST_OPT_ADDR_PORT=${remain%%/*} remain=${remain#*/} readonly WSREP_SST_OPT_MODULE=${remain%%/*} readonly WSREP_SST_OPT_PATH=${WSREP_SST_OPT_ADDR#*/} From e6b31df6dfce1a412d20f32092da676768cff8ad Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 16 Oct 2017 17:49:52 +0200 Subject: [PATCH 081/203] MDEV-13968 sst fails with "WSREP_SST_OPT_PORT: readonly variable" --- scripts/wsrep_sst_common.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index c69dbf92448..bb91eeaeba2 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -129,6 +129,17 @@ done readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG +if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then + if [ -n "$WSREP_SST_OPT_PORT" ]; then + if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then + wsrep_log_error "port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" + exit 2 + fi + else + readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" + fi +fi + # try to use my_print_defaults, mysql and mysqldump that come with the sources # (for MTR suite) SCRIPTS_DIR="$(cd $(dirname "$0"); pwd -P)" From fe3c4a41829582f41f2db00cd410229dca2b6c38 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 24 Oct 2017 20:59:54 +0200 Subject: [PATCH 082/203] MDEV-13968 sst fails with "WSREP_SST_OPT_PORT: readonly variable" followup for 990c4318a779645ee fix broken script (set -u) --- scripts/wsrep_sst_common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index bb91eeaeba2..c421bbfdc64 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -129,8 +129,8 @@ done readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG -if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then - if [ -n "$WSREP_SST_OPT_PORT" ]; then +if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then + if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then wsrep_log_error "port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" exit 2 From 82f26dafcb72e74a11916e1d22a2fabac4f51ce8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 12 Mar 2018 15:23:58 +1100 Subject: [PATCH 083/203] MDEV-13968: wsrep_log_error not defined until later in wsrep_sst_common Fix for 990c4318a779645ee dash -vx './wsrep_sst_common.sh' --address '128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' --port 1111 2>&1 | more .... + readonly WSREP_SST_OPT_ADDR=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_HOST=128.0.0.5 + readonly WSREP_SST_OPT_HOST_UNESCAPED=128.0.0.5 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_ADDR_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver + shift + shift + [ 2 -gt 0 ] + readonly WSREP_SST_OPT_PORT=1111 + shift + shift + [ 0 -gt 0 ] readonly WSREP_SST_OPT_BYPASS + readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG + readonly WSREP_SST_OPT_BINLOG if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 exit 2 fi else readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" fi fi + [ -n 3333 ] + [ -n 1111 ] + [ 1111 != 3333 ] + echo WSREP_SST: [ERROR] port in --port=1111 differs from port in --address=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 WSREP_SST: [ERROR] port in --port=1111 differs from port in --address=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + exit 2 dash -vx './wsrep_sst_common.sh' --address '128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more ... + [ 2 -gt 0 ] + readonly WSREP_SST_OPT_ADDR=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_HOST=128.0.0.5 + readonly WSREP_SST_OPT_HOST_UNESCAPED=128.0.0.5 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_ADDR_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver + shift + shift + [ 0 -gt 0 ] readonly WSREP_SST_OPT_BYPASS + readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG + readonly WSREP_SST_OPT_BINLOG if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 exit 2 fi else readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" fi fi + [ -n 3333 ] + [ -n ] + readonly WSREP_SST_OPT_PORT=3333 --- scripts/wsrep_sst_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index c421bbfdc64..d20e3bc36c9 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -132,7 +132,7 @@ readonly WSREP_SST_OPT_BINLOG if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then - wsrep_log_error "port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" + echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 exit 2 fi else From 215d652c66474c4578b0476385d055e7feae2d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 9 May 2018 09:16:20 +0300 Subject: [PATCH 084/203] MDEV-15351: wsrep_sst_xtrabackup is broken in 10.1.31 Remove the setup_ports function call. This is related to https://github.com/MariaDB/server/pull/717 Thanks to Daniel Black and Bart S. --- scripts/wsrep_sst_xtrabackup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh index 75aedb7ae63..eb059f8bb9d 100644 --- a/scripts/wsrep_sst_xtrabackup.sh +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -394,7 +394,6 @@ if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' fi read_cnf -setup_ports get_stream get_transfer From 6a04c2a1aa60e67c07d4e6ada63f65cc3b26a5a9 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Tue, 22 May 2018 12:09:05 -0700 Subject: [PATCH 085/203] MDEV-16235 Server crashes in my_utf8_uni or in my_strtod_int upon SELECT .. LIMIT 0 The code must differentiate between a SELECT with contradictory WHERE/HAVING and one with LIMIT 0. Also for the latter printed 'Zero limit' instead of 'Impossible where' in the EXPLAIN output. --- mysql-test/r/limit.result | 16 ++++++++++++++++ mysql-test/r/subselect4.result | 2 +- mysql-test/r/subselect_mat_cost_bugs.result | 2 +- mysql-test/t/limit.test | 14 ++++++++++++++ mysql-test/t/subselect4.test | 2 +- sql/sql_select.cc | 17 +++++++++++++---- 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/mysql-test/r/limit.result b/mysql-test/r/limit.result index 176a93c7a46..deb4ca2ab95 100644 --- a/mysql-test/r/limit.result +++ b/mysql-test/r/limit.result @@ -146,3 +146,19 @@ a 16 DROP TABLE t1; End of 5.1 tests +# +# mdev-16235: SELECT over a table with LIMIT 0 +# +EXPLAIN +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Zero limit +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; +start_time user_host query_time lock_time rows_sent rows_examined db last_insert_id insert_id server_id sql_text +EXPLAIN +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Zero limit +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; +help_topic_id name help_category_id description example url +End of 5.5 tests diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 6accc23d18a..8973330c7da 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2512,7 +2512,7 @@ SELECT 2 IN (SELECT 2 from DUAL WHERE 1 != 1); SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; # -# mfrv-14515: Wrong results for tableless query with subquery in WHERE +# mdev-14515: Wrong results for tableless query with subquery in WHERE # and implicit aggregation # create table t1 (i1 int, i2 int); diff --git a/mysql-test/r/subselect_mat_cost_bugs.result b/mysql-test/r/subselect_mat_cost_bugs.result index df6b543bab8..d33f1488e4d 100644 --- a/mysql-test/r/subselect_mat_cost_bugs.result +++ b/mysql-test/r/subselect_mat_cost_bugs.result @@ -334,7 +334,7 @@ SELECT * FROM t1 WHERE (f1) IN (SELECT f1 FROM t2) LIMIT 0; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Zero limit 2 DEPENDENT SUBQUERY t2 ALL NULL NULL NULL NULL 2 Using where SELECT * FROM t1 WHERE (f1) IN (SELECT f1 FROM t2) diff --git a/mysql-test/t/limit.test b/mysql-test/t/limit.test index 4dbe13096d4..668d3b74518 100644 --- a/mysql-test/t/limit.test +++ b/mysql-test/t/limit.test @@ -115,3 +115,17 @@ SELECT a FROM t1 ORDER BY a LIMIT 2 OFFSET 14; DROP TABLE t1; --echo End of 5.1 tests + +--echo # +--echo # mdev-16235: SELECT over a table with LIMIT 0 +--echo # + +EXPLAIN +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; + +EXPLAIN +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; + +--echo End of 5.5 tests diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index 2b53b55b735..2727fd9daed 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -2047,7 +2047,7 @@ SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; --echo # ---echo # mfrv-14515: Wrong results for tableless query with subquery in WHERE +--echo # mdev-14515: Wrong results for tableless query with subquery in WHERE --echo # and implicit aggregation --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1e9f1c0848b..d6d269a700f 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1136,10 +1136,19 @@ JOIN::optimize() if (cond_value == Item::COND_FALSE || having_value == Item::COND_FALSE || (!unit->select_limit_cnt && !(select_options & OPTION_FOUND_ROWS))) { /* Impossible cond */ - DBUG_PRINT("info", (having_value == Item::COND_FALSE ? - "Impossible HAVING" : "Impossible WHERE")); - zero_result_cause= having_value == Item::COND_FALSE ? - "Impossible HAVING" : "Impossible WHERE"; + if (unit->select_limit_cnt) + { + DBUG_PRINT("info", (having_value == Item::COND_FALSE ? + "Impossible HAVING" : "Impossible WHERE")); + zero_result_cause= having_value == Item::COND_FALSE ? + "Impossible HAVING" : "Impossible WHERE"; + } + else + { + DBUG_PRINT("info", ("Zero limit")); + zero_result_cause= "Zero limit"; + conds= 0; + } table_count= top_join_tab_count= 0; error= 0; goto setup_subq_exit; From 1ada4afb0a51f7283b6187a95019ec2cb80c8a0b Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Sun, 29 Apr 2018 19:47:48 +0300 Subject: [PATCH 086/203] mtr: use process launch -- args to start mysqld in lldb gdb's "set args" equivalent for lldb would be "settings set target.run-args", however it doesn't play well with double dashed args (--). --- mysql-test/mysql-test-run.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index de1871cbf03..ba49d88a85f 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -6051,7 +6051,7 @@ sub lldb_arguments { $input = $input ? "< $input" : ""; # write init file for mysqld or client - mtr_tofile($lldb_init_file, "set args $str $input\n"); + mtr_tofile($lldb_init_file, "process launch --stop-at-entry -- $str $input\n"); print "\nTo start lldb for $type, type in another window:\n"; print "cd $glob_mysql_test_dir && lldb -s $lldb_init_file $$exe\n"; From 5fb2c586f225361deae0f04dc6f72a3f0d168ac2 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 2 Jun 2018 11:52:48 +0530 Subject: [PATCH 087/203] MDEV-16225: wrong resultset from query with semijoin=on For non-semi-join subquery optimization we do a cost based decision between Materialisation and IN -> EXIST transformation. The issue in this case is that for IN->EXIST transformation we run JOIN::reoptimize with the IN->EXISt conditions and we come up with a new query plan. But when we compare the cost with Materialization, we make the decision to chose Materialization so we need to restore the query plan for Materilization. The saving and restoring for keyuse array and join_tab keyuse is only done when we have atleast one element in the keyuse_array , we are now changing to do it even for 0 elements to main the generality. --- mysql-test/r/subselect_sj2_mat.result | 57 +++++++++++++++++++++++++++ mysql-test/t/subselect_sj2_mat.test | 42 ++++++++++++++++++++ sql/sql_select.cc | 46 ++++++++++----------- 3 files changed, 119 insertions(+), 26 deletions(-) diff --git a/mysql-test/r/subselect_sj2_mat.result b/mysql-test/r/subselect_sj2_mat.result index 835742a3ff4..19aa3bd3f02 100644 --- a/mysql-test/r/subselect_sj2_mat.result +++ b/mysql-test/r/subselect_sj2_mat.result @@ -1601,3 +1601,60 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY eq_ref distinct_key distinct_key 11 func 1 2 MATERIALIZED t2 ALL NULL NULL NULL NULL 3 Using where DROP TABLE t1,t2; +# +# MDEV-16225: wrong resultset from query with semijoin=on +# +CREATE TABLE t1 ( +`id` int(10) NOT NULL AUTO_INCREMENT, +`local_name` varchar(64) NOT NULL, +PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=latin1; +insert into t1(`id`,`local_name`) values +(1,'Cash Advance'), +(2,'Cash Advance'), +(3,'Rollover'), +(4,'AL Installment'), +(5,'AL Installment'), +(6,'AL Installment'), +(7,'AL Installment'), +(8,'AL Installment'), +(9,'AL Installment'), +(10,'Internet Payday'), +(11,'Rollover - Internet Payday'), +(12,'AL Monthly Installment'), +(13,'AL Semi-Monthly Installment'); +explain +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( +t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) +OR +(t.id IN (0,4,12,13,1,10,3,11)) +); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t index PRIMARY PRIMARY 4 NULL 13 Using where; Using index +2 MATERIALIZED ALL distinct_key NULL NULL NULL 8 +2 MATERIALIZED A ALL PRIMARY NULL NULL NULL 13 Using where; Using join buffer (flat, BNL join) +3 MATERIALIZED B ALL PRIMARY NULL NULL NULL 13 Using where +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( +t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) +OR +(t.id IN (0,4,12,13,1,10,3,11)) +); +id +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +drop table t1; diff --git a/mysql-test/t/subselect_sj2_mat.test b/mysql-test/t/subselect_sj2_mat.test index cfb6c8c2819..0665cdf68fe 100644 --- a/mysql-test/t/subselect_sj2_mat.test +++ b/mysql-test/t/subselect_sj2_mat.test @@ -303,3 +303,45 @@ eval $q; eval explain $q; DROP TABLE t1,t2; + +--echo # +--echo # MDEV-16225: wrong resultset from query with semijoin=on +--echo # + +CREATE TABLE t1 ( + `id` int(10) NOT NULL AUTO_INCREMENT, + `local_name` varchar(64) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=latin1; + +insert into t1(`id`,`local_name`) values +(1,'Cash Advance'), +(2,'Cash Advance'), +(3,'Rollover'), +(4,'AL Installment'), +(5,'AL Installment'), +(6,'AL Installment'), +(7,'AL Installment'), +(8,'AL Installment'), +(9,'AL Installment'), +(10,'Internet Payday'), +(11,'Rollover - Internet Payday'), +(12,'AL Monthly Installment'), +(13,'AL Semi-Monthly Installment'); + +explain +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( + t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) + OR + (t.id IN (0,4,12,13,1,10,3,11)) +); +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( + t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) + OR + (t.id IN (0,4,12,13,1,10,3,11)) +); +drop table t1; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index d6d269a700f..016dd594e08 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -23739,21 +23739,18 @@ void JOIN::set_allowed_join_cache_types() void JOIN::save_query_plan(Join_plan_state *save_to) { - if (keyuse.elements) - { - DYNAMIC_ARRAY tmp_keyuse; - /* Swap the current and the backup keyuse internal arrays. */ - tmp_keyuse= keyuse; - keyuse= save_to->keyuse; /* keyuse is reset to an empty array. */ - save_to->keyuse= tmp_keyuse; + DYNAMIC_ARRAY tmp_keyuse; + /* Swap the current and the backup keyuse internal arrays. */ + tmp_keyuse= keyuse; + keyuse= save_to->keyuse; /* keyuse is reset to an empty array. */ + save_to->keyuse= tmp_keyuse; - for (uint i= 0; i < table_count; i++) - { - save_to->join_tab_keyuse[i]= join_tab[i].keyuse; - join_tab[i].keyuse= NULL; - save_to->join_tab_checked_keys[i]= join_tab[i].checked_keys; - join_tab[i].checked_keys.clear_all(); - } + for (uint i= 0; i < table_count; i++) + { + save_to->join_tab_keyuse[i]= join_tab[i].keyuse; + join_tab[i].keyuse= NULL; + save_to->join_tab_checked_keys[i]= join_tab[i].checked_keys; + join_tab[i].checked_keys.clear_all(); } memcpy((uchar*) save_to->best_positions, (uchar*) best_positions, sizeof(POSITION) * (table_count + 1)); @@ -23791,20 +23788,17 @@ void JOIN::reset_query_plan() void JOIN::restore_query_plan(Join_plan_state *restore_from) { - if (restore_from->keyuse.elements) + DYNAMIC_ARRAY tmp_keyuse; + tmp_keyuse= keyuse; + keyuse= restore_from->keyuse; + restore_from->keyuse= tmp_keyuse; + + for (uint i= 0; i < table_count; i++) { - DYNAMIC_ARRAY tmp_keyuse; - tmp_keyuse= keyuse; - keyuse= restore_from->keyuse; - restore_from->keyuse= tmp_keyuse; - - for (uint i= 0; i < table_count; i++) - { - join_tab[i].keyuse= restore_from->join_tab_keyuse[i]; - join_tab[i].checked_keys= restore_from->join_tab_checked_keys[i]; - } - + join_tab[i].keyuse= restore_from->join_tab_keyuse[i]; + join_tab[i].checked_keys= restore_from->join_tab_checked_keys[i]; } + memcpy((uchar*) best_positions, (uchar*) restore_from->best_positions, sizeof(POSITION) * (table_count + 1)); /* Restore SJM nests */ From 72b6d01848e56a75349d663bc61bbe71f97a280b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 5 Jun 2018 22:13:19 +0100 Subject: [PATCH 088/203] MDEV-10246 ssl-* config file options have no effect without mysql_ssl_set() Partially revert 4ef74979969ac9339d0d42c11a6f26632e6776f1 that caused regression. Any ssl- option must imply use_ssl=1, even if mysql_set_ssl() was not used. --- sql-common/client.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sql-common/client.c b/sql-common/client.c index 6321e64bc75..ec992f80e8d 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -2541,6 +2541,10 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio, mysql->client_flag|= CLIENT_MULTI_RESULTS; #ifdef HAVE_OPENSSL + if (mysql->options.ssl_key || mysql->options.ssl_cert || + mysql->options.ssl_ca || mysql->options.ssl_capath || + mysql->options.ssl_cipher) + mysql->options.use_ssl = 1; if (mysql->options.use_ssl) mysql->client_flag|= CLIENT_SSL; #endif /* HAVE_OPENSSL */ From 75b4eb5cc969a1f5fc221d03c62987b8e57d6332 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 6 Jun 2018 15:27:57 +0200 Subject: [PATCH 089/203] Catch of OOM situation. --- sql/sql_base.cc | 3 +++ sql/table.cc | 2 ++ 2 files changed, 5 insertions(+) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index f5864256440..71bbb538b5c 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -6426,6 +6426,9 @@ find_field_in_natural_join(THD *thd, TABLE_LIST *table_ref, const char *name, column reference. See create_view_field() for details. */ item= nj_col->create_item(thd); + if (!item) + DBUG_RETURN(NULL); + /* *ref != NULL means that *ref contains the item that we need to replace. If the item was aliased by the user, set the alias to diff --git a/sql/table.cc b/sql/table.cc index bb4eae9b1e2..552f514283d 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -5252,6 +5252,8 @@ Item *create_view_field(THD *thd, TABLE_LIST *view, Item **field_ref, Item *item= new Item_direct_view_ref(&view->view->select_lex.context, field_ref, view->alias, name, view); + if (!item) + return NULL; /* Force creation of nullable item for the result tmp table for outer joined views/derived tables. From 3627dd7f6a1cf5bd7151ff23290d64cb6dffea90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 7 Jun 2018 10:31:39 +0300 Subject: [PATCH 090/203] MDEV-16416 Crash on IMPORT TABLESPACE of a ROW_FORMAT=COMPRESSED table fil_iterate(): Invoke fil_encrypt_buf() correctly when a ROW_FORMAT=COMPRESSED table with a physical page size of innodb_page_size is being imported. Also, validate the page checksum before decryption, and reduce the scope of some variables. AbstractCallback::operator()(): Remove the parameter 'offset'. The check for it in FetchIndexRootPages::operator() was basically redundant and dead code since the previous refactoring. --- .../encryption/r/innodb-discard-import.result | 6 + .../encryption/t/innodb-discard-import.test | 32 ++-- storage/innobase/row/row0import.cc | 145 ++++++++---------- storage/xtradb/row/row0import.cc | 145 ++++++++---------- 4 files changed, 147 insertions(+), 181 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb-discard-import.result b/mysql-test/suite/encryption/r/innodb-discard-import.result index 06f4abab9f4..406460d0a23 100644 --- a/mysql-test/suite/encryption/r/innodb-discard-import.result +++ b/mysql-test/suite/encryption/r/innodb-discard-import.result @@ -1,5 +1,8 @@ call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); +SET @innodb_file_format_orig = @@GLOBAL.innodb_file_format; +SET @innodb_file_per_table_orig = @@GLOBAL.innodb_file_per_table; +SET @innodb_compression_algo = @@GLOBAL.innodb_compression_algorithm; SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; @@ -134,3 +137,6 @@ NOT FOUND /tmpres/ in t3.ibd NOT FOUND /mysql/ in t4.ibd DROP PROCEDURE innodb_insert_proc; DROP TABLE t1,t2,t3,t4; +SET GLOBAL innodb_file_format = @innodb_file_format_orig; +SET GLOBAL innodb_file_per_table = @innodb_file_per_table_orig; +SET GLOBAL innodb_compression_algorithm = @innodb_compression_algo; diff --git a/mysql-test/suite/encryption/t/innodb-discard-import.test b/mysql-test/suite/encryption/t/innodb-discard-import.test index 9feaacc41e5..54471ae3bae 100644 --- a/mysql-test/suite/encryption/t/innodb-discard-import.test +++ b/mysql-test/suite/encryption/t/innodb-discard-import.test @@ -1,10 +1,6 @@ -- source include/have_innodb.inc +-- source include/innodb_page_size_small.inc -- source include/have_file_key_management_plugin.inc -# embedded does not support restart --- source include/not_embedded.inc --- source include/not_valgrind.inc -# Avoid CrashReporter popup on Mac --- source include/not_crashrep.inc # # MDEV-8770: Incorrect error message when importing page compressed tablespace @@ -13,17 +9,13 @@ call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); ---disable_query_log -let $innodb_file_format_orig = `SELECT @@innodb_file_format`; -let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; -let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`; ---enable_query_log +SET @innodb_file_format_orig = @@GLOBAL.innodb_file_format; +SET @innodb_file_per_table_orig = @@GLOBAL.innodb_file_per_table; +SET @innodb_compression_algo = @@GLOBAL.innodb_compression_algorithm; ---disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; ---enable_warnings --let $MYSQLD_TMPDIR = `SELECT @@tmpdir` --let $MYSQLD_DATADIR = `SELECT @@datadir` @@ -37,7 +29,9 @@ create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes en show warnings; create table t2(c1 bigint not null, b char(200)) engine=innodb page_compressed=1 encrypted=yes encryption_key_id=4; show warnings; -create table t3(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4; +let $kbs= `select floor(@@global.innodb_page_size/1024)`; +--replace_regex / key_block_size=\d+//i +eval create table t3(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4 key_block_size=$kbs; show warnings; create table t4(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; show warnings; @@ -98,6 +92,7 @@ ALTER TABLE t2 IMPORT TABLESPACE; SHOW CREATE TABLE t2; SELECT COUNT(*) FROM t2; ALTER TABLE t3 IMPORT TABLESPACE; +--replace_regex / key_block_size=\d+//i SHOW CREATE TABLE t3; SELECT COUNT(*) FROM t3; ALTER TABLE t4 IMPORT TABLESPACE; @@ -125,11 +120,6 @@ SELECT COUNT(*) FROM t4; DROP PROCEDURE innodb_insert_proc; DROP TABLE t1,t2,t3,t4; -# reset system ---disable_warnings ---disable_query_log -EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; -EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; -EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo; ---enable_query_log ---enable_warnings +SET GLOBAL innodb_file_format = @innodb_file_format_orig; +SET GLOBAL innodb_file_per_table = @innodb_file_per_table_orig; +SET GLOBAL innodb_compression_algorithm = @innodb_compression_algo; diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index dfd6b4bfbea..9ce172f1d07 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -427,12 +427,9 @@ public: updated then its state must be set to BUF_PAGE_NOT_USED. For compressed tables the page descriptor memory will be at offset: block->frame + UNIV_PAGE_SIZE; - @param offset - physical offset within the file - @param block - block read from file, note it is not from the buffer pool + @param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ - virtual dberr_t operator()( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW = 0; + virtual dberr_t operator()(buf_block_t* block) UNIV_NOTHROW = 0; /** @return the space id of the tablespace */ @@ -689,12 +686,9 @@ struct FetchIndexRootPages : public AbstractCallback { /** Called for each block as it is read from the file. - @param offset - physical offset in the file - @param block - block to convert, it is not from the buffer pool. + @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; /** Update the import configuration that will be used to import the tablespace. */ @@ -712,13 +706,9 @@ Called for each block as it is read from the file. Check index pages to determine the exact row format. We can't get that from the tablespace header flags alone. -@param offset - physical offset in the file -@param block - block to convert, it is not from the buffer pool. +@param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ -dberr_t -FetchIndexRootPages::operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW +dberr_t FetchIndexRootPages::operator()(buf_block_t* block) UNIV_NOTHROW { if (is_interrupted()) return DB_INTERRUPTED; @@ -726,15 +716,7 @@ FetchIndexRootPages::operator() ( ulint page_type = fil_page_get_type(page); - if (block->page.offset * m_page_size != offset) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Page offset doesn't match file offset: " - "page offset: %u, file offset: " ULINTPF, - block->page.offset, - (ulint) (offset / m_page_size)); - - return DB_CORRUPTION; - } else if (page_type == FIL_PAGE_TYPE_XDES) { + if (page_type == FIL_PAGE_TYPE_XDES) { return set_current_xdes(block->page.offset, page); } else if (page_type == FIL_PAGE_INDEX && !is_free(block->page.offset) @@ -877,12 +859,9 @@ public: /** Called for each block as it is read from the file. - @param offset - physical offset in the file - @param block - block to convert, it is not from the buffer pool. + @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; private: /** Update the page, set the space id, max trx id and index id. @@ -2039,10 +2018,9 @@ PageConverter::update_page( /** Called for every page in the tablespace. If the page was not updated then its state must be set to BUF_PAGE_NOT_USED. -@param block - block read from file, note it is not from the buffer pool +@param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ -dberr_t -PageConverter::operator() (os_offset_t, buf_block_t* block) UNIV_NOTHROW +dberr_t PageConverter::operator()(buf_block_t* block) UNIV_NOTHROW { /* If we already had an old page with matching number in the buffer pool, evict it now, because @@ -3430,26 +3408,15 @@ fil_iterate( } bool updated = false; - os_offset_t page_off = offset; ulint n_pages_read = (ulint) n_bytes / iter.page_size; const ulint size = iter.page_size; - block->page.offset = page_off / size; + block->page.offset = offset / size; for (ulint i = 0; i < n_pages_read; - ++i, page_off += size, block->frame += size, - block->page.offset++) { - bool decrypted = false; - dberr_t err = DB_SUCCESS; + ++i, block->frame += size, block->page.offset++) { byte* src = readptr + (i * size); - byte* dst = io_buffer + (i * size); - bool frame_changed = false; - ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); - const bool page_compressed - = page_type - == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_COMPRESSED; const ulint page_no = page_get_page_no(src); - if (!page_no && page_off) { + if (!page_no && block->page.offset) { const ulint* b = reinterpret_cast (src); const ulint* const e = b + size / sizeof *b; @@ -3464,11 +3431,46 @@ fil_iterate( continue; } - if (page_no != page_off / size) { - goto page_corrupted; + if (page_no != block->page.offset) { +page_corrupted: + ib_logf(IB_LOG_LEVEL_WARN, + "%s: Page %lu at offset " + UINT64PF " looks corrupted.", + callback.filename(), + ulong(offset / size), offset); + return DB_CORRUPTION; } - if (encrypted) { + bool decrypted = false; + dberr_t err = DB_SUCCESS; + byte* dst = io_buffer + (i * size); + bool frame_changed = false; + ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); + const bool page_compressed + = page_type + == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED + || page_type == FIL_PAGE_PAGE_COMPRESSED; + + if (!encrypted) { + } else if (!mach_read_from_4( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + src)) { +not_encrypted: + if (!page_compressed + && !block->page.zip.data) { + block->frame = src; + frame_changed = true; + } else { + ut_ad(dst != src); + memcpy(dst, src, size); + } + } else { + if (!fil_space_verify_crypt_checksum( + src, callback.get_zip_size(), + NULL, block->page.offset)) { + goto page_corrupted; + } + decrypted = fil_space_decrypt( iter.crypt_data, dst, iter.page_size, src, &err); @@ -3477,18 +3479,11 @@ fil_iterate( return err; } - if (decrypted) { - updated = true; - } else { - if (!page_compressed - && !block->page.zip.data) { - block->frame = src; - frame_changed = true; - } else { - ut_ad(dst != src); - memcpy(dst, src, size); - } + if (!decrypted) { + goto not_encrypted; } + + updated = true; } /* If the original page is page_compressed, we need @@ -3502,16 +3497,10 @@ fil_iterate( encrypted && !frame_changed ? dst : src, callback.get_zip_size(), NULL)) { -page_corrupted: - ib_logf(IB_LOG_LEVEL_WARN, - "%s: Page %lu at offset " - UINT64PF " looks corrupted.", - callback.filename(), - ulong(offset / size), offset); - return DB_CORRUPTION; + goto page_corrupted; } - if ((err = callback(page_off, block)) != DB_SUCCESS) { + if ((err = callback(block)) != DB_SUCCESS) { return err; } else if (!updated) { updated = buf_block_get_state(block) @@ -3584,19 +3573,15 @@ page_corrupted: buffer pool is not being used at all! */ if (decrypted && encrypted) { byte *dest = writeptr + (i * size); - ulint space = mach_read_from_4( - src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); - ib_uint64_t lsn = mach_read_from_8(src + FIL_PAGE_LSN); byte* tmp = fil_encrypt_buf( - iter.crypt_data, - space, - offset, - lsn, - src, - iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size, - dest); + iter.crypt_data, + callback.get_space_id(), + block->page.offset, + mach_read_from_8(src + FIL_PAGE_LSN), + src, + callback.get_zip_size(), + dest); if (tmp == src) { /* TODO: remove unnecessary memcpy's */ diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index a3478dc2fc4..2b08c819b6d 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -427,12 +427,9 @@ public: updated then its state must be set to BUF_PAGE_NOT_USED. For compressed tables the page descriptor memory will be at offset: block->frame + UNIV_PAGE_SIZE; - @param offset - physical offset within the file - @param block - block read from file, note it is not from the buffer pool + @param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ - virtual dberr_t operator()( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW = 0; + virtual dberr_t operator()(buf_block_t* block) UNIV_NOTHROW = 0; /** @return the space id of the tablespace */ @@ -689,12 +686,9 @@ struct FetchIndexRootPages : public AbstractCallback { /** Called for each block as it is read from the file. - @param offset - physical offset in the file - @param block - block to convert, it is not from the buffer pool. + @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; /** Update the import configuration that will be used to import the tablespace. */ @@ -712,13 +706,9 @@ Called for each block as it is read from the file. Check index pages to determine the exact row format. We can't get that from the tablespace header flags alone. -@param offset - physical offset in the file -@param block - block to convert, it is not from the buffer pool. +@param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ -dberr_t -FetchIndexRootPages::operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW +dberr_t FetchIndexRootPages::operator()(buf_block_t* block) UNIV_NOTHROW { if (is_interrupted()) return DB_INTERRUPTED; @@ -726,15 +716,7 @@ FetchIndexRootPages::operator() ( ulint page_type = fil_page_get_type(page); - if (block->page.offset * m_page_size != offset) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Page offset doesn't match file offset: " - "page offset: %u, file offset: " ULINTPF, - block->page.offset, - (ulint) (offset / m_page_size)); - - return DB_CORRUPTION; - } else if (page_type == FIL_PAGE_TYPE_XDES) { + if (page_type == FIL_PAGE_TYPE_XDES) { return set_current_xdes(block->page.offset, page); } else if (page_type == FIL_PAGE_INDEX && !is_free(block->page.offset) @@ -877,12 +859,9 @@ public: /** Called for each block as it is read from the file. - @param offset - physical offset in the file - @param block - block to convert, it is not from the buffer pool. + @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; private: /** Update the page, set the space id, max trx id and index id. @@ -2038,10 +2017,9 @@ PageConverter::update_page( /** Called for every page in the tablespace. If the page was not updated then its state must be set to BUF_PAGE_NOT_USED. -@param block - block read from file, note it is not from the buffer pool +@param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ -dberr_t -PageConverter::operator() (os_offset_t, buf_block_t* block) UNIV_NOTHROW +dberr_t PageConverter::operator()(buf_block_t* block) UNIV_NOTHROW { /* If we already had an old page with matching number in the buffer pool, evict it now, because @@ -3429,26 +3407,15 @@ fil_iterate( } bool updated = false; - os_offset_t page_off = offset; ulint n_pages_read = (ulint) n_bytes / iter.page_size; const ulint size = iter.page_size; - block->page.offset = page_off / size; + block->page.offset = offset / size; for (ulint i = 0; i < n_pages_read; - ++i, page_off += size, block->frame += size, - block->page.offset++) { - bool decrypted = false; - dberr_t err = DB_SUCCESS; + ++i, block->frame += size, block->page.offset++) { byte* src = readptr + (i * size); - byte* dst = io_buffer + (i * size); - bool frame_changed = false; - ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); - const bool page_compressed - = page_type - == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_COMPRESSED; const ulint page_no = page_get_page_no(src); - if (!page_no && page_off) { + if (!page_no && block->page.offset) { const ulint* b = reinterpret_cast (src); const ulint* const e = b + size / sizeof *b; @@ -3463,11 +3430,46 @@ fil_iterate( continue; } - if (page_no != page_off / size) { - goto page_corrupted; + if (page_no != block->page.offset) { +page_corrupted: + ib_logf(IB_LOG_LEVEL_WARN, + "%s: Page %lu at offset " + UINT64PF " looks corrupted.", + callback.filename(), + ulong(offset / size), offset); + return DB_CORRUPTION; } - if (encrypted) { + bool decrypted = false; + dberr_t err = DB_SUCCESS; + byte* dst = io_buffer + (i * size); + bool frame_changed = false; + ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); + const bool page_compressed + = page_type + == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED + || page_type == FIL_PAGE_PAGE_COMPRESSED; + + if (!encrypted) { + } else if (!mach_read_from_4( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + src)) { +not_encrypted: + if (!page_compressed + && !block->page.zip.data) { + block->frame = src; + frame_changed = true; + } else { + ut_ad(dst != src); + memcpy(dst, src, size); + } + } else { + if (!fil_space_verify_crypt_checksum( + src, callback.get_zip_size(), + NULL, block->page.offset)) { + goto page_corrupted; + } + decrypted = fil_space_decrypt( iter.crypt_data, dst, iter.page_size, src, &err); @@ -3476,18 +3478,11 @@ fil_iterate( return err; } - if (decrypted) { - updated = true; - } else { - if (!page_compressed - && !block->page.zip.data) { - block->frame = src; - frame_changed = true; - } else { - ut_ad(dst != src); - memcpy(dst, src, size); - } + if (!decrypted) { + goto not_encrypted; } + + updated = true; } /* If the original page is page_compressed, we need @@ -3501,16 +3496,10 @@ fil_iterate( encrypted && !frame_changed ? dst : src, callback.get_zip_size(), NULL)) { -page_corrupted: - ib_logf(IB_LOG_LEVEL_WARN, - "%s: Page %lu at offset " - UINT64PF " looks corrupted.", - callback.filename(), - ulong(offset / size), offset); - return DB_CORRUPTION; + goto page_corrupted; } - if ((err = callback(page_off, block)) != DB_SUCCESS) { + if ((err = callback(block)) != DB_SUCCESS) { return err; } else if (!updated) { updated = buf_block_get_state(block) @@ -3583,19 +3572,15 @@ page_corrupted: buffer pool is not being used at all! */ if (decrypted && encrypted) { byte *dest = writeptr + (i * size); - ulint space = mach_read_from_4( - src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); - ib_uint64_t lsn = mach_read_from_8(src + FIL_PAGE_LSN); byte* tmp = fil_encrypt_buf( - iter.crypt_data, - space, - offset, - lsn, - src, - iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size, - dest); + iter.crypt_data, + callback.get_space_id(), + block->page.offset, + mach_read_from_8(src + FIL_PAGE_LSN), + src, + callback.get_zip_size(), + dest); if (tmp == src) { /* TODO: remove unnecessary memcpy's */ From d9b159a2027c56c5c87385cfe1ae43b8c73a97b6 Mon Sep 17 00:00:00 2001 From: Chris Calender Date: Tue, 17 Apr 2018 15:00:15 -0400 Subject: [PATCH 091/203] MDEV-15789 - mysqlslap use incorrect table def The bug arises when one uses --auto-generate-sql-guid-primary (and --auto-generate-sql-secondary-indexes) with mysqlslap and also have sql_mode=STRICT_TRANS_TABLE. When using this option, mysqlslap should create a column with varchar(36), but it appears to create it as a varchar(32) only. Then if one has sql_mode=STRICT_TRANS_TABLES, it throws an error, like: mysqlslap: Cannot run query INSERT INTO t1 VALUES (...) ERROR : Data too long for column 'id' at row 1 Upstream bug report: BUG#80329. --- client/mysqlslap.c | 4 ++-- mysql-test/r/mysqlslap.result | 3 +++ mysql-test/t/mysqlslap.test | 8 ++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index a9746916411..ba7b8cea6a9 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -840,7 +840,7 @@ build_table_string(void) if (auto_generate_sql_guid_primary) { - dynstr_append(&table_string, "id varchar(32) primary key"); + dynstr_append(&table_string, "id varchar(36) primary key"); if (num_int_cols || num_char_cols || auto_generate_sql_guid_primary) dynstr_append(&table_string, ","); @@ -855,7 +855,7 @@ build_table_string(void) if (count) /* Except for the first pass we add a comma */ dynstr_append(&table_string, ","); - if (snprintf(buf, HUGE_STRING_LENGTH, "id%d varchar(32) unique key", count) + if (snprintf(buf, HUGE_STRING_LENGTH, "id%d varchar(36) unique key", count) > HUGE_STRING_LENGTH) { fprintf(stderr, "Memory Allocation error in create table\n"); diff --git a/mysql-test/r/mysqlslap.result b/mysql-test/r/mysqlslap.result index af78677647f..50bd53f4263 100644 --- a/mysql-test/r/mysqlslap.result +++ b/mysql-test/r/mysqlslap.result @@ -255,3 +255,6 @@ Benchmark # MDEV-4684 - Enhancement request: --init-command support for mysqlslap # DROP TABLE t1; +# +# Bug MDEV-15789 (Upstream: #80329): MYSQLSLAP OPTIONS --AUTO-GENERATE-SQL-GUID-PRIMARY and --AUTO-GENERATE-SQL-SECONDARY-INDEXES DONT WORK +# diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index c49c4ab3d7d..81115d59d09 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -80,3 +80,11 @@ DROP DATABASE bug58090; --exec $MYSQL_SLAP --create-schema=test --init-command="CREATE TABLE t1(a INT)" --silent --concurrency=1 --iterations=1 DROP TABLE t1; + +--echo # +--echo # Bug MDEV-15789 (Upstream: #80329): MYSQLSLAP OPTIONS --AUTO-GENERATE-SQL-GUID-PRIMARY and --AUTO-GENERATE-SQL-SECONDARY-INDEXES DONT WORK +--echo # + +--exec $MYSQL_SLAP --concurrency=1 --silent --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --create-schema=slap + +--exec $MYSQL_SLAP --concurrency=1 --silent --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-secondary-indexes=1 --create-schema=slap From 141bc58ac9926aff0a4eb5bac204f44f60d58840 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 8 Jun 2018 19:52:30 +0100 Subject: [PATCH 092/203] MDEV-16430: mysql_upgrade_service does not write log file. Fixed CreateFile() for the log file - TRUNCATE_EXISTING does not create a new file, use CREATE_ALWAYS - Make log file handle inheritable --- sql/mysql_upgrade_service.cc | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/sql/mysql_upgrade_service.cc b/sql/mysql_upgrade_service.cc index db916101eb1..0723d1c0356 100644 --- a/sql/mysql_upgrade_service.cc +++ b/sql/mysql_upgrade_service.cc @@ -146,6 +146,11 @@ static void die(const char *fmt, ...) exit(1); } +#define WRITE_LOG(fmt,...) {\ + char log_buf[1024]; \ + snprintf(log_buf,sizeof(log_buf), fmt, __VA_ARGS__);\ + WriteFile(logfile_handle,log_buf, strlen(log_buf), 0 , 0);\ +} /* spawn-like function to run subprocesses. @@ -187,17 +192,22 @@ static intptr_t run_tool(int wait_flag, const char *program,...) { char tmpdir[FN_REFLEN]; GetTempPath(FN_REFLEN, tmpdir); - sprintf_s(logfile_path, "%s\\mysql_upgrade_service.%s.log", tmpdir, + sprintf_s(logfile_path, "%smysql_upgrade_service.%s.log", tmpdir, opt_service); - logfile_handle= CreateFile(logfile_path, GENERIC_WRITE, FILE_SHARE_READ, - NULL, TRUNCATE_EXISTING, 0, NULL); - if (!logfile_handle) + SECURITY_ATTRIBUTES attr= {0}; + attr.nLength= sizeof(SECURITY_ATTRIBUTES); + attr.bInheritHandle= TRUE; + logfile_handle= CreateFile(logfile_path, FILE_APPEND_DATA, + FILE_SHARE_READ|FILE_SHARE_WRITE, &attr, CREATE_ALWAYS, 0, NULL); + if (logfile_handle == INVALID_HANDLE_VALUE) { die("Cannot open log file %s, windows error %u", logfile_path, GetLastError()); } } + WRITE_LOG("Executing %s\r\n", cmdline); + /* Start child process */ STARTUPINFO si= {0}; si.cb= sizeof(si); From 5bfd562a00828035ec873e5a6ee0af2c3f52a33a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 8 Jun 2018 20:42:25 +0100 Subject: [PATCH 093/203] MDEV-16445 mysql_upgrade_service should add skip-slave-start to server start parameters --- sql/mysql_upgrade_service.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/mysql_upgrade_service.cc b/sql/mysql_upgrade_service.cc index 0723d1c0356..809161193a5 100644 --- a/sql/mysql_upgrade_service.cc +++ b/sql/mysql_upgrade_service.cc @@ -468,7 +468,7 @@ int main(int argc, char **argv) log("Phase 3/8: Starting mysqld for upgrade"); mysqld_process= (HANDLE)run_tool(P_NOWAIT, mysqld_path, defaults_file_param, "--skip-networking", "--skip-grant-tables", - "--enable-named-pipe", socket_param, NULL); + "--enable-named-pipe", socket_param,"--skip-slave-start", NULL); if (mysqld_process == INVALID_HANDLE_VALUE) { From 15155ecd3483c47d8f2c05727184f6bf47bbf0b0 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 8 Jun 2018 20:42:39 +0100 Subject: [PATCH 094/203] fix typo --- sql/mysql_install_db.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index de799874a8f..f9343aab011 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -195,7 +195,7 @@ int main(int argc, char **argv) die("database creation failed"); } - printf("Creation of the database was successfull"); + printf("Creation of the database was successful"); return 0; } From cd33280b682692f0517a26178d7b5337db648751 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 9 Jun 2018 11:26:52 +0530 Subject: [PATCH 095/203] MDEV-16374: Filtered shows 0 for materilization scan for a semi join, which makes optimizer always picks materialization scan over materialization lookup For non-mergeable semi-joins we don't store the estimates of the IN subquery in table->file->stats.records. In the function TABLE_LIST::fetch_number_of_rows, we store the number of rows in the tables (estimates in case of derived table/views). Currently we don't store the estimates for non-mergeable semi-joins, which leads to a problem of selecting materialization scan over materialization lookup. Fixed this by storing these estimated appropriately --- mysql-test/r/selectivity.result | 70 ++++++++++++++++++++-- mysql-test/r/selectivity_innodb.result | 70 ++++++++++++++++++++-- mysql-test/r/subselect_sj_nonmerged.result | 6 +- mysql-test/t/selectivity.test | 18 ++++++ sql/table.cc | 8 +++ 5 files changed, 161 insertions(+), 11 deletions(-) diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index 61a77d135e7..3e8fb8e2e41 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -356,13 +356,13 @@ and o_orderkey = l_orderkey group by c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice order by o_totalprice desc, o_orderdate; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL distinct_key NULL NULL NULL 6005 0.00 Using temporary; Using filesort -1 PRIMARY orders eq_ref PRIMARY,i_o_custkey PRIMARY 4 .l_orderkey 1 100.00 Using where +1 PRIMARY orders ALL PRIMARY,i_o_custkey NULL NULL NULL 1500 100.00 Using where; Using temporary; Using filesort +1 PRIMARY eq_ref distinct_key distinct_key 4 dbt3_s001.orders.o_orderkey 1 100.00 1 PRIMARY customer eq_ref PRIMARY PRIMARY 4 dbt3_s001.orders.o_custkey 1 100.00 -1 PRIMARY lineitem ref PRIMARY,i_l_orderkey,i_l_orderkey_quantity PRIMARY 4 .l_orderkey 4 100.00 +1 PRIMARY lineitem ref PRIMARY,i_l_orderkey,i_l_orderkey_quantity i_l_orderkey_quantity 4 dbt3_s001.orders.o_orderkey 4 100.00 Using index 2 MATERIALIZED lineitem index NULL i_l_orderkey_quantity 13 NULL 6005 100.00 Using index Warnings: -Note 1003 select `dbt3_s001`.`customer`.`c_name` AS `c_name`,`dbt3_s001`.`customer`.`c_custkey` AS `c_custkey`,`dbt3_s001`.`orders`.`o_orderkey` AS `o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE` AS `o_orderdate`,`dbt3_s001`.`orders`.`o_totalprice` AS `o_totalprice`,sum(`dbt3_s001`.`lineitem`.`l_quantity`) AS `sum(l_quantity)` from (select `dbt3_s001`.`lineitem`.`l_orderkey` from `dbt3_s001`.`lineitem` group by `dbt3_s001`.`lineitem`.`l_orderkey` having (sum(`dbt3_s001`.`lineitem`.`l_quantity`) > 250)) join `dbt3_s001`.`customer` join `dbt3_s001`.`orders` join `dbt3_s001`.`lineitem` where ((`dbt3_s001`.`customer`.`c_custkey` = `dbt3_s001`.`orders`.`o_custkey`) and (`dbt3_s001`.`orders`.`o_orderkey` = ``.`l_orderkey`) and (`dbt3_s001`.`lineitem`.`l_orderkey` = ``.`l_orderkey`)) group by `dbt3_s001`.`customer`.`c_name`,`dbt3_s001`.`customer`.`c_custkey`,`dbt3_s001`.`orders`.`o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE`,`dbt3_s001`.`orders`.`o_totalprice` order by `dbt3_s001`.`orders`.`o_totalprice` desc,`dbt3_s001`.`orders`.`o_orderDATE` +Note 1003 select `dbt3_s001`.`customer`.`c_name` AS `c_name`,`dbt3_s001`.`customer`.`c_custkey` AS `c_custkey`,`dbt3_s001`.`orders`.`o_orderkey` AS `o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE` AS `o_orderdate`,`dbt3_s001`.`orders`.`o_totalprice` AS `o_totalprice`,sum(`dbt3_s001`.`lineitem`.`l_quantity`) AS `sum(l_quantity)` from (select `dbt3_s001`.`lineitem`.`l_orderkey` from `dbt3_s001`.`lineitem` group by `dbt3_s001`.`lineitem`.`l_orderkey` having (sum(`dbt3_s001`.`lineitem`.`l_quantity`) > 250)) join `dbt3_s001`.`customer` join `dbt3_s001`.`orders` join `dbt3_s001`.`lineitem` where ((`dbt3_s001`.`customer`.`c_custkey` = `dbt3_s001`.`orders`.`o_custkey`) and (``.`l_orderkey` = `dbt3_s001`.`orders`.`o_orderkey`) and (`dbt3_s001`.`lineitem`.`l_orderkey` = `dbt3_s001`.`orders`.`o_orderkey`)) group by `dbt3_s001`.`customer`.`c_name`,`dbt3_s001`.`customer`.`c_custkey`,`dbt3_s001`.`orders`.`o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE`,`dbt3_s001`.`orders`.`o_totalprice` order by `dbt3_s001`.`orders`.`o_totalprice` desc,`dbt3_s001`.`orders`.`o_orderDATE` select c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice, sum(l_quantity) from customer, orders, lineitem @@ -1535,6 +1535,68 @@ t 10:00:00 11:00:00 DROP TABLE t1; +# +# MDEV-16374: filtered shows 0 for materilization scan for a semi join, which makes optimizer +# always pick materialization scan over materialization lookup +# +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 (a int, b int); +insert into t1 values (0,0),(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10), +(11,11),(12,12),(13,13),(14,14),(15,15); +set @@optimizer_use_condition_selectivity=2; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 16 100.00 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t1.a 1 100.00 +2 MATERIALIZED t1 ALL NULL NULL NULL NULL 16 100.00 Using temporary +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t1`.`a`) from `test`.`t1` group by `test`.`t1`.`b`) join `test`.`t1` where (``.`max(a)` = `test`.`t1`.`a`) +select * from t1 where a in (select max(a) from t1 group by b); +a b +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +set @@optimizer_use_condition_selectivity=1; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 16 100.00 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t1.a 1 100.00 +2 MATERIALIZED t1 ALL NULL NULL NULL NULL 16 100.00 Using temporary +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t1`.`a`) from `test`.`t1` group by `test`.`t1`.`b`) join `test`.`t1` where (``.`max(a)` = `test`.`t1`.`a`) +select * from t1 where a in (select max(a) from t1 group by b); +a b +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +drop table t1,t0; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index a026c2e6d92..748ef0cb6ca 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -359,13 +359,13 @@ and o_orderkey = l_orderkey group by c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice order by o_totalprice desc, o_orderdate; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL distinct_key NULL NULL NULL 6005 0.00 Using temporary; Using filesort -1 PRIMARY orders eq_ref PRIMARY,i_o_custkey PRIMARY 4 .l_orderkey 1 100.00 Using where +1 PRIMARY orders ALL PRIMARY,i_o_custkey NULL NULL NULL 1500 100.00 Using where; Using temporary; Using filesort +1 PRIMARY eq_ref distinct_key distinct_key 4 dbt3_s001.orders.o_orderkey 1 100.00 1 PRIMARY customer eq_ref PRIMARY PRIMARY 4 dbt3_s001.orders.o_custkey 1 100.00 -1 PRIMARY lineitem ref PRIMARY,i_l_orderkey,i_l_orderkey_quantity PRIMARY 4 .l_orderkey 4 100.00 +1 PRIMARY lineitem ref PRIMARY,i_l_orderkey,i_l_orderkey_quantity i_l_orderkey_quantity 4 dbt3_s001.orders.o_orderkey 4 100.00 Using index 2 MATERIALIZED lineitem index NULL PRIMARY 8 NULL 6005 100.00 Warnings: -Note 1003 select `dbt3_s001`.`customer`.`c_name` AS `c_name`,`dbt3_s001`.`customer`.`c_custkey` AS `c_custkey`,`dbt3_s001`.`orders`.`o_orderkey` AS `o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE` AS `o_orderdate`,`dbt3_s001`.`orders`.`o_totalprice` AS `o_totalprice`,sum(`dbt3_s001`.`lineitem`.`l_quantity`) AS `sum(l_quantity)` from (select `dbt3_s001`.`lineitem`.`l_orderkey` from `dbt3_s001`.`lineitem` group by `dbt3_s001`.`lineitem`.`l_orderkey` having (sum(`dbt3_s001`.`lineitem`.`l_quantity`) > 250)) join `dbt3_s001`.`customer` join `dbt3_s001`.`orders` join `dbt3_s001`.`lineitem` where ((`dbt3_s001`.`customer`.`c_custkey` = `dbt3_s001`.`orders`.`o_custkey`) and (`dbt3_s001`.`orders`.`o_orderkey` = ``.`l_orderkey`) and (`dbt3_s001`.`lineitem`.`l_orderkey` = ``.`l_orderkey`)) group by `dbt3_s001`.`customer`.`c_name`,`dbt3_s001`.`customer`.`c_custkey`,`dbt3_s001`.`orders`.`o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE`,`dbt3_s001`.`orders`.`o_totalprice` order by `dbt3_s001`.`orders`.`o_totalprice` desc,`dbt3_s001`.`orders`.`o_orderDATE` +Note 1003 select `dbt3_s001`.`customer`.`c_name` AS `c_name`,`dbt3_s001`.`customer`.`c_custkey` AS `c_custkey`,`dbt3_s001`.`orders`.`o_orderkey` AS `o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE` AS `o_orderdate`,`dbt3_s001`.`orders`.`o_totalprice` AS `o_totalprice`,sum(`dbt3_s001`.`lineitem`.`l_quantity`) AS `sum(l_quantity)` from (select `dbt3_s001`.`lineitem`.`l_orderkey` from `dbt3_s001`.`lineitem` group by `dbt3_s001`.`lineitem`.`l_orderkey` having (sum(`dbt3_s001`.`lineitem`.`l_quantity`) > 250)) join `dbt3_s001`.`customer` join `dbt3_s001`.`orders` join `dbt3_s001`.`lineitem` where ((`dbt3_s001`.`customer`.`c_custkey` = `dbt3_s001`.`orders`.`o_custkey`) and (``.`l_orderkey` = `dbt3_s001`.`orders`.`o_orderkey`) and (`dbt3_s001`.`lineitem`.`l_orderkey` = `dbt3_s001`.`orders`.`o_orderkey`)) group by `dbt3_s001`.`customer`.`c_name`,`dbt3_s001`.`customer`.`c_custkey`,`dbt3_s001`.`orders`.`o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE`,`dbt3_s001`.`orders`.`o_totalprice` order by `dbt3_s001`.`orders`.`o_totalprice` desc,`dbt3_s001`.`orders`.`o_orderDATE` select c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice, sum(l_quantity) from customer, orders, lineitem @@ -1539,6 +1539,68 @@ t 10:00:00 11:00:00 DROP TABLE t1; +# +# MDEV-16374: filtered shows 0 for materilization scan for a semi join, which makes optimizer +# always pick materialization scan over materialization lookup +# +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 (a int, b int); +insert into t1 values (0,0),(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10), +(11,11),(12,12),(13,13),(14,14),(15,15); +set @@optimizer_use_condition_selectivity=2; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 16 100.00 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t1.a 1 100.00 +2 MATERIALIZED t1 ALL NULL NULL NULL NULL 16 100.00 Using temporary +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t1`.`a`) from `test`.`t1` group by `test`.`t1`.`b`) join `test`.`t1` where (``.`max(a)` = `test`.`t1`.`a`) +select * from t1 where a in (select max(a) from t1 group by b); +a b +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +set @@optimizer_use_condition_selectivity=1; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 16 100.00 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t1.a 1 100.00 +2 MATERIALIZED t1 ALL NULL NULL NULL NULL 16 100.00 Using temporary +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t1`.`a`) from `test`.`t1` group by `test`.`t1`.`b`) join `test`.`t1` where (``.`max(a)` = `test`.`t1`.`a`) +select * from t1 where a in (select max(a) from t1 group by b); +a b +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +drop table t1,t0; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/r/subselect_sj_nonmerged.result b/mysql-test/r/subselect_sj_nonmerged.result index c7e04225ffe..47970668ae5 100644 --- a/mysql-test/r/subselect_sj_nonmerged.result +++ b/mysql-test/r/subselect_sj_nonmerged.result @@ -77,9 +77,9 @@ explain select * from t4 where t4.a in (select max(t2.a) from t1, t2 group by t2.b) and t4.b in (select max(t2.a) from t1, t2 group by t2.b); id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY ALL distinct_key NULL NULL NULL 5 -1 PRIMARY ALL distinct_key NULL NULL NULL 5 Using join buffer (flat, BNL join) -1 PRIMARY t4 ref a a 10 .max(t2.a),.max(t2.a) 12 +1 PRIMARY ALL distinct_key NULL NULL NULL 5 +1 PRIMARY t4 ref a a 5 .max(t2.a) 12 Using index condition +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t4.b 1 3 MATERIALIZED t2 ALL NULL NULL NULL NULL 5 Using temporary 3 MATERIALIZED t1 ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) 2 MATERIALIZED t2 ALL NULL NULL NULL NULL 5 Using temporary diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index 548ef295fb2..afaa937c360 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -1043,6 +1043,24 @@ SELECT * FROM (SELECT t FROM t1 WHERE d IS NULL) sq; DROP TABLE t1; +--echo # +--echo # MDEV-16374: filtered shows 0 for materilization scan for a semi join, which makes optimizer +--echo # always pick materialization scan over materialization lookup +--echo # + +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 (a int, b int); +insert into t1 values (0,0),(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10), +(11,11),(12,12),(13,13),(14,14),(15,15); +set @@optimizer_use_condition_selectivity=2; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +select * from t1 where a in (select max(a) from t1 group by b); +set @@optimizer_use_condition_selectivity=1; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +select * from t1 where a in (select max(a) from t1 group by b); +drop table t1,t0; + set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; diff --git a/sql/table.cc b/sql/table.cc index bc6e1e754ee..b5082df7076 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -7099,7 +7099,15 @@ int TABLE_LIST::fetch_number_of_rows() { int error= 0; if (jtbm_subselect) + { + if (jtbm_subselect->is_jtbm_merged) + { + table->file->stats.records= jtbm_subselect->jtbm_record_count; + set_if_bigger(table->file->stats.records, 2); + table->used_stat_records= table->file->stats.records; + } return 0; + } if (is_materialized_derived() && !fill_me) { From c17468d4abe59339af11ef4a17a59156741c6a21 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 9 Jun 2018 19:04:34 +0530 Subject: [PATCH 096/203] MDEV-16191: Analyze format=json gives incorrect value for r_limit inside a dependent subquery when ORDER BY is present Currently for setting r_limit we divide with the number of iterations we invoke the dependent subquery. This is not needed for the case of limit. For varying limits we produce the output that the limit varies with execution. Also there is a type for filtered , we forgot to multiply by 100 as it is represented as a percent. --- mysql-test/r/analyze_stmt_orderby.result | 2 +- sql/sql_analyze_stmt.cc | 2 +- sql/sql_explain.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/analyze_stmt_orderby.result b/mysql-test/r/analyze_stmt_orderby.result index be1f01a2a52..238baff50e1 100644 --- a/mysql-test/r/analyze_stmt_orderby.result +++ b/mysql-test/r/analyze_stmt_orderby.result @@ -303,7 +303,7 @@ ANALYZE "r_rows": 10, "r_total_time_ms": "REPLACED", "filtered": 100, - "r_filtered": 1, + "r_filtered": 100, "attached_condition": "(t0.a is not null)" } } diff --git a/sql/sql_analyze_stmt.cc b/sql/sql_analyze_stmt.cc index 098e99e88fc..68299d024fd 100644 --- a/sql/sql_analyze_stmt.cc +++ b/sql/sql_analyze_stmt.cc @@ -39,7 +39,7 @@ void Filesort_tracker::print_json_members(Json_writer *writer) if (r_limit == 0) writer->add_str(varied_str); else - writer->add_ll((longlong) rint(r_limit/get_r_loops())); + writer->add_ll((longlong) rint(r_limit)); } writer->add_member("r_used_priority_queue"); diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc index ac6bee05001..82107f9b922 100644 --- a/sql/sql_explain.cc +++ b/sql/sql_explain.cc @@ -1642,7 +1642,7 @@ void Explain_table_access::print_explain_json(Explain_query *query, { /* Get r_filtered value from filesort */ if (fs_tracker->get_r_loops()) - writer->add_double(fs_tracker->get_r_filtered()); + writer->add_double(fs_tracker->get_r_filtered()*100); else writer->add_null(); } From b8e267c0c5007dbfebc2dee8a4f408f28b637ba5 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Wed, 9 May 2018 15:14:57 +0300 Subject: [PATCH 097/203] MDEV-15778: Manually backport TokuDB macOS fixes from 10.0 Fix build on macOS 10.13: 39dceaae607 MDEV-10983: TokuDB does not compile on OS X 10.12 Make use of a different function to get the current tid. Additionally, librt doesn't exist on OS X. Use System library instead. storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake | 4 +++- storage/tokudb/PerconaFT/portability/portability.cc | 9 ++++++++- storage/tokudb/PerconaFT/portability/tests/test-xid.cc | 9 ++++++++- storage/tokudb/PerconaFT/portability/toku_config.h.in | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) --- .../ft-index/cmake_modules/TokuFeatureDetection.cmake | 4 +++- storage/tokudb/ft-index/portability/portability.cc | 9 ++++++++- storage/tokudb/ft-index/portability/tests/test-xid.cc | 9 ++++++++- storage/tokudb/ft-index/portability/toku_config.h.in | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake b/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake index e7fd27525d5..ef7934a5a09 100644 --- a/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake +++ b/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake @@ -96,7 +96,7 @@ if (NOT HAVE_BACKTRACE_WITHOUT_EXECINFO) endif () endif () -if(HAVE_CLOCK_REALTIME) +if(HAVE_CLOCK_REALTIME AND (NOT APPLE)) list(APPEND EXTRA_SYSTEM_LIBS rt) else() list(APPEND EXTRA_SYSTEM_LIBS System) @@ -108,6 +108,8 @@ check_function_exists(pthread_rwlockattr_setkind_np HAVE_PTHREAD_RWLOCKATTR_SETK ## check for the right way to yield using pthreads check_function_exists(pthread_yield HAVE_PTHREAD_YIELD) check_function_exists(pthread_yield_np HAVE_PTHREAD_YIELD_NP) +## check if we have pthread_threadid_np() (i.e. osx) +check_function_exists(pthread_threadid_np HAVE_PTHREAD_THREADID_NP) ## check if we have pthread_getthreadid_np() (i.e. freebsd) check_function_exists(pthread_getthreadid_np HAVE_PTHREAD_GETTHREADID_NP) diff --git a/storage/tokudb/ft-index/portability/portability.cc b/storage/tokudb/ft-index/portability/portability.cc index 09c1ccd50be..9863c45c60d 100644 --- a/storage/tokudb/ft-index/portability/portability.cc +++ b/storage/tokudb/ft-index/portability/portability.cc @@ -115,6 +115,9 @@ PATENT RIGHTS GRANT: #if defined(HAVE_SYS_SYSCTL_H) # include #endif +#if defined(HAVE_PTHREAD_H) +# include +#endif #if defined(HAVE_PTHREAD_NP_H) # include #endif @@ -154,7 +157,11 @@ toku_os_getpid(void) { int toku_os_gettid(void) { -#if defined(__NR_gettid) +#if defined(HAVE_PTHREAD_THREADID_NP) + uint64_t result; + pthread_threadid_np(NULL, &result); + return (int) result; // Used for instrumentation so overflow is ok here. +#elif defined(__NR_gettid) return syscall(__NR_gettid); #elif defined(SYS_gettid) return syscall(SYS_gettid); diff --git a/storage/tokudb/ft-index/portability/tests/test-xid.cc b/storage/tokudb/ft-index/portability/tests/test-xid.cc index 9277f984b43..fb1abd33793 100644 --- a/storage/tokudb/ft-index/portability/tests/test-xid.cc +++ b/storage/tokudb/ft-index/portability/tests/test-xid.cc @@ -103,11 +103,18 @@ PATENT RIGHTS GRANT: #if defined(HAVE_PTHREAD_NP_H) # include #endif +#if defined(HAVE_PTHREAD_H) +# include +#endif // since we implement the same thing here as in toku_os_gettid, this test // is pretty pointless static int gettid(void) { -#if defined(__NR_gettid) +#if defined(HAVE_PTHREAD_THREADID_NP) + uint64_t result; + pthread_threadid_np(NULL, &result); + return (int) result; +#elif defined(__NR_gettid) return syscall(__NR_gettid); #elif defined(SYS_gettid) return syscall(SYS_gettid); diff --git a/storage/tokudb/ft-index/portability/toku_config.h.in b/storage/tokudb/ft-index/portability/toku_config.h.in index c56674ece9e..b8a89e81da6 100644 --- a/storage/tokudb/ft-index/portability/toku_config.h.in +++ b/storage/tokudb/ft-index/portability/toku_config.h.in @@ -57,6 +57,7 @@ #cmakedefine HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP 1 #cmakedefine HAVE_PTHREAD_YIELD 1 #cmakedefine HAVE_PTHREAD_YIELD_NP 1 +#cmakedefine HAVE_PTHREAD_THREADID_NP 1 #cmakedefine HAVE_PTHREAD_GETTHREADID_NP 1 #cmakedefine PTHREAD_YIELD_RETURNS_INT 1 From 1735fa340a9d7ca8683f18fe2ecc148423e78ba7 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Wed, 9 May 2018 16:54:16 +0300 Subject: [PATCH 098/203] MDEV-15778: Remove packed attr from omt_ and subtree_ classes This is happening because they are declared as packed and clang has -Waddress-of-packed-member when passing the address of a packed member, a legit concern on different architectures. The easiest way to get rid of the errors is to remove the packed attribute from said structs. --- storage/tokudb/ft-index/util/omt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ft-index/util/omt.h b/storage/tokudb/ft-index/util/omt.h index 02f3f0d759a..6cf3f80d2bc 100644 --- a/storage/tokudb/ft-index/util/omt.h +++ b/storage/tokudb/ft-index/util/omt.h @@ -181,7 +181,7 @@ public: paranoid_invariant(index != NODE_NULL); m_index = index; } -} __attribute__((__packed__,aligned(4))); +} ; template<> class subtree_templated { @@ -238,7 +238,7 @@ public: inline void disable_bit(void) { m_bitfield &= MASK_INDEX; } -} __attribute__((__packed__)) ; +} ; template class omt_node_templated { @@ -251,7 +251,7 @@ public: // this needs to be in both implementations because we don't have // a "static if" the caller can use inline void clear_stolen_bits(void) {} -} __attribute__((__packed__,aligned(4))); +} ; template class omt_node_templated { @@ -288,7 +288,7 @@ public: this->unset_marked_bit(); this->unset_marks_below_bit(); } -} __attribute__((__packed__,aligned(4))); +} ; } From 8f82c4844342f42c27e2336b64f7bdd1a1344cbd Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Wed, 9 May 2018 16:29:18 +0300 Subject: [PATCH 099/203] MDEV-15778: Restore file permissions lost in merge Permissions were probably due to a file copy in: 15f7f5c6bb4 Merge branch 'merge-tokudb-5.6' into 10.0 --- storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/autogen.sh | 0 .../tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/compile | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/config.rpath | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/config.sub | 0 .../tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/depcomp | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/install-sh | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/ltmain.sh | 0 .../tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/missing | 0 .../third_party/xz-4.999.9beta/extra/7z2lzma/7z2lzma.bash | 0 .../PerconaFT/third_party/xz-4.999.9beta/tests/test_compress.sh | 0 .../PerconaFT/third_party/xz-4.999.9beta/tests/test_files.sh | 0 12 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/autogen.sh mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/compile mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.rpath mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.sub mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/depcomp mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/install-sh mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/ltmain.sh mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/missing mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/extra/7z2lzma/7z2lzma.bash mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_compress.sh mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_files.sh diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/autogen.sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/autogen.sh old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/compile b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/compile old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.rpath b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.rpath old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.sub b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.sub old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/depcomp b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/depcomp old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/install-sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/install-sh old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/ltmain.sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/ltmain.sh old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/missing b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/missing old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/extra/7z2lzma/7z2lzma.bash b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/extra/7z2lzma/7z2lzma.bash old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_compress.sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_compress.sh old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_files.sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_files.sh old mode 100644 new mode 100755 From 7053e26e1869962ef473043686996f40ac0fb88c Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Thu, 10 May 2018 10:00:51 +0300 Subject: [PATCH 100/203] MDEV-15778: Fix TokuDB build issues on macOS 10.13.4 Several issues were encountered and fixed as explained bellow: * missing link to dbug lib; * user proper fprintf format specifier; * ZERO_COND_INITIALIZER was using wrong toku_cond_t struct initializer for first member of type pthread_cond_t and not considering the TOKU_PTHREAD_DEBUG case which has one extra struct member of type pfs_key_t; * Remove likely(!opt_debug_sync_timeout), argument is declared extern and not available to Toku; * pthread_mutex_timedlock() is not available in pthreads for Mac, as it's not part of the POSIX pthreads spec. The encompassing event_t::wait(ms) methods are unused, thus have been removed; --- storage/tokudb/PerconaFT/ft/ft-ops.cc | 4 +- .../PerconaFT/ft/serialize/ft-serialize.cc | 6 +- .../ft/serialize/ft_node-serialize.cc | 58 +++++++++---------- .../PerconaFT/portability/toku_debug_sync.h | 3 - .../PerconaFT/portability/toku_pthread.h | 12 +++- storage/tokudb/PerconaFT/src/CMakeLists.txt | 2 +- storage/tokudb/tokudb_thread.h | 49 ---------------- 7 files changed, 46 insertions(+), 88 deletions(-) diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.cc b/storage/tokudb/PerconaFT/ft/ft-ops.cc index d036366dd63..d7ea5efdff5 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.cc +++ b/storage/tokudb/PerconaFT/ft/ft-ops.cc @@ -821,7 +821,7 @@ int toku_ftnode_fetch_callback(CACHEFILE UU(cachefile), fprintf( stderr, "%s:%d:toku_ftnode_fetch_callback - " - "file[%s], blocknum[%ld], toku_deserialize_ftnode_from " + "file[%s], blocknum[%lld], toku_deserialize_ftnode_from " "failed with a checksum error.\n", __FILE__, __LINE__, @@ -831,7 +831,7 @@ int toku_ftnode_fetch_callback(CACHEFILE UU(cachefile), fprintf( stderr, "%s:%d:toku_ftnode_fetch_callback - " - "file[%s], blocknum[%ld], toku_deserialize_ftnode_from " + "file[%s], blocknum[%lld], toku_deserialize_ftnode_from " "failed with %d.\n", __FILE__, __LINE__, diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc index b24d72a5dff..fe68e1b20e0 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc @@ -656,9 +656,9 @@ exit: fprintf(stderr, \ "%s:%d toku_deserialize_ft_from: " \ "filename[%s] " \ - "r[%d] max_acceptable_lsn[%lu]" \ - "r0[%d] checkpoint_lsn_0[%lu] checkpoint_count_0[%lu] " \ - "r1[%d] checkpoint_lsn_1[%lu] checkpoint_count_1[%lu]\n", \ + "r[%d] max_acceptable_lsn[%llu]" \ + "r0[%d] checkpoint_lsn_0[%llu] checkpoint_count_0[%llu] " \ + "r1[%d] checkpoint_lsn_1[%llu] checkpoint_count_1[%llu]\n", \ __FILE__, \ __LINE__, \ fn, \ diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc index 55899905baf..02d0cc691cd 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc @@ -1170,7 +1170,7 @@ int verify_ftnode_sub_block(struct sub_block *sb, fprintf( stderr, "%s:%d:verify_ftnode_sub_block - " - "file[%s], blocknum[%ld], stored_xsum[%u] != actual_xsum[%u]\n", + "file[%s], blocknum[%lld], stored_xsum[%u] != actual_xsum[%u]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1197,7 +1197,7 @@ static int deserialize_ftnode_info(struct sub_block *sb, FTNODE node) { fprintf( stderr, "%s:%d:deserialize_ftnode_info - " - "file[%s], blocknum[%ld], verify_ftnode_sub_block failed with %d\n", + "file[%s], blocknum[%lld], verify_ftnode_sub_block failed with %d\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1253,7 +1253,7 @@ static int deserialize_ftnode_info(struct sub_block *sb, FTNODE node) { fprintf( stderr, "%s:%d:deserialize_ftnode_info - " - "file[%s], blocknum[%ld], data_size[%d] != rb.ndone[%d]\n", + "file[%s], blocknum[%lld], data_size[%d] != rb.ndone[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1388,7 +1388,7 @@ static int deserialize_ftnode_partition( if (r != 0) { fprintf(stderr, "%s:%d:deserialize_ftnode_partition - " - "file[%s], blocknum[%ld], " + "file[%s], blocknum[%lld], " "verify_ftnode_sub_block failed with %d\n", __FILE__, __LINE__, @@ -1410,7 +1410,7 @@ static int deserialize_ftnode_partition( if (ch != FTNODE_PARTITION_MSG_BUFFER) { fprintf(stderr, "%s:%d:deserialize_ftnode_partition - " - "file[%s], blocknum[%ld], ch[%d] != " + "file[%s], blocknum[%lld], ch[%d] != " "FTNODE_PARTITION_MSG_BUFFER[%d]\n", __FILE__, __LINE__, @@ -1433,7 +1433,7 @@ static int deserialize_ftnode_partition( if (ch != FTNODE_PARTITION_DMT_LEAVES) { fprintf(stderr, "%s:%d:deserialize_ftnode_partition - " - "file[%s], blocknum[%ld], ch[%d] != " + "file[%s], blocknum[%lld], ch[%d] != " "FTNODE_PARTITION_DMT_LEAVES[%d]\n", __FILE__, __LINE__, @@ -1457,7 +1457,7 @@ static int deserialize_ftnode_partition( if (rb.ndone != rb.size) { fprintf(stderr, "%s:%d:deserialize_ftnode_partition - " - "file[%s], blocknum[%ld], rb.ndone[%d] != rb.size[%d]\n", + "file[%s], blocknum[%lld], rb.ndone[%d] != rb.size[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1485,7 +1485,7 @@ static int decompress_and_deserialize_worker(struct rbuf curr_rbuf, const char *fname = toku_ftnode_get_cachefile_fname_in_env(node); fprintf(stderr, "%s:%d:decompress_and_deserialize_worker - " - "file[%s], blocknum[%ld], read_and_decompress_sub_block failed " + "file[%s], blocknum[%lld], read_and_decompress_sub_block failed " "with %d\n", __FILE__, __LINE__, @@ -1502,7 +1502,7 @@ static int decompress_and_deserialize_worker(struct rbuf curr_rbuf, const char *fname = toku_ftnode_get_cachefile_fname_in_env(node); fprintf(stderr, "%s:%d:decompress_and_deserialize_worker - " - "file[%s], blocknum[%ld], deserialize_ftnode_partition failed " + "file[%s], blocknum[%lld], deserialize_ftnode_partition failed " "with %d\n", __FILE__, __LINE__, @@ -1582,7 +1582,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], rb->size[%u] < 24\n", + "file[%s], blocknum[%lld], rb->size[%u] < 24\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1602,7 +1602,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], unrecognized magic number " + "file[%s], blocknum[%lld], unrecognized magic number " "%2.2x %2.2x %2.2x %2.2x %2.2x %2.2x %2.2x %2.2x\n", __FILE__, __LINE__, @@ -1627,7 +1627,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], node->layout_version_read_from_disk[%d] " + "file[%s], blocknum[%lld], node->layout_version_read_from_disk[%d] " "< FT_FIRST_LAYOUT_VERSION_WITH_BASEMENT_NODES[%d]\n", __FILE__, __LINE__, @@ -1667,7 +1667,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], needed_size[%d] > rb->size[%d]\n", + "file[%s], blocknum[%lld], needed_size[%d] > rb->size[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1695,7 +1695,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], stored_checksum[%d] != checksum[%d]\n", + "file[%s], blocknum[%lld], stored_checksum[%d] != checksum[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1717,7 +1717,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], rb->size[%d] - rb->ndone[%d] < " + "file[%s], blocknum[%lld], rb->size[%d] - rb->ndone[%d] < " "sb_node_info.compressed_size[%d] + 8\n", __FILE__, __LINE__, @@ -1744,7 +1744,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], sb_node_info.xsum[%d] != actual_xsum[%d]\n", + "file[%s], blocknum[%lld], sb_node_info.xsum[%d] != actual_xsum[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1774,7 +1774,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], deserialize_ftnode_info failed with " + "file[%s], blocknum[%lld], deserialize_ftnode_info failed with " "%d\n", __FILE__, __LINE__, @@ -1812,7 +1812,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], toku_ftnode_pf_callback failed with " + "file[%s], blocknum[%lld], toku_ftnode_pf_callback failed with " "%d\n", __FILE__, __LINE__, @@ -2164,7 +2164,7 @@ static int deserialize_and_upgrade_ftnode(FTNODE node, const char* fname = toku_cachefile_fname_in_env(bfe->ft->cf); fprintf(stderr, "%s:%d:deserialize_and_upgrade_ftnode - " - "file[%s], blocknum[%ld], " + "file[%s], blocknum[%lld], " "read_and_decompress_block_from_fd_into_rbuf failed with %d\n", __FILE__, __LINE__, @@ -2190,7 +2190,7 @@ static int deserialize_and_upgrade_ftnode(FTNODE node, const char* fname = toku_cachefile_fname_in_env(bfe->ft->cf); fprintf(stderr, "%s:%d:deserialize_and_upgrade_ftnode - " - "file[%s], blocknum[%ld], version[%d] > " + "file[%s], blocknum[%lld], version[%d] > " "FT_LAYOUT_VERSION_14[%d]\n", __FILE__, __LINE__, @@ -2278,7 +2278,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, memcmp(magic, "tokunode", 8) != 0) { fprintf(stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], unrecognized magic number " + "file[%s], blocknum[%lld], unrecognized magic number " "%2.2x %2.2x %2.2x %2.2x %2.2x %2.2x %2.2x %2.2x\n", __FILE__, __LINE__, @@ -2309,7 +2309,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, if (r != 0) { fprintf(stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], deserialize_and_upgrade_ftnode " + "file[%s], blocknum[%lld], deserialize_and_upgrade_ftnode " "failed with %d\n", __FILE__, __LINE__, @@ -2355,7 +2355,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], stored_checksum[%d] != checksum[%d]\n", + "file[%s], blocknum[%lld], stored_checksum[%d] != checksum[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -2377,7 +2377,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], read_and_decompress_sub_block failed " + "file[%s], blocknum[%lld], read_and_decompress_sub_block failed " "with %d\n", __FILE__, __LINE__, @@ -2398,7 +2398,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], deserialize_ftnode_info failed with " + "file[%s], blocknum[%lld], deserialize_ftnode_info failed with " "%d\n", __FILE__, __LINE__, @@ -2470,7 +2470,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], childnum[%d], " + "file[%s], blocknum[%lld], childnum[%d], " "decompress_and_deserialize_worker failed with %d\n", __FILE__, __LINE__, @@ -2490,7 +2490,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], childnum[%d], " + "file[%s], blocknum[%lld], childnum[%d], " "check_and_copy_compressed_sub_block_worker failed with " "%d\n", __FILE__, @@ -2641,7 +2641,7 @@ int toku_deserialize_bp_from_compressed(FTNODE node, const char* fname = toku_cachefile_fname_in_env(bfe->ft->cf); fprintf(stderr, "%s:%d:toku_deserialize_bp_from_compressed - " - "file[%s], blocknum[%ld], " + "file[%s], blocknum[%lld], " "deserialize_ftnode_partition failed with %d\n", __FILE__, __LINE__, @@ -2689,7 +2689,7 @@ static int deserialize_ftnode_from_fd(int fd, fprintf( stderr, "%s:%d:deserialize_ftnode_from_fd - " - "file[%s], blocknum[%ld], deserialize_ftnode_from_rbuf failed with " + "file[%s], blocknum[%lld], deserialize_ftnode_from_rbuf failed with " "%d\n", __FILE__, __LINE__, diff --git a/storage/tokudb/PerconaFT/portability/toku_debug_sync.h b/storage/tokudb/PerconaFT/portability/toku_debug_sync.h index b5394e58d68..493075c36c3 100644 --- a/storage/tokudb/PerconaFT/portability/toku_debug_sync.h +++ b/storage/tokudb/PerconaFT/portability/toku_debug_sync.h @@ -62,9 +62,6 @@ inline void toku_debug_sync(struct tokutxn *txn, const char *sync_point_name) { void *client_extra; THD *thd; - if (likely(!opt_debug_sync_timeout)) - return; - toku_txn_get_client_id(txn, &client_id, &client_extra); thd = reinterpret_cast(client_extra); DEBUG_SYNC(thd, sync_point_name); diff --git a/storage/tokudb/PerconaFT/portability/toku_pthread.h b/storage/tokudb/PerconaFT/portability/toku_pthread.h index e3bd3bce598..a0dfcc246a7 100644 --- a/storage/tokudb/PerconaFT/portability/toku_pthread.h +++ b/storage/tokudb/PerconaFT/portability/toku_pthread.h @@ -162,10 +162,20 @@ typedef struct toku_mutex_aligned { #define ZERO_COND_INITIALIZER \ { 0 } #elif defined(__APPLE__) +#if TOKU_PTHREAD_DEBUG #define ZERO_COND_INITIALIZER \ { \ - { 0 } \ + { 0 , { 0 } }, \ + nullptr, \ + 0 \ } +#else +#define ZERO_COND_INITIALIZER \ + { \ + { 0 , { 0 } }, \ + nullptr \ + } +#endif #else // __linux__, at least #define ZERO_COND_INITIALIZER \ {} diff --git a/storage/tokudb/PerconaFT/src/CMakeLists.txt b/storage/tokudb/PerconaFT/src/CMakeLists.txt index 65bf4814cf8..bae37389004 100644 --- a/storage/tokudb/PerconaFT/src/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/src/CMakeLists.txt @@ -18,7 +18,7 @@ set(tokudb_srcs ## make the shared library add_library(${LIBTOKUDB} SHARED ${tokudb_srcs}) add_dependencies(${LIBTOKUDB} install_tdb_h generate_log_code) -target_link_libraries(${LIBTOKUDB} LINK_PRIVATE locktree_static ft_static util_static lzma snappy ${LIBTOKUPORTABILITY}) +target_link_libraries(${LIBTOKUDB} LINK_PRIVATE locktree_static ft_static util_static lzma snappy dbug ${LIBTOKUPORTABILITY}) target_link_libraries(${LIBTOKUDB} LINK_PUBLIC ${ZLIB_LIBRARY} ) ## make the static library diff --git a/storage/tokudb/tokudb_thread.h b/storage/tokudb/tokudb_thread.h index dec58f3fd35..5df0159901f 100644 --- a/storage/tokudb/tokudb_thread.h +++ b/storage/tokudb/tokudb_thread.h @@ -111,7 +111,6 @@ public: // wait for the event to become signalled void wait(void); - int wait(ulonglong microseconds); // signal the event void signal(void); @@ -152,7 +151,6 @@ public: // wait for the semaphore to become signalled E_WAIT wait(void); - E_WAIT wait(ulonglong microseconds); // signal the semaphore to increase the count // return true if signalled, false if ignored due to count @@ -372,28 +370,6 @@ inline void event_t::wait(void) { assert_debug(r == 0); return; } -inline int event_t::wait(ulonglong microseconds) { - timespec waittime = time::offset_timespec(microseconds); - int r = pthread_mutex_timedlock(&_mutex, &waittime); - if (r == ETIMEDOUT) return ETIMEDOUT; - assert_debug(r == 0); - while (_signalled == false && _pulsed == false) { - r = pthread_cond_timedwait(&_cond, &_mutex, &waittime); - if (r == ETIMEDOUT) { - r = pthread_mutex_unlock(&_mutex); - assert_debug(r == 0); - return ETIMEDOUT; - } - assert_debug(r == 0); - } - if (_manual_reset == false) - _signalled = false; - if (_pulsed) - _pulsed = false; - r = pthread_mutex_unlock(&_mutex); - assert_debug(r == 0); - return 0; -} inline void event_t::signal(void) { int r MY_ATTRIBUTE((unused)) = pthread_mutex_lock(&_mutex); assert_debug(r == 0); @@ -479,31 +455,6 @@ inline semaphore_t::E_WAIT semaphore_t::wait(void) { assert_debug(r == 0); return ret; } -inline semaphore_t::E_WAIT semaphore_t::wait(ulonglong microseconds) { - E_WAIT ret; - timespec waittime = time::offset_timespec(microseconds); - int r = pthread_mutex_timedlock(&_mutex, &waittime); - if (r == ETIMEDOUT) return E_TIMEDOUT; - assert_debug(r == 0); - while (_signalled == 0 && _interrupted == false) { - r = pthread_cond_timedwait(&_cond, &_mutex, &waittime); - if (r == ETIMEDOUT) { - r = pthread_mutex_unlock(&_mutex); - assert_debug(r == 0); - return E_TIMEDOUT; - } - assert_debug(r == 0); - } - if (_interrupted) { - ret = E_INTERRUPTED; - } else { - _signalled--; - ret = E_SIGNALLED; - } - r = pthread_mutex_unlock(&_mutex); - assert_debug(r == 0); - return ret; -} inline bool semaphore_t::signal(void) { bool ret = false; int r MY_ATTRIBUTE((unused)) = pthread_mutex_lock(&_mutex); From 814a284f22b617e2d1ced74ba10d1a870ee6267b Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Thu, 12 Apr 2018 13:33:39 +0300 Subject: [PATCH 101/203] Ignore .cbp QtCreator && CodeBlocks project files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4e3d2a1e66e..0e00a27cbf0 100644 --- a/.gitignore +++ b/.gitignore @@ -236,3 +236,6 @@ storage/mroonga/vendor/groonga/src/grnslap storage/mroonga/vendor/groonga/src/groonga storage/mroonga/vendor/groonga/src/groonga-benchmark storage/mroonga/vendor/groonga/src/suggest/groonga-suggest-create-dataset + +# QtCreator && CodeBlocks +*.cbp From 0e6d6354bf32ac26d36a5a1bb854f2b22de3a494 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Tue, 15 May 2018 10:25:47 +0300 Subject: [PATCH 102/203] Also ignore macOS .DS_Store Finder junk. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0e00a27cbf0..5f2bf5a0fcd 100644 --- a/.gitignore +++ b/.gitignore @@ -237,5 +237,8 @@ storage/mroonga/vendor/groonga/src/groonga storage/mroonga/vendor/groonga/src/groonga-benchmark storage/mroonga/vendor/groonga/src/suggest/groonga-suggest-create-dataset +# macOS garbage +.DS_Store + # QtCreator && CodeBlocks *.cbp From d39629f01ebdd5b89186e6c8a4a8d3dd528bd26a Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Mon, 7 May 2018 22:40:27 +0300 Subject: [PATCH 103/203] MDEV-16075: Workaround to run MTR test suite for make test Assign all tests added via MY_ADD_TEST to a bogus default_ignore target, so that they are not ran by default when doing bare make test. Add default test named MTR that calls mysql-test-run suite, which is now the single test run by make test. In consequence, modified unit/suite.pm to exclude the MTR test and run the real ctests flagged for default_ignore target, thus no circular loop. --- CMakeLists.txt | 4 ++++ cmake/ctest.cmake | 2 +- mysql-test/suite/unit/suite.pm | 2 +- unittest/sql/CMakeLists.txt | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ab17ebf64c..21ed1fcc24b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -366,6 +366,10 @@ INCLUDE(maintainer) IF(WITH_UNIT_TESTS) ENABLE_TESTING() + # This is the only instance where ADD_TEST should be used, + # to make sure that make test will run MTR, + # use MY_ADD_TEST macro to add other tests + ADD_TEST(NAME MTR COMMAND ./mysql-test-run WORKING_DIRECTORY "mysql-test") ADD_SUBDIRECTORY(unittest/mytap) ADD_SUBDIRECTORY(unittest/strings) ADD_SUBDIRECTORY(unittest/examples) diff --git a/cmake/ctest.cmake b/cmake/ctest.cmake index 08852a366f6..fde7e1632f6 100644 --- a/cmake/ctest.cmake +++ b/cmake/ctest.cmake @@ -2,7 +2,7 @@ INCLUDE(${MYSQL_CMAKE_SCRIPT_DIR}/cmake_parse_arguments.cmake) MACRO(MY_ADD_TEST name) - ADD_TEST(${name} ${name}-t) + ADD_TEST(NAME ${name} COMMAND ${name}-t CONFIGURATIONS default_ignore) ENDMACRO() MACRO (MY_ADD_TESTS) diff --git a/mysql-test/suite/unit/suite.pm b/mysql-test/suite/unit/suite.pm index 78d82ccb31d..43eaa970350 100644 --- a/mysql-test/suite/unit/suite.pm +++ b/mysql-test/suite/unit/suite.pm @@ -31,7 +31,7 @@ sub start_test { return "Not run for embedded server" if $::opt_embedded_server; return "Not configured to run ctest" unless -f "../CTestTestfile.cmake"; my ($ctest_vs)= $opt_vs_config ? "--build-config $opt_vs_config" : ""; - my (@ctest_list)= `cd .. && ctest $opt_vs_config --show-only --verbose`; + my (@ctest_list)= `cd .. && ctest $opt_vs_config -E MTR -C default_ignore --show-only --verbose`; return "No ctest" if $?; my ($command, %tests); diff --git a/unittest/sql/CMakeLists.txt b/unittest/sql/CMakeLists.txt index f80f2a5ae70..1efb0ac9cd1 100644 --- a/unittest/sql/CMakeLists.txt +++ b/unittest/sql/CMakeLists.txt @@ -26,4 +26,4 @@ ELSE() ENDIF() TARGET_LINK_LIBRARIES(explain_filename-t sql mytap) -ADD_TEST(explain_filename explain_filename-t) +MY_ADD_TEST(explain_filename explain_filename-t) From 7fca4b50ffbe750532cfcdb95bcd425ec1b8e22b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 15:20:43 +0300 Subject: [PATCH 104/203] Revert "MDEV-15778: Remove packed attr from omt_ and subtree_ classes" This reverts commit 1735fa340a9d7ca8683f18fe2ecc148423e78ba7. --- storage/tokudb/ft-index/util/omt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ft-index/util/omt.h b/storage/tokudb/ft-index/util/omt.h index 6cf3f80d2bc..02f3f0d759a 100644 --- a/storage/tokudb/ft-index/util/omt.h +++ b/storage/tokudb/ft-index/util/omt.h @@ -181,7 +181,7 @@ public: paranoid_invariant(index != NODE_NULL); m_index = index; } -} ; +} __attribute__((__packed__,aligned(4))); template<> class subtree_templated { @@ -238,7 +238,7 @@ public: inline void disable_bit(void) { m_bitfield &= MASK_INDEX; } -} ; +} __attribute__((__packed__)) ; template class omt_node_templated { @@ -251,7 +251,7 @@ public: // this needs to be in both implementations because we don't have // a "static if" the caller can use inline void clear_stolen_bits(void) {} -} ; +} __attribute__((__packed__,aligned(4))); template class omt_node_templated { @@ -288,7 +288,7 @@ public: this->unset_marked_bit(); this->unset_marks_below_bit(); } -} ; +} __attribute__((__packed__,aligned(4))); } From 21246066b296bd9c6e44272a8f93cb7dc73ea43c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 15:54:57 +0300 Subject: [PATCH 105/203] Make TokuDB compile with GCC-8 GCC-8 introduced multiple warnings and increased the level of strictness. * -Wshadow will warn if a local variable shadows a typedef. * GCC will also warn when memsetting a non-trivial type. In this case a non-trivial type can not have a custom constructor. For all intents and purposes, the class is trivially-copyable. * GCC will also warn if you use too many paranthesses which are not necessary --- storage/tokudb/CMakeLists.txt | 6 ++++++ storage/tokudb/ft-index/portability/memory.h | 2 +- storage/tokudb/ft-index/tools/ba_replay.cc | 2 +- storage/tokudb/ft-index/util/dmt.cc | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 903471b097c..9bff7d4729f 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -17,6 +17,12 @@ IF(NOT LIBJEMALLOC) MESSAGE(WARNING "TokuDB is enabled, but jemalloc is not. This configuration is not supported") ENDIF() +CHECK_C_COMPILER_FLAG("-Wshadow" HAVE_WSHADOW) +IF (HAVE_WSHADOW) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-shadow") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-shadow") +ENDIF() + IF (HAVE_WVLA) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-vla") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-vla") diff --git a/storage/tokudb/ft-index/portability/memory.h b/storage/tokudb/ft-index/portability/memory.h index 837b0a70265..52e0c69575f 100644 --- a/storage/tokudb/ft-index/portability/memory.h +++ b/storage/tokudb/ft-index/portability/memory.h @@ -160,7 +160,7 @@ size_t toku_malloc_usable_size(void *p) __attribute__((__visibility__("default") #define XMALLOC(v) CAST_FROM_VOIDP(v, toku_xmalloc(sizeof(*v))) #define XMALLOC_N(n,v) CAST_FROM_VOIDP(v, toku_xmalloc((n)*sizeof(*v))) #define XCALLOC_N(n,v) CAST_FROM_VOIDP(v, toku_xcalloc((n), (sizeof(*v)))) -#define XCALLOC(v) XCALLOC_N(1,(v)) +#define XCALLOC(v) XCALLOC_N(1,v) #define XREALLOC(v,s) CAST_FROM_VOIDP(v, toku_xrealloc(v, s)) #define XREALLOC_N(n,v) CAST_FROM_VOIDP(v, toku_xrealloc(v, (n)*sizeof(*v))) diff --git a/storage/tokudb/ft-index/tools/ba_replay.cc b/storage/tokudb/ft-index/tools/ba_replay.cc index e274ac0a1e8..8f2bc3a02ec 100644 --- a/storage/tokudb/ft-index/tools/ba_replay.cc +++ b/storage/tokudb/ft-index/tools/ba_replay.cc @@ -638,7 +638,7 @@ int main(int argc, char *argv[]) { it == candidate_strategies.begin() ? &stats : &dummy_stats); struct fragmentation_report aggregate_report; - memset(&aggregate_report, 0, sizeof(aggregate_report)); + memset(static_cast(&aggregate_report), 0, sizeof(aggregate_report)); for (map::iterator rp = reports.begin(); rp != reports.end(); rp++) { const struct fragmentation_report &report = rp->second; diff --git a/storage/tokudb/ft-index/util/dmt.cc b/storage/tokudb/ft-index/util/dmt.cc index 3e0b512d7a7..1a7e2db9f40 100644 --- a/storage/tokudb/ft-index/util/dmt.cc +++ b/storage/tokudb/ft-index/util/dmt.cc @@ -132,8 +132,8 @@ void dmt::create_from_sorted_memory_of_fix paranoid_invariant(numvalues > 0); void *ptr = toku_mempool_malloc(&this->mp, aligned_memsize); paranoid_invariant_notnull(ptr); - uint8_t * const CAST_FROM_VOIDP(dest, ptr); - const uint8_t * const CAST_FROM_VOIDP(src, mem); + uint8_t * CAST_FROM_VOIDP(dest, ptr); + const uint8_t * CAST_FROM_VOIDP(src, mem); if (pad_bytes == 0) { paranoid_invariant(aligned_memsize == mem_length); memcpy(dest, src, aligned_memsize); From 953d70f960421ef4a6a2d711b66ef3d04592efc7 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Sun, 10 Jun 2018 16:37:49 +0300 Subject: [PATCH 106/203] MDEV-15778: Remove packed attr from omt_ and subtree_ classes Undo the revert that happened by mystake in commit 7fca4b50ffbe750532cfcdb95bcd425ec1b8e22b. --- storage/tokudb/ft-index/util/omt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ft-index/util/omt.h b/storage/tokudb/ft-index/util/omt.h index 02f3f0d759a..6cf3f80d2bc 100644 --- a/storage/tokudb/ft-index/util/omt.h +++ b/storage/tokudb/ft-index/util/omt.h @@ -181,7 +181,7 @@ public: paranoid_invariant(index != NODE_NULL); m_index = index; } -} __attribute__((__packed__,aligned(4))); +} ; template<> class subtree_templated { @@ -238,7 +238,7 @@ public: inline void disable_bit(void) { m_bitfield &= MASK_INDEX; } -} __attribute__((__packed__)) ; +} ; template class omt_node_templated { @@ -251,7 +251,7 @@ public: // this needs to be in both implementations because we don't have // a "static if" the caller can use inline void clear_stolen_bits(void) {} -} __attribute__((__packed__,aligned(4))); +} ; template class omt_node_templated { @@ -288,7 +288,7 @@ public: this->unset_marked_bit(); this->unset_marks_below_bit(); } -} __attribute__((__packed__,aligned(4))); +} ; } From 1d43f71c7b4c39a6fd196c104b5ebafb65376199 Mon Sep 17 00:00:00 2001 From: Rutuja Surve Date: Sun, 10 Jun 2018 11:19:39 +0300 Subject: [PATCH 107/203] MDEV-15021: mysqldump --tables --routines generates non importable dump file The order of outputting stored procedures is important. Stored procedures must be available on view creation, for views which make use of them. Make sure to print them before outputting tables. --- client/mysqldump.c | 12 ++++++------ mysql-test/r/mysqldump.result | 15 +++++++++++++++ mysql-test/t/mysqldump.test | 22 ++++++++++++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/client/mysqldump.c b/client/mysqldump.c index 4e611ed5cb2..ad71a4dd0ce 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -4701,6 +4701,12 @@ static int dump_selected_tables(char *db, char **table_names, int tables) if (opt_xml) print_xml_tag(md_result_file, "", "\n", "database", "name=", db, NullS); + /* obtain dump of routines (procs/functions) */ + if (opt_routines && mysql_get_server_version(mysql) >= 50009) + { + DBUG_PRINT("info", ("Dumping routines for database %s", db)); + dump_routines_for_db(db); + } /* Dump each selected table */ for (pos= dump_tables; pos < end; pos++) { @@ -4729,12 +4735,6 @@ static int dump_selected_tables(char *db, char **table_names, int tables) DBUG_PRINT("info", ("Dumping events for database %s", db)); dump_events_for_db(db); } - /* obtain dump of routines (procs/functions) */ - if (opt_routines && mysql_get_server_version(mysql) >= 50009) - { - DBUG_PRINT("info", ("Dumping routines for database %s", db)); - dump_routines_for_db(db); - } free_root(&root, MYF(0)); my_free(order_by); order_by= 0; diff --git a/mysql-test/r/mysqldump.result b/mysql-test/r/mysqldump.result index 68de653742c..99b14189282 100644 --- a/mysql-test/r/mysqldump.result +++ b/mysql-test/r/mysqldump.result @@ -5401,3 +5401,18 @@ DELIMITER ; /*!50003 SET collation_connection = @saved_col_connection */ ; ALTER DATABASE `a\"'``b` CHARACTER SET utf8 COLLATE utf8_general_ci ; DROP DATABASE `a\"'``b`; +# +# MDEV-15021: Fix the order in which routines are called +# +use test; +CREATE FUNCTION f() RETURNS INT RETURN 1; +CREATE VIEW v1 AS SELECT f(); +# Running mysqldump -uroot test --routines --tables v1 > **vardir**/test.dmp +DROP VIEW v1; +DROP FUNCTION f; +# Running mysql -uroot test < **vardir**/test.dmp +# +# Cleanup after succesful import. +# +DROP VIEW v1; +DROP FUNCTION f; diff --git a/mysql-test/t/mysqldump.test b/mysql-test/t/mysqldump.test index 39ca5b3c58e..cb12919a823 100644 --- a/mysql-test/t/mysqldump.test +++ b/mysql-test/t/mysqldump.test @@ -2567,3 +2567,25 @@ let SEARCH_FILE=$MYSQLTEST_VARDIR/tmp/bug11505.sql; let SEARCH_PATTERN=Database: mysql; exec $MYSQL_DUMP mysql func > $SEARCH_FILE; source include/search_pattern_in_file.inc; + +--echo # +--echo # MDEV-15021: Fix the order in which routines are called +--echo # +use test; +CREATE FUNCTION f() RETURNS INT RETURN 1; +CREATE VIEW v1 AS SELECT f(); + +--echo # Running mysqldump -uroot test --routines --tables v1 > **vardir**/test.dmp +--exec $MYSQL_DUMP -uroot test --routines --tables v1 > $MYSQLTEST_VARDIR/test.dmp + +DROP VIEW v1; +DROP FUNCTION f; + +--echo # Running mysql -uroot test < **vardir**/test.dmp +--exec $MYSQL -uroot test < $MYSQLTEST_VARDIR/test.dmp + +--echo # +--echo # Cleanup after succesful import. +--echo # +DROP VIEW v1; +DROP FUNCTION f; From 719ed09e5e96b53467ede331631de641f98a348f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 18:25:11 +0300 Subject: [PATCH 108/203] Update test results post-merge --- mysql-test/r/limit.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/r/limit.result b/mysql-test/r/limit.result index deb4ca2ab95..6c295bb86af 100644 --- a/mysql-test/r/limit.result +++ b/mysql-test/r/limit.result @@ -154,7 +154,7 @@ SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Zero limit SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; -start_time user_host query_time lock_time rows_sent rows_examined db last_insert_id insert_id server_id sql_text +start_time user_host query_time lock_time rows_sent rows_examined db last_insert_id insert_id server_id sql_text thread_id EXPLAIN SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; id select_type table type possible_keys key key_len ref rows Extra From e5a3d24b876a266c67b9a6f4390ae31435ec746d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 21:45:05 +0300 Subject: [PATCH 109/203] Followup for make TokuDB compile with GCC-8 Missed printfs from: 21246066b296bd9c6e44272a8f93cb7dc73ea43c --- storage/tokudb/PerconaFT/ft/ft-ops.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.cc b/storage/tokudb/PerconaFT/ft/ft-ops.cc index d7ea5efdff5..eb8696423ca 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.cc +++ b/storage/tokudb/PerconaFT/ft/ft-ops.cc @@ -826,7 +826,7 @@ int toku_ftnode_fetch_callback(CACHEFILE UU(cachefile), __FILE__, __LINE__, toku_cachefile_fname_in_env(cachefile), - blocknum.b); + (longlong)blocknum.b); } else { fprintf( stderr, @@ -836,7 +836,7 @@ int toku_ftnode_fetch_callback(CACHEFILE UU(cachefile), __FILE__, __LINE__, toku_cachefile_fname_in_env(cachefile), - blocknum.b, + (longlong)blocknum.b, r); } // make absolutely sure we crash before doing anything else. From 24d7cbe1e0a16f75e2325c84f23531742e2a035d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 16:26:57 +0300 Subject: [PATCH 110/203] Ensure TokuDB compiles both on Linux and OS X On OS X, (u)int64_t is defined as (unsigned) long long int while on 74 bit Linux it is defined as (unsigned) long int. Ensure the type matches when doing printf on all systems. --- .../PerconaFT/ft/serialize/ft-serialize.cc | 10 ++-- .../ft/serialize/ft_node-serialize.cc | 58 +++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc index fe68e1b20e0..0d6573972d7 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc @@ -663,13 +663,13 @@ exit: __LINE__, \ fn, \ r, \ - max_acceptable_lsn.lsn, \ + (ulonglong)max_acceptable_lsn.lsn, \ r0, \ - checkpoint_lsn_0.lsn, \ - checkpoint_count_0, \ + (ulonglong)checkpoint_lsn_0.lsn, \ + (ulonglong)checkpoint_count_0, \ r1, \ - checkpoint_lsn_1.lsn, \ - checkpoint_count_1); + (ulonglong)checkpoint_lsn_1.lsn, \ + (ulonglong)checkpoint_count_1); int toku_deserialize_ft_from(int fd, const char *fn, diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc index 02d0cc691cd..46bb8f81412 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc @@ -1174,7 +1174,7 @@ int verify_ftnode_sub_block(struct sub_block *sb, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, stored_xsum, actual_xsum); dump_bad_block((Bytef *) sb->uncompressed_ptr, sb->uncompressed_size); @@ -1201,7 +1201,7 @@ static int deserialize_ftnode_info(struct sub_block *sb, FTNODE node) { __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); dump_bad_block(static_cast(sb->uncompressed_ptr), sb->uncompressed_size); @@ -1257,7 +1257,7 @@ static int deserialize_ftnode_info(struct sub_block *sb, FTNODE node) { __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, data_size, rb.ndone); dump_bad_block(rb.buf, rb.size); @@ -1393,7 +1393,7 @@ static int deserialize_ftnode_partition( __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); goto exit; } @@ -1415,7 +1415,7 @@ static int deserialize_ftnode_partition( __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, ch, FTNODE_PARTITION_MSG_BUFFER); dump_bad_block(rb.buf, rb.size); @@ -1438,7 +1438,7 @@ static int deserialize_ftnode_partition( __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, ch, FTNODE_PARTITION_DMT_LEAVES); dump_bad_block(rb.buf, rb.size); @@ -1461,7 +1461,7 @@ static int deserialize_ftnode_partition( __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, rb.ndone, rb.size); dump_bad_block(rb.buf, rb.size); @@ -1490,7 +1490,7 @@ static int decompress_and_deserialize_worker(struct rbuf curr_rbuf, __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); dump_bad_block(curr_rbuf.buf, curr_rbuf.size); goto exit; @@ -1507,7 +1507,7 @@ static int decompress_and_deserialize_worker(struct rbuf curr_rbuf, __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); dump_bad_block(curr_rbuf.buf, curr_rbuf.size); goto exit; @@ -1586,7 +1586,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, rb->size); dump_bad_block(rb->buf, rb->size); // TODO: What error do we return here? @@ -1607,7 +1607,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, static_cast(magic)[0], static_cast(magic)[1], static_cast(magic)[2], @@ -1632,7 +1632,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, node->layout_version_read_from_disk, FT_FIRST_LAYOUT_VERSION_WITH_BASEMENT_NODES); dump_bad_block(rb->buf, rb->size); @@ -1671,7 +1671,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, needed_size, rb->size); dump_bad_block(rb->buf, rb->size); @@ -1699,7 +1699,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, stored_checksum, checksum); dump_bad_block(rb->buf, rb->size); @@ -1722,7 +1722,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, rb->size, rb->ndone, sb_node_info.compressed_size); @@ -1748,7 +1748,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, sb_node_info.xsum, actual_xsum); dump_bad_block(rb->buf, rb->size); @@ -1779,7 +1779,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block( static_cast(sb_node_info.uncompressed_ptr), @@ -1817,7 +1817,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block(rb->buf, rb->size); goto cleanup; @@ -2169,7 +2169,7 @@ static int deserialize_and_upgrade_ftnode(FTNODE node, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); goto exit; } @@ -2195,7 +2195,7 @@ static int deserialize_and_upgrade_ftnode(FTNODE node, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, version, FT_LAYOUT_VERSION_14); dump_bad_block(rb.buf, rb.size); @@ -2283,7 +2283,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, static_cast(magic)[0], static_cast(magic)[1], static_cast(magic)[2], @@ -2314,7 +2314,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block(rb->buf, rb->size); goto cleanup; @@ -2359,7 +2359,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, stored_checksum, checksum); dump_bad_block(rb->buf, rb->size); @@ -2382,7 +2382,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block( static_cast(sb_node_info.uncompressed_ptr), @@ -2403,7 +2403,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block(rb->buf, rb->size); goto cleanup; @@ -2475,7 +2475,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, i, r); dump_bad_block(rb->buf, rb->size); @@ -2496,7 +2496,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, i, r); dump_bad_block(rb->buf, rb->size); @@ -2646,7 +2646,7 @@ int toku_deserialize_bp_from_compressed(FTNODE node, __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); dump_bad_block(static_cast(curr_sb->compressed_ptr), curr_sb->compressed_size); @@ -2694,7 +2694,7 @@ static int deserialize_ftnode_from_fd(int fd, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block(rb.buf, rb.size); } From e7ca377cb73a64e40832e9306194412fb07b8fa5 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 5 Jun 2018 15:21:45 +0200 Subject: [PATCH 111/203] MDEV-16342 SHOW ENGINES: MyISAM description is useless rewrite tautological engine descriptions --- mysql-test/r/information_schema.result | 2 +- mysql-test/suite/federated/have_federatedx.inc | 2 +- mysql-test/suite/funcs_1/r/is_engines_archive.result | 2 +- mysql-test/suite/funcs_1/r/is_engines_csv.result | 2 +- mysql-test/suite/funcs_1/r/is_engines_federated.result | 2 +- mysql-test/suite/funcs_1/r/is_engines_myisam.result | 2 +- storage/archive/ha_archive.cc | 2 +- storage/csv/ha_tina.cc | 2 +- storage/federated/ha_federated.cc | 2 +- storage/federatedx/ha_federatedx.cc | 2 +- storage/myisam/ha_myisam.cc | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index 1f765a70137..dfa7b12562d 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -1387,7 +1387,7 @@ USE test; End of 5.0 tests. select * from information_schema.engines WHERE ENGINE="MyISAM"; ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -MyISAM DEFAULT MyISAM storage engine NO NO NO +MyISAM DEFAULT Non-transactional engine with good performance and small data footprint NO NO NO grant select on *.* to user3148@localhost; select user,db from information_schema.processlist; user db diff --git a/mysql-test/suite/federated/have_federatedx.inc b/mysql-test/suite/federated/have_federatedx.inc index 56ce31f5b2f..2250dd205bd 100644 --- a/mysql-test/suite/federated/have_federatedx.inc +++ b/mysql-test/suite/federated/have_federatedx.inc @@ -1,5 +1,5 @@ if (!`SELECT count(*) FROM information_schema.plugins WHERE plugin_name = 'federated' AND plugin_status = 'active' AND - plugin_description LIKE '%FederatedX%'`){ + plugin_description LIKE '%transactions%'`){ skip Need FederatedX engine; } diff --git a/mysql-test/suite/funcs_1/r/is_engines_archive.result b/mysql-test/suite/funcs_1/r/is_engines_archive.result index 2772992495c..52802b17acd 100644 --- a/mysql-test/suite/funcs_1/r/is_engines_archive.result +++ b/mysql-test/suite/funcs_1/r/is_engines_archive.result @@ -2,7 +2,7 @@ SELECT * FROM information_schema.engines WHERE ENGINE = 'ARCHIVE'; ENGINE ARCHIVE SUPPORT YES -COMMENT Archive storage engine +COMMENT gzip-compresses tables for a low storage footprint TRANSACTIONS NO XA NO SAVEPOINTS NO diff --git a/mysql-test/suite/funcs_1/r/is_engines_csv.result b/mysql-test/suite/funcs_1/r/is_engines_csv.result index 2a7e61ee4d3..7e413b9af6f 100644 --- a/mysql-test/suite/funcs_1/r/is_engines_csv.result +++ b/mysql-test/suite/funcs_1/r/is_engines_csv.result @@ -2,7 +2,7 @@ SELECT * FROM information_schema.engines WHERE ENGINE = 'CSV'; ENGINE CSV SUPPORT YES -COMMENT CSV storage engine +COMMENT Stores tables as CSV files TRANSACTIONS NO XA NO SAVEPOINTS NO diff --git a/mysql-test/suite/funcs_1/r/is_engines_federated.result b/mysql-test/suite/funcs_1/r/is_engines_federated.result index 8057a0266c5..20926458ed0 100644 --- a/mysql-test/suite/funcs_1/r/is_engines_federated.result +++ b/mysql-test/suite/funcs_1/r/is_engines_federated.result @@ -2,7 +2,7 @@ SELECT * FROM information_schema.engines WHERE ENGINE = 'FEDERATED'; ENGINE FEDERATED SUPPORT YES -COMMENT FederatedX pluggable storage engine +COMMENT Allows to access tables on other MariaDB servers, supports transactions and more TRANSACTIONS YES XA NO SAVEPOINTS YES diff --git a/mysql-test/suite/funcs_1/r/is_engines_myisam.result b/mysql-test/suite/funcs_1/r/is_engines_myisam.result index 7e42c864187..d307ce4be6a 100644 --- a/mysql-test/suite/funcs_1/r/is_engines_myisam.result +++ b/mysql-test/suite/funcs_1/r/is_engines_myisam.result @@ -2,7 +2,7 @@ SELECT * FROM information_schema.engines WHERE ENGINE = 'MyISAM'; ENGINE MyISAM SUPPORT DEFAULT -COMMENT MyISAM storage engine +COMMENT Non-transactional engine with good performance and small data footprint TRANSACTIONS NO XA NO SAVEPOINTS NO diff --git a/storage/archive/ha_archive.cc b/storage/archive/ha_archive.cc index d39d1187ce6..3b8e58134b0 100644 --- a/storage/archive/ha_archive.cc +++ b/storage/archive/ha_archive.cc @@ -1840,7 +1840,7 @@ maria_declare_plugin(archive) &archive_storage_engine, "ARCHIVE", "Brian Aker, MySQL AB", - "Archive storage engine", + "gzip-compresses tables for a low storage footprint", PLUGIN_LICENSE_GPL, archive_db_init, /* Plugin Init */ archive_db_done, /* Plugin Deinit */ diff --git a/storage/csv/ha_tina.cc b/storage/csv/ha_tina.cc index 54a85ec3a44..fe39d0e2fa6 100644 --- a/storage/csv/ha_tina.cc +++ b/storage/csv/ha_tina.cc @@ -1800,7 +1800,7 @@ maria_declare_plugin(csv) &csv_storage_engine, "CSV", "Brian Aker, MySQL AB", - "CSV storage engine", + "Stores tables as CSV files", PLUGIN_LICENSE_GPL, tina_init_func, /* Plugin Init */ tina_done_func, /* Plugin Deinit */ diff --git a/storage/federated/ha_federated.cc b/storage/federated/ha_federated.cc index 8680c3aac25..c621653f9f0 100644 --- a/storage/federated/ha_federated.cc +++ b/storage/federated/ha_federated.cc @@ -3503,7 +3503,7 @@ maria_declare_plugin(federated) &federated_storage_engine, "FEDERATED", "Patrick Galbraith and Brian Aker, MySQL AB", - "Federated MySQL storage engine", + "Allows to access tables on other MariaDB servers", PLUGIN_LICENSE_GPL, federated_db_init, /* Plugin Init */ federated_done, /* Plugin Deinit */ diff --git a/storage/federatedx/ha_federatedx.cc b/storage/federatedx/ha_federatedx.cc index bafae614fab..647cc57e101 100644 --- a/storage/federatedx/ha_federatedx.cc +++ b/storage/federatedx/ha_federatedx.cc @@ -3608,7 +3608,7 @@ maria_declare_plugin(federatedx) &federatedx_storage_engine, "FEDERATED", "Patrick Galbraith", - "FederatedX pluggable storage engine", + "Allows to access tables on other MariaDB servers, supports transactions and more", PLUGIN_LICENSE_GPL, federatedx_db_init, /* Plugin Init */ federatedx_done, /* Plugin Deinit */ diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index f63b9c85372..4305fab1778 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -2346,7 +2346,7 @@ maria_declare_plugin(myisam) &myisam_storage_engine, "MyISAM", "MySQL AB", - "MyISAM storage engine", + "Non-transactional engine with good performance and small data footprint", PLUGIN_LICENSE_GPL, myisam_init, /* Plugin Init */ NULL, /* Plugin Deinit */ From 6da8192174f033c2958ddafb2a15c14360bb1ecc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 10 Jun 2018 17:23:53 +0200 Subject: [PATCH 112/203] mysqltest: Allow HANDLER READ in --ps-protocol tests adjust tests --- client/mysqltest.cc | 1 + mysql-test/r/lock.result | 2 +- mysql-test/suite/handler/handler.inc | 2 ++ mysql-test/suite/handler/interface.result | 2 +- mysql-test/suite/handler/interface.test | 2 +- mysql-test/t/lock.test | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 98e35136b2a..43dbff981d3 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -8672,6 +8672,7 @@ void init_re(void) "[[:space:]]*SELECT[[:space:]]|" "[[:space:]]*CREATE[[:space:]]+TABLE[[:space:]]|" "[[:space:]]*DO[[:space:]]|" + "[[:space:]]*HANDLER[[:space:]]+.*[[:space:]]+READ[[:space:]]|" "[[:space:]]*SET[[:space:]]+OPTION[[:space:]]|" "[[:space:]]*DELETE[[:space:]]+MULTI[[:space:]]|" "[[:space:]]*UPDATE[[:space:]]+MULTI[[:space:]]|" diff --git a/mysql-test/r/lock.result b/mysql-test/r/lock.result index 501c379b257..0dcc0de828f 100644 --- a/mysql-test/r/lock.result +++ b/mysql-test/r/lock.result @@ -407,7 +407,7 @@ LOCK TABLE t1 WRITE; HANDLER t1 OPEN; ERROR HY000: Can't execute the given command because you have active locked tables or an active transaction HANDLER t1 READ FIRST; -ERROR HY000: Can't execute the given command because you have active locked tables or an active transaction +Got one of the listed errors HANDLER t1 CLOSE; ERROR HY000: Can't execute the given command because you have active locked tables or an active transaction UNLOCK TABLES; diff --git a/mysql-test/suite/handler/handler.inc b/mysql-test/suite/handler/handler.inc index 5b2dd5ef7e9..bfc21a3b2c3 100644 --- a/mysql-test/suite/handler/handler.inc +++ b/mysql-test/suite/handler/handler.inc @@ -377,7 +377,9 @@ send optimize table t1; # client 1 --echo proceed with the normal connection connection default; +--disable_ps_protocol handler t1 read next; +--enable_ps_protocol handler t1 close; # client 2 --echo read the result from the other connection diff --git a/mysql-test/suite/handler/interface.result b/mysql-test/suite/handler/interface.result index fb633bb94c4..c9cffba33f7 100644 --- a/mysql-test/suite/handler/interface.result +++ b/mysql-test/suite/handler/interface.result @@ -269,7 +269,7 @@ handler t1 open; lock table t1 write; alter table t1 engine=csv; handler t1 read a next; -ERROR HY000: Table storage engine for 't1' doesn't have this option +Got one of the listed errors handler t1 close; unlock tables; drop table t1; diff --git a/mysql-test/suite/handler/interface.test b/mysql-test/suite/handler/interface.test index 2ef617c3ce7..06797ed6980 100644 --- a/mysql-test/suite/handler/interface.test +++ b/mysql-test/suite/handler/interface.test @@ -326,7 +326,7 @@ let $wait_condition= info = "alter table t1 engine=csv"; --source include/wait_condition.inc connection default; ---error ER_ILLEGAL_HA +--error ER_ILLEGAL_HA,ER_KEY_DOES_NOT_EXITS handler t1 read a next; handler t1 close; connection con1; diff --git a/mysql-test/t/lock.test b/mysql-test/t/lock.test index 78f0e2ecf8d..6cfaf9fc320 100644 --- a/mysql-test/t/lock.test +++ b/mysql-test/t/lock.test @@ -481,7 +481,7 @@ LOCK TABLE t1 WRITE; --echo # HANDLER commands are not allowed in LOCK TABLES mode --error ER_LOCK_OR_ACTIVE_TRANSACTION HANDLER t1 OPEN; ---error ER_LOCK_OR_ACTIVE_TRANSACTION +--error ER_LOCK_OR_ACTIVE_TRANSACTION,ER_UNKNOWN_TABLE HANDLER t1 READ FIRST; --error ER_LOCK_OR_ACTIVE_TRANSACTION HANDLER t1 CLOSE; From ca733d03c82b02cd842ff2a226fee7b12eb86f8d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 10 Jun 2018 21:19:11 +0200 Subject: [PATCH 113/203] MDEV-15729 Server crashes in Field::make_field upon HANDLER READ executed with PS protocol update table->pos_in_table_list during prepare, just like it's done in normal execution. otherwise it'll be a dangling pointer --- mysql-test/suite/handler/ps.result | 9 +++++++++ mysql-test/suite/handler/ps.test | 11 +++++++++++ sql/sql_handler.cc | 1 + 3 files changed, 21 insertions(+) create mode 100644 mysql-test/suite/handler/ps.result create mode 100644 mysql-test/suite/handler/ps.test diff --git a/mysql-test/suite/handler/ps.result b/mysql-test/suite/handler/ps.result new file mode 100644 index 00000000000..54685f9156b --- /dev/null +++ b/mysql-test/suite/handler/ps.result @@ -0,0 +1,9 @@ +create table t1 (i int); +handler test.t1 open handler_a; +flush status; +handler handler_a read first; +i +show status like 'Com_stmt_prepare%'; +Variable_name Value +Com_stmt_prepare OK +drop table t1; diff --git a/mysql-test/suite/handler/ps.test b/mysql-test/suite/handler/ps.test new file mode 100644 index 00000000000..68091190c85 --- /dev/null +++ b/mysql-test/suite/handler/ps.test @@ -0,0 +1,11 @@ +# +# MDEV-15729 Server crashes in Field::make_field upon HANDLER READ executed with PS protocol +# +create table t1 (i int); +handler test.t1 open handler_a; +flush status; +handler handler_a read first; +# handler...read must be prepared in --ps-protocol mode +--replace_result $PS_PROTOCOL OK +show status like 'Com_stmt_prepare%'; +drop table t1; diff --git a/sql/sql_handler.cc b/sql/sql_handler.cc index f5c79e59bf2..778507ebc38 100644 --- a/sql/sql_handler.cc +++ b/sql/sql_handler.cc @@ -980,6 +980,7 @@ SQL_HANDLER *mysql_ha_read_prepare(THD *thd, TABLE_LIST *tables, if (!(handler= mysql_ha_find_handler(thd, tables->alias))) DBUG_RETURN(0); tables->table= handler->table; // This is used by fix_fields + handler->table->pos_in_table_list= tables; if (mysql_ha_fix_cond_and_key(handler, mode, keyname, key_expr, cond, 1)) DBUG_RETURN(0); DBUG_RETURN(handler); From 5227198908d4f12ab3e5c95e4c8f15e6b9805f10 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 11 Jun 2018 16:29:22 +0400 Subject: [PATCH 114/203] MDEV-16190 Server crashes in Item_null_result::field_type on SELECT with time field, ROLLUP and HAVING virtual Item_null_result::get_date() was not overridden. It used the inherited Item::get_date(), which tests field_type(), which in case of Item_null_result calls result_field->field_type(), and result_field is not really always set (e.g. it's not set in the test case from the bug report). Overriding Item_null::get_date() like it's done for other val_xxx() methods. This make the code more symmetric across data types. In the new reduction, get_date() immediately returns NULL without entering into any data type specific code. --- mysql-test/r/olap.result | 22 ++++++++++++++++++++++ mysql-test/t/olap.test | 23 +++++++++++++++++++++++ sql/item.cc | 28 +++++++++++++++++++++------- sql/item.h | 7 +++++++ 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/mysql-test/r/olap.result b/mysql-test/r/olap.result index b10f175b63e..24140583d13 100644 --- a/mysql-test/r/olap.result +++ b/mysql-test/r/olap.result @@ -766,3 +766,25 @@ b NULL DROP TABLE t1, t2; End of 5.0 tests +# +# Start of 10.1 tests +# +# +# MDEV-16190 Server crashes in Item_null_result::field_type on SELECT with time field, ROLLUP and HAVING +# +CREATE TABLE t1 (t TIME) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('12:12:12'); +SELECT t, COUNT(*) FROM t1 GROUP BY t WITH ROLLUP HAVING t > '00:00:00'; +t COUNT(*) +12:12:12 1 +DROP TABLE t1; +CREATE TABLE t1 (t TIME) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('12:12:12'),('12:12:13'); +SELECT t, COUNT(*) FROM t1 GROUP BY t WITH ROLLUP HAVING t > '00:00:00'; +t COUNT(*) +12:12:12 1 +12:12:13 1 +DROP TABLE t1; +# +# End of 10.1 tests +# diff --git a/mysql-test/t/olap.test b/mysql-test/t/olap.test index fec5df1a1c7..4a61cebdc0d 100644 --- a/mysql-test/t/olap.test +++ b/mysql-test/t/olap.test @@ -404,3 +404,26 @@ SELECT DISTINCT b FROM t1, t2 GROUP BY a, b WITH ROLLUP; DROP TABLE t1, t2; --echo End of 5.0 tests + + +--echo # +--echo # Start of 10.1 tests +--echo # + +--echo # +--echo # MDEV-16190 Server crashes in Item_null_result::field_type on SELECT with time field, ROLLUP and HAVING +--echo # +CREATE TABLE t1 (t TIME) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('12:12:12'); +SELECT t, COUNT(*) FROM t1 GROUP BY t WITH ROLLUP HAVING t > '00:00:00'; +DROP TABLE t1; + +CREATE TABLE t1 (t TIME) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('12:12:12'),('12:12:13'); +SELECT t, COUNT(*) FROM t1 GROUP BY t WITH ROLLUP HAVING t > '00:00:00'; +DROP TABLE t1; + + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/sql/item.cc b/sql/item.cc index be04f64862c..e23829c750c 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -1287,7 +1287,7 @@ bool Item::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) ltime, fuzzydate, field_name_or_null())) goto err; - break; + return null_value= false; } case REAL_RESULT: { @@ -1295,7 +1295,7 @@ bool Item::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) if (null_value || double_to_datetime_with_warn(value, ltime, fuzzydate, field_name_or_null())) goto err; - break; + return null_value= false; } case DECIMAL_RESULT: { @@ -1304,7 +1304,7 @@ bool Item::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) decimal_to_datetime_with_warn(res, ltime, fuzzydate, field_name_or_null())) goto err; - break; + return null_value= false; } case STRING_RESULT: { @@ -1314,15 +1314,20 @@ bool Item::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) str_to_datetime_with_warn(res->charset(), res->ptr(), res->length(), ltime, fuzzydate)) goto err; - break; + return null_value= false; } default: + null_value= true; DBUG_ASSERT(0); } - return null_value= 0; - err: + return null_value|= make_zero_date(ltime, fuzzydate); +} + + +bool Item::make_zero_date(MYSQL_TIME *ltime, ulonglong fuzzydate) +{ /* if the item was not null and convertion failed, we return a zero date if allowed, otherwise - null. @@ -1344,7 +1349,7 @@ err: */ ltime->time_type= MYSQL_TIMESTAMP_TIME; } - return null_value|= !(fuzzydate & TIME_FUZZY_DATES); + return !(fuzzydate & TIME_FUZZY_DATES); } bool Item::get_seconds(ulonglong *sec, ulong *sec_part) @@ -3132,6 +3137,15 @@ my_decimal *Item_null::val_decimal(my_decimal *decimal_value) } +bool Item_null::get_date(MYSQL_TIME *ltime, ulonglong fuzzydate) +{ + // following assert is redundant, because fixed=1 assigned in constructor + DBUG_ASSERT(fixed == 1); + make_zero_date(ltime, fuzzydate); + return (null_value= true); +} + + Item *Item_null::safe_charset_converter(THD *thd, CHARSET_INFO *tocs) { return this; diff --git a/sql/item.h b/sql/item.h index 60073f054c9..139ce40b25b 100644 --- a/sql/item.h +++ b/sql/item.h @@ -663,6 +663,12 @@ protected: SEL_TREE *get_mm_tree_for_const(RANGE_OPT_PARAM *param); Field *create_tmp_field(bool group, TABLE *table, uint convert_int_length); + /* + This method is used if the item was not null but convertion to + TIME/DATE/DATETIME failed. We return a zero date if allowed, + otherwise - null. + */ + bool make_zero_date(MYSQL_TIME *ltime, ulonglong fuzzydate); public: /* @@ -2565,6 +2571,7 @@ public: longlong val_int(); String *val_str(String *str); my_decimal *val_decimal(my_decimal *); + bool get_date(MYSQL_TIME *ltime, ulonglong fuzzydate); int save_in_field(Field *field, bool no_conversions); int save_safe_in_field(Field *field); bool send(Protocol *protocol, String *str); From 147744d455d57c753ca75fccca106d3326d251ab Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Mon, 11 Jun 2018 08:52:26 -0700 Subject: [PATCH 115/203] MDEV-16235 Server crashes in my_utf8_uni or in my_strtod_int upon SELECT .. LIMIT 0 (new variant) This is another attempt to fix the problem of mdev-14515. --- mysql-test/r/having.result | 11 +++++++++++ mysql-test/r/subselect4.result | 4 ++-- mysql-test/t/having.test | 9 +++++++++ sql/opt_subselect.cc | 3 ++- sql/sql_select.cc | 1 - 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/having.result b/mysql-test/r/having.result index 18915da1a09..9e0466efb9f 100644 --- a/mysql-test/r/having.result +++ b/mysql-test/r/having.result @@ -712,3 +712,14 @@ a ct 4 2 set sql_mode=@save_sql_mode; drop table t1; +# +# mdev-16235: impossible HAVING in query without aggregation +# +explain extended +select * from mysql.help_topic where example = 'foo' having description is null; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible HAVING +Warnings: +Note 1003 select `mysql`.`help_topic`.`help_topic_id` AS `help_topic_id`,`mysql`.`help_topic`.`name` AS `name`,`mysql`.`help_topic`.`help_category_id` AS `help_category_id`,`mysql`.`help_topic`.`description` AS `description`,`mysql`.`help_topic`.`example` AS `example`,`mysql`.`help_topic`.`url` AS `url` from `mysql`.`help_topic` where (`mysql`.`help_topic`.`example` = 'foo') having 0 +select * from mysql.help_topic where example = 'foo' having description is null; +help_topic_id name help_category_id description example url diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 8973330c7da..f726bcfa15e 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -1056,7 +1056,7 @@ EXPLAIN SELECT * FROM t1 WHERE (2, 0) NOT IN (SELECT min(f3)+f3, min(f4)+f3+max(f4) FROM t2 WHERE f3 > 10); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE -2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL No matching min/max row SELECT * FROM t1 WHERE (2, 0) NOT IN (SELECT min(f3)+f3, min(f4)+f3+max(f4) FROM t2 WHERE f3 > 10); f1 f2 SET @@optimizer_switch = 'materialization=off,in_to_exists=on,semijoin=off'; @@ -1147,7 +1147,7 @@ EXPLAIN SELECT * FROM t1 WHERE (2, 0) NOT IN (SELECT min(f3)+f3, min(f4)+f3+max(f4) FROM t2 WHERE f3 > 10); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE -2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL No matching min/max row SELECT * FROM t1 WHERE (2, 0) NOT IN (SELECT min(f3)+f3, min(f4)+f3+max(f4) FROM t2 WHERE f3 > 10); f1 f2 set @@optimizer_switch=@save_optimizer_switch; diff --git a/mysql-test/t/having.test b/mysql-test/t/having.test index a470f462d6a..c8bd47940cc 100644 --- a/mysql-test/t/having.test +++ b/mysql-test/t/having.test @@ -744,3 +744,12 @@ SELECT a, COUNT(a) as ct FROM t1 GROUP BY a HAVING ct>0; set sql_mode=@save_sql_mode; drop table t1; + +--echo # +--echo # mdev-16235: impossible HAVING in query without aggregation +--echo # + +explain extended +select * from mysql.help_topic where example = 'foo' having description is null; + +select * from mysql.help_topic where example = 'foo' having description is null; diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index a7edd64e68b..ec7b10f20c8 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -5880,6 +5880,7 @@ bool JOIN::choose_tableless_subquery_plan() functions produce empty subquery result. There is no need to further rewrite the subquery because it will not be executed at all. */ + exec_const_cond= 0; return FALSE; } @@ -5911,6 +5912,6 @@ bool JOIN::choose_tableless_subquery_plan() tmp_having= having; } } - exec_const_cond= conds; + exec_const_cond= zero_result_cause ? 0 : conds; return FALSE; } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 016dd594e08..b79431ae2f6 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1147,7 +1147,6 @@ JOIN::optimize() { DBUG_PRINT("info", ("Zero limit")); zero_result_cause= "Zero limit"; - conds= 0; } table_count= top_join_tab_count= 0; error= 0; From e425216045c7a998139bd953b0aed83c3a085c8c Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 31 Jan 2018 09:35:38 +0100 Subject: [PATCH 116/203] MDEV-15113: Hang in Aria loghandler Added unregistering writers in case of log error. Added more debugging control about adding/removing writers to the buffers. --- storage/maria/ma_loghandler.c | 219 +++++++++++++++++++++++++++++----- 1 file changed, 187 insertions(+), 32 deletions(-) diff --git a/storage/maria/ma_loghandler.c b/storage/maria/ma_loghandler.c index 2118b3b6ce6..7cca14d0785 100644 --- a/storage/maria/ma_loghandler.c +++ b/storage/maria/ma_loghandler.c @@ -78,6 +78,32 @@ typedef union uchar buffer[TRANSLOG_PAGE_SIZE]; } TRANSLOG_PAGE_SIZE_BUFF; +#define MAX_TRUNSLOG_USED_BUFFERS 3 + +typedef struct +{ + struct st_translog_buffer *buff[MAX_TRUNSLOG_USED_BUFFERS]; + uint8 wrt_ptr; + uint8 unlck_ptr; +} TRUNSLOG_USED_BUFFERS; + +static void +used_buffs_init(TRUNSLOG_USED_BUFFERS *buffs) +{ + buffs->unlck_ptr= buffs->wrt_ptr= 0; +} + +static void +used_buffs_add(TRUNSLOG_USED_BUFFERS *buffs, + struct st_translog_buffer *buff); + +static void +used_buffs_register_unlock(TRUNSLOG_USED_BUFFERS *buffs, + struct st_translog_buffer *buff); + +static void +used_buffs_urgent_unlock(TRUNSLOG_USED_BUFFERS *buffs); + /* min chunk length */ #define TRANSLOG_MIN_CHUNK 3 /* @@ -156,7 +182,28 @@ struct st_translog_buffer TRANSLOG_FILE *file; /* Threads which are waiting for buffer filling/freeing */ mysql_cond_t waiting_filling_buffer; - /* Number of records which are in copy progress */ + /* + Number of records which are in copy progress. + + Controlled via translog_buffer_increase_writers() and + translog_buffer_decrease_writers(). + + 1 Simple case: translog_force_current_buffer_to_finish both called in + the same procedure. + + 2 Simple case: translog_write_variable_record_1group: + translog_advance_pointer() increase writer of the buffer and + translog_buffer_decrease_writers() decrease it. + + Usual case: + 1) translog_advance_pointer (i.e. reserve place for future writing) + increase writers for all buffers where place reserved. + Simpliest case: just all space reserved in one buffer + complex case: end of the first buffer, all second buffer, beginning + of the third buffer. + 2) When we finish with writing translog_chaser_page_next() will be + called and unlock the buffer by decreasing number of writers. + */ uint copy_to_buffer_in_progress; /* list of waiting buffer ready threads */ struct st_my_thread_var *waiting_flush; @@ -214,6 +261,7 @@ struct st_translog_buffer struct st_buffer_cursor { + TRUNSLOG_USED_BUFFERS buffs; /* pointer into the buffer */ uchar *ptr; /* current buffer */ @@ -1663,15 +1711,12 @@ static my_bool translog_create_new_file() DBUG_PRINT("info", ("file_no: %lu", (ulong)file_no)); if (translog_write_file_header()) - DBUG_RETURN(1); + goto error; if (ma_control_file_write_and_force(last_checkpoint_lsn, file_no, max_trid_in_control_file, recovery_failures)) - { - translog_stop_writing(); - DBUG_RETURN(1); - } + goto error; DBUG_RETURN(0); @@ -1711,10 +1756,6 @@ static void translog_buffer_lock(struct st_translog_buffer *buffer) SYNOPSIS translog_buffer_unlock() buffer This buffer which should be unlocked - - RETURN - 0 OK - 1 Error */ static void translog_buffer_unlock(struct st_translog_buffer *buffer) @@ -1908,7 +1949,10 @@ static void translog_finish_page(TRANSLOG_ADDRESS *horizon, (ulong) cursor->buffer->size, (ulong) (cursor->ptr -cursor->buffer->buffer), (uint) cursor->current_page_fill, (uint) left)); - DBUG_ASSERT(LSN_FILE_NO(*horizon) == LSN_FILE_NO(cursor->buffer->offset)); + DBUG_ASSERT(LSN_FILE_NO(*horizon) == LSN_FILE_NO(cursor->buffer->offset) + || translog_status == TRANSLOG_UNINITED); + if ((LSN_FILE_NO(*horizon) != LSN_FILE_NO(cursor->buffer->offset))) + DBUG_VOID_RETURN; // everything wrong do not write to awoid more problems translog_check_cursor(cursor); if (cursor->protected) { @@ -4607,6 +4651,7 @@ static my_bool translog_chaser_page_next(TRANSLOG_ADDRESS *horizon, { translog_buffer_lock(buffer_to_flush); translog_buffer_decrease_writers(buffer_to_flush); + used_buffs_register_unlock(&cursor->buffs, buffer_to_flush); if (!rc) rc= translog_buffer_flush(buffer_to_flush); translog_buffer_unlock(buffer_to_flush); @@ -4711,7 +4756,8 @@ translog_write_variable_record_chunk3_page(struct st_translog_parts *parts, 1 Error */ -static my_bool translog_advance_pointer(int pages, uint16 last_page_data) +static my_bool translog_advance_pointer(int pages, uint16 last_page_data, + TRUNSLOG_USED_BUFFERS *buffs) { translog_size_t last_page_offset= (log_descriptor.page_overhead + last_page_data); @@ -4728,6 +4774,8 @@ static my_bool translog_advance_pointer(int pages, uint16 last_page_data) (uint) last_page_data)); translog_lock_assert_owner(); + used_buffs_init(buffs); + if (pages == -1) { /* @@ -4805,8 +4853,10 @@ static my_bool translog_advance_pointer(int pages, uint16 last_page_data) translog_wait_for_buffer_free(new_buffer); #ifndef DBUG_OFF /* We keep the handler locked so nobody can start this new buffer */ - DBUG_ASSERT(offset == new_buffer->offset && new_buffer->file == NULL && - (file == NULL ? ver : (uint8)(ver + 1)) == new_buffer->ver); + DBUG_ASSERT((offset == new_buffer->offset && new_buffer->file == NULL && + (file == NULL ? ver : (uint8)(ver + 1)) == + new_buffer->ver) || + translog_status == TRANSLOG_READONLY); } #endif @@ -4827,6 +4877,8 @@ static my_bool translog_advance_pointer(int pages, uint16 last_page_data) DBUG_ASSERT(log_descriptor.bc.buffer->buffer_no == log_descriptor.bc.buffer_no); translog_buffer_increase_writers(log_descriptor.bc.buffer); + // register for case of error + used_buffs_add(buffs, log_descriptor.bc.buffer); if (file_end_offset <= buffer_end_offset) { @@ -4837,6 +4889,10 @@ static my_bool translog_advance_pointer(int pages, uint16 last_page_data) (ulong) LSN_FILE_NO(log_descriptor.horizon))); if (translog_create_new_file()) { + struct st_translog_buffer *ob= log_descriptor.bc.buffer; + translog_buffer_unlock(ob); + used_buffs_urgent_unlock(buffs); + translog_buffer_lock(ob); DBUG_RETURN(1); } } @@ -4858,6 +4914,7 @@ end: log_descriptor.bc.ptr+= offset; log_descriptor.bc.buffer->size+= offset; translog_buffer_increase_writers(log_descriptor.bc.buffer); + used_buffs_add(buffs, log_descriptor.bc.buffer); log_descriptor.horizon+= offset; /* offset increasing */ log_descriptor.bc.current_page_fill= last_page_offset; DBUG_PRINT("info", ("NewP buffer #%u: 0x%lx chaser: %d Size: %lu (%lu) " @@ -4878,6 +4935,56 @@ end: DBUG_RETURN(0); } +static void +used_buffs_add(TRUNSLOG_USED_BUFFERS *buffs, + struct st_translog_buffer *buff) +{ + DBUG_ENTER("used_buffs_add"); + DBUG_PRINT("enter", ("ADD buffs: %p unlk %u (%p) wrt_ptr: %u (%p)" + " buff %p (%u)", + buffs, + buffs->wrt_ptr, buffs->buff[buffs->wrt_ptr], + buffs->unlck_ptr, buffs->buff[buffs->unlck_ptr], + buff, buff->buffer_no)); + DBUG_ASSERT(buffs->wrt_ptr < MAX_TRUNSLOG_USED_BUFFERS); + buffs->buff[buffs->wrt_ptr++]= buff; + DBUG_VOID_RETURN; +} + +static void +used_buffs_register_unlock(TRUNSLOG_USED_BUFFERS *buffs, + struct st_translog_buffer *buff + __attribute__((unused)) ) +{ + DBUG_ENTER("used_buffs_register_unlock"); + DBUG_PRINT("enter", ("SUB buffs: %p unlk %u (%p) wrt_ptr: %u (%p)" + " buff %p (%u)", + buffs, + buffs->wrt_ptr, buffs->buff[buffs->wrt_ptr], + buffs->unlck_ptr, buffs->buff[buffs->unlck_ptr], + buff, buff->buffer_no)); + DBUG_ASSERT(buffs->buff[buffs->unlck_ptr] == buff); + buffs->unlck_ptr++; + DBUG_VOID_RETURN; +} +static void used_buffs_urgent_unlock(TRUNSLOG_USED_BUFFERS *buffs) +{ + uint i; + DBUG_ENTER("used_buffs_urgent_unlock"); + translog_lock(); + translog_stop_writing(); + translog_unlock(); + for (i= buffs->unlck_ptr; i < buffs->wrt_ptr; i++) + { + struct st_translog_buffer *buf= buffs->buff[i]; + translog_buffer_lock(buf); + translog_buffer_decrease_writers(buf); + translog_buffer_unlock(buf); + buffs->buff[i]= NULL; + } + used_buffs_init(buffs); + DBUG_VOID_RETURN; +} /* Get page rest @@ -5016,6 +5123,11 @@ translog_write_variable_record_1group(LSN *lsn, lsn, hook_arg))) { translog_unlock(); + if (buffer_to_flush != NULL) + { + translog_buffer_flush(buffer_to_flush); + translog_buffer_unlock(buffer_to_flush); + } DBUG_RETURN(1); } cursor= log_descriptor.bc; @@ -5046,8 +5158,9 @@ translog_write_variable_record_1group(LSN *lsn, (log_descriptor.page_capacity_chunk_2 - 1), record_rest, parts->record_length)); /* record_rest + 3 is chunk type 3 overhead + record_rest */ - rc|= translog_advance_pointer((int)(full_pages + additional_chunk3_page), - (record_rest ? record_rest + 3 : 0)); + rc= translog_advance_pointer((int)(full_pages + additional_chunk3_page), + (record_rest ? record_rest + 3 : 0), + &cursor.buffs); log_descriptor.bc.buffer->last_lsn= *lsn; DBUG_PRINT("info", ("last_lsn set to (%lu,0x%lx) buffer: 0x%lx", LSN_IN_PARTS(log_descriptor.bc.buffer->last_lsn), @@ -5066,7 +5179,11 @@ translog_write_variable_record_1group(LSN *lsn, translog_buffer_unlock(buffer_to_flush); } if (rc) + { + //translog_advance_pointer decreased writers so it is OK + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); DBUG_RETURN(1); + } translog_write_variable_record_1group_header(parts, type, short_trid, header_length, chunk0_header); @@ -5081,7 +5198,7 @@ translog_write_variable_record_1group(LSN *lsn, for (i= 0; i < full_pages; i++) { if (translog_write_variable_record_chunk2_page(parts, &horizon, &cursor)) - DBUG_RETURN(1); + goto error; DBUG_PRINT("info", ("absolute horizon: (%lu,0x%lx) local: (%lu,0x%lx)", LSN_IN_PARTS(log_descriptor.horizon), @@ -5094,7 +5211,7 @@ translog_write_variable_record_1group(LSN *lsn, log_descriptor. page_capacity_chunk_2 - 2, &horizon, &cursor)) - DBUG_RETURN(1); + goto error; DBUG_PRINT("info", ("absolute horizon: (%lu,0x%lx) local: (%lu,0x%lx)", LSN_IN_PARTS(log_descriptor.horizon), LSN_IN_PARTS(horizon))); @@ -5104,17 +5221,22 @@ translog_write_variable_record_1group(LSN *lsn, if (translog_write_variable_record_chunk3_page(parts, record_rest, &horizon, &cursor)) - DBUG_RETURN(1); - DBUG_PRINT("info", ("absolute horizon: (%lu,0x%lx) local: (%lu,0x%lx)", - (ulong) LSN_FILE_NO(log_descriptor.horizon), - (ulong) LSN_OFFSET(log_descriptor.horizon), - (ulong) LSN_FILE_NO(horizon), - (ulong) LSN_OFFSET(horizon))); + goto error; + DBUG_PRINT("info", ("absolute horizon: (%lu,0x%lx) local: (%lu,0x%lx)", + (ulong) LSN_FILE_NO(log_descriptor.horizon), + (ulong) LSN_OFFSET(log_descriptor.horizon), + (ulong) LSN_FILE_NO(horizon), + (ulong) LSN_OFFSET(horizon))); translog_buffer_lock(cursor.buffer); translog_buffer_decrease_writers(cursor.buffer); + used_buffs_register_unlock(&cursor.buffs, cursor.buffer); translog_buffer_unlock(cursor.buffer); - DBUG_RETURN(rc); + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); + DBUG_RETURN(0); +error: + used_buffs_urgent_unlock(&cursor.buffs); + DBUG_RETURN(1); } @@ -5168,7 +5290,8 @@ translog_write_variable_record_1chunk(LSN *lsn, lsn, hook_arg))) { translog_unlock(); - DBUG_RETURN(1); + rc= 1; + goto err; } rc= translog_write_parts_on_page(&log_descriptor.horizon, @@ -5184,6 +5307,7 @@ translog_write_variable_record_1chunk(LSN *lsn, check if we switched buffer and need process it (current buffer is unlocked already => we will not delay other threads */ +err: if (buffer_to_flush != NULL) { if (!rc) @@ -5523,9 +5647,11 @@ translog_write_variable_record_mgroup(LSN *lsn, uint file_of_the_first_group; int pages_to_skip; struct st_translog_buffer *buffer_of_last_lsn; + my_bool external_buffer_to_flush= TRUE; DBUG_ENTER("translog_write_variable_record_mgroup"); translog_lock_assert_owner(); + used_buffs_init(&cursor.buffs); chunk2_header[0]= TRANSLOG_CHUNK_NOHDR; if (my_init_dynamic_array(&groups, @@ -5533,6 +5659,11 @@ translog_write_variable_record_mgroup(LSN *lsn, 10, 10)) { translog_unlock(); + if (buffer_to_flush != NULL) + { + translog_buffer_flush(buffer_to_flush); + translog_buffer_unlock(buffer_to_flush); + } DBUG_PRINT("error", ("init array failed")); DBUG_RETURN(1); } @@ -5559,6 +5690,7 @@ translog_write_variable_record_mgroup(LSN *lsn, translog_mark_file_unfinished(file_of_the_first_group); do { + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); group.addr= horizon= log_descriptor.horizon; cursor= log_descriptor.bc; cursor.chaser= 1; @@ -5591,21 +5723,26 @@ translog_write_variable_record_mgroup(LSN *lsn, (ulong)(parts->record_length - (first_page - 1 + buffer_rest) - done))); - rc|= translog_advance_pointer((int)full_pages, 0); + rc= translog_advance_pointer((int)full_pages, 0, &cursor.buffs); translog_unlock(); if (buffer_to_flush != NULL) { - translog_buffer_decrease_writers(buffer_to_flush); + if (!external_buffer_to_flush) + translog_buffer_decrease_writers(buffer_to_flush); if (!rc) rc= translog_buffer_flush(buffer_to_flush); translog_buffer_unlock(buffer_to_flush); buffer_to_flush= NULL; } + external_buffer_to_flush= FALSE; + if (rc) { DBUG_PRINT("error", ("flush of unlock buffer failed")); + //translog_advance_pointer decreased writers so it is OK + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); goto err; } @@ -5642,6 +5779,7 @@ translog_write_variable_record_mgroup(LSN *lsn, } translog_buffer_lock(cursor.buffer); translog_buffer_decrease_writers(cursor.buffer); + used_buffs_register_unlock(&cursor.buffs, cursor.buffer); translog_buffer_unlock(cursor.buffer); translog_lock(); @@ -5656,6 +5794,11 @@ translog_write_variable_record_mgroup(LSN *lsn, first_page= translog_get_current_page_rest(); } buffer_rest= translog_get_current_group_size(); + + if (buffer_to_flush) + used_buffs_register_unlock(&cursor.buffs, + buffer_to_flush); // will be unlocked + } while ((translog_size_t)(first_page + buffer_rest) < (translog_size_t)(parts->record_length - done)); @@ -5751,17 +5894,21 @@ translog_write_variable_record_mgroup(LSN *lsn, (ulong) full_pages * log_descriptor.page_capacity_chunk_2, chunk3_pages, (uint) chunk3_size, (uint) record_rest)); + + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); rc= translog_advance_pointer(pages_to_skip + (int)(chunk0_pages - 1), record_rest + header_fixed_part + (groups.elements - ((page_capacity - header_fixed_part) / (7 + 1)) * - (chunk0_pages - 1)) * (7 + 1)); + (chunk0_pages - 1)) * (7 + 1), + &cursor.buffs); buffer_of_last_lsn= log_descriptor.bc.buffer; translog_unlock(); if (buffer_to_flush != NULL) { + DBUG_ASSERT(!external_buffer_to_flush); translog_buffer_decrease_writers(buffer_to_flush); if (!rc) rc= translog_buffer_flush(buffer_to_flush); @@ -5925,8 +6072,10 @@ translog_write_variable_record_mgroup(LSN *lsn, } while (chunk0_pages != 0); translog_buffer_lock(cursor.buffer); translog_buffer_decrease_writers(cursor.buffer); + used_buffs_register_unlock(&cursor.buffs, cursor.buffer); translog_buffer_unlock(cursor.buffer); rc= 0; + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); if (translog_set_lsn_for_files(file_of_the_first_group, LSN_FILE_NO(*lsn), *lsn, FALSE)) @@ -5935,17 +6084,22 @@ translog_write_variable_record_mgroup(LSN *lsn, translog_mark_file_finished(file_of_the_first_group); delete_dynamic(&groups); - DBUG_RETURN(rc); + DBUG_RETURN(0); err_unlock: translog_unlock(); err: + + if (cursor.buffs.unlck_ptr != cursor.buffs.wrt_ptr) + used_buffs_urgent_unlock(&cursor.buffs); + if (buffer_to_flush != NULL) { /* This is to prevent locking buffer forever in case of error */ - translog_buffer_decrease_writers(buffer_to_flush); + if (!external_buffer_to_flush) + translog_buffer_decrease_writers(buffer_to_flush); if (!rc) rc= translog_buffer_flush(buffer_to_flush); translog_buffer_unlock(buffer_to_flush); @@ -7500,7 +7654,8 @@ static void translog_force_current_buffer_to_finish() DBUG_ASSERT(log_descriptor.bc.ptr !=NULL); DBUG_ASSERT(LSN_FILE_NO(log_descriptor.horizon) == - LSN_FILE_NO(old_buffer->offset)); + LSN_FILE_NO(old_buffer->offset) || + translog_status == TRANSLOG_READONLY ); translog_check_cursor(&log_descriptor.bc); DBUG_ASSERT(left < TRANSLOG_PAGE_SIZE); if (left) From 26be5072429082e634b8fc102609370975443439 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 12 Jun 2018 10:41:25 +0400 Subject: [PATCH 117/203] MDEV-12060 Crash in EXECUTE IMMEDIATE with an expression returning a GRANT command Backporting (partially) the fix for MDEV-14603. --- mysql-test/r/ps.result | 42 ++++++++++++++++++++++++++++++++ mysql-test/t/ps.test | 55 ++++++++++++++++++++++++++++++++++++++++++ sql/sql_prepare.cc | 38 +++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+) diff --git a/mysql-test/r/ps.result b/mysql-test/r/ps.result index e63b820d4dd..1b40a734263 100644 --- a/mysql-test/r/ps.result +++ b/mysql-test/r/ps.result @@ -4351,3 +4351,45 @@ LINE2 2 LINE3 3 drop table t1; # End of 5.5 tests +# +# Start of 10.1 tests +# +# +# MDEV-12060 Crash in EXECUTE IMMEDIATE with an expression returning a GRANT command +# (the 10.1 part) +# +CREATE PROCEDURE p2 () +BEGIN +SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +END; +/ +CALL p2(); +1 +1 +DROP PROCEDURE p2; +BEGIN NOT ATOMIC +SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +END; +/ +1 +1 +BEGIN NOT ATOMIC +SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; +DEALLOCATE PREPARE stmt; +END; +/ +BEGIN NOT ATOMIC +PREPARE stmt FROM 'SELECT 1'; +SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR EXECUTE stmt; +DEALLOCATE PREPARE stmt; +END; +/ +1 +1 +# +# End of 10.1 tests +# diff --git a/mysql-test/t/ps.test b/mysql-test/t/ps.test index f7008f570b8..e1e8dfeeb17 100644 --- a/mysql-test/t/ps.test +++ b/mysql-test/t/ps.test @@ -3876,3 +3876,58 @@ FROM ) X; drop table t1; --echo # End of 5.5 tests + +--echo # +--echo # Start of 10.1 tests +--echo # + +--echo # +--echo # MDEV-12060 Crash in EXECUTE IMMEDIATE with an expression returning a GRANT command +--echo # (the 10.1 part) +--echo # + +DELIMITER /; +CREATE PROCEDURE p2 () +BEGIN + SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; +END; +/ +DELIMITER ;/ +CALL p2(); +DROP PROCEDURE p2; + + +DELIMITER /; +BEGIN NOT ATOMIC + SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; +END; +/ +DELIMITER ;/ + + +DELIMITER /; +BEGIN NOT ATOMIC + SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; + DEALLOCATE PREPARE stmt; +END; +/ +DELIMITER ;/ + + +DELIMITER /; +BEGIN NOT ATOMIC + PREPARE stmt FROM 'SELECT 1'; + SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR EXECUTE stmt; + DEALLOCATE PREPARE stmt; +END; +/ +DELIMITER ;/ + + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index d39ed6aa637..92d2b1c68b0 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -2736,6 +2736,15 @@ void mysql_sql_stmt_prepare(THD *thd) DBUG_VOID_RETURN; } +#if MYSQL_VERSION_ID < 100200 + /* + Backpoiting MDEV-14603 from 10.2 to 10.1 + Remove the code between #if..#endif when merging. + */ + Item_change_list change_list_save_point; + thd->change_list.move_elements_to(&change_list_save_point); +#endif + if (stmt->prepare(query, query_len)) { /* Statement map deletes the statement on erase */ @@ -2744,6 +2753,15 @@ void mysql_sql_stmt_prepare(THD *thd) else my_ok(thd, 0L, 0L, "Statement prepared"); +#if MYSQL_VERSION_ID < 100200 + /* + Backpoiting MDEV-14603 from 10.2 to 10.1 + Remove the code between #if..#endif when merging. + */ + thd->rollback_item_tree_changes(); + change_list_save_point.move_elements_to(&thd->change_list); +#endif + DBUG_VOID_RETURN; } @@ -3039,7 +3057,27 @@ void mysql_sql_stmt_execute(THD *thd) */ Item *free_list_backup= thd->free_list; thd->free_list= NULL; // Hide the external (e.g. "SET STATEMENT") Items + +#if MYSQL_VERSION_ID < 100200 + /* + Backpoiting MDEV-14603 from 10.2 to 10.1 + Remove the code between #if..#endif when merging. + */ + Item_change_list change_list_save_point; + thd->change_list.move_elements_to(&change_list_save_point); +#endif + (void) stmt->execute_loop(&expanded_query, FALSE, NULL, NULL); + +#if MYSQL_VERSION_ID < 100200 + /* + Backpoiting MDEV-14603 from 10.2 to 10.1 + Remove the code between #if..#endif when merging. + */ + thd->rollback_item_tree_changes(); + change_list_save_point.move_elements_to(&thd->change_list); +#endif + thd->free_items(); // Free items created by execute_loop() /* Now restore the "external" (e.g. "SET STATEMENT") Item list. From 0ad9c3a0160d1dd46139b3e6b6b05d0fba01540b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 11 Jun 2018 13:02:47 +0300 Subject: [PATCH 118/203] MDEV-16456 InnoDB error "returned OS error 71" complains about wrong path When attempting to rename a table to a non-existing database, InnoDB would misleadingly report "OS error 71" when in fact the error code is InnoDB's own (OS_FILE_NOT_FOUND), and not report both pathnames. Errors on rename could occur due to reasons connected to either pathname. os_file_handle_rename_error(): New function, to report errors in renaming files. --- mysql-test/suite/innodb/r/rename_table.result | 5 ++++ .../suite/innodb/t/innodb-mdev7046.test | 7 +++--- mysql-test/suite/innodb/t/rename_table.test | 11 +++++++++ storage/innobase/os/os0file.cc | 24 +++++++++++++++---- storage/xtradb/os/os0file.cc | 22 +++++++++++++++-- 5 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 mysql-test/suite/innodb/r/rename_table.result create mode 100644 mysql-test/suite/innodb/t/rename_table.test diff --git a/mysql-test/suite/innodb/r/rename_table.result b/mysql-test/suite/innodb/r/rename_table.result new file mode 100644 index 00000000000..b2e15c87348 --- /dev/null +++ b/mysql-test/suite/innodb/r/rename_table.result @@ -0,0 +1,5 @@ +call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +RENAME TABLE t1 TO non_existing_db.t1; +ERROR HY000: Error on rename of './test/t1' to './non_existing_db/t1' (errno: -1 "Internal error < 0 (Not system error)") +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-mdev7046.test b/mysql-test/suite/innodb/t/innodb-mdev7046.test index b4085228e02..85a257a1739 100644 --- a/mysql-test/suite/innodb/t/innodb-mdev7046.test +++ b/mysql-test/suite/innodb/t/innodb-mdev7046.test @@ -9,9 +9,10 @@ # Ignore OS errors -call mtr.add_suppression("InnoDB: File ./test/t1*"); -call mtr.add_suppression("InnoDB: Error number*"); -call mtr.add_suppression("InnoDB: File ./test/t1#p#p1#sp#p1sp0.ibd: 'rename' returned OS error*"); +call mtr.add_suppression("InnoDB: File ./test/t1"); +call mtr.add_suppression("InnoDB: Error number"); +call mtr.add_suppression("InnoDB: Cannot rename file '.*/test/t1#[Pp]#p1#[Ss][Pp]#p1sp0\\.ibd' to"); +call mtr.add_suppression("InnoDB: Operating system error number .* in a file operation."); # MDEV-7046: MySQL#74480 - Failing assertion: os_file_status(newpath, &exists, &type) # after Operating system error number 36 in a file operation diff --git a/mysql-test/suite/innodb/t/rename_table.test b/mysql-test/suite/innodb/t/rename_table.test new file mode 100644 index 00000000000..60df8b5d2ec --- /dev/null +++ b/mysql-test/suite/innodb/t/rename_table.test @@ -0,0 +1,11 @@ +--source include/have_innodb.inc + +call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); + +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +--replace_result "\\" "/" +--error ER_ERROR_ON_RENAME +RENAME TABLE t1 TO non_existing_db.t1; + +# Cleanup +DROP TABLE t1; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 8f3f3716fc2..038d59c2170 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -1939,6 +1939,24 @@ loop: #endif } +/** Handle RENAME error. +@param name old name of the file +@param new_name new name of the file */ +static void os_file_handle_rename_error(const char* name, const char* new_name) +{ + if (os_file_get_last_error(true) != OS_FILE_DISK_FULL) { + ib_logf(IB_LOG_LEVEL_ERROR, "Cannot rename file '%s' to '%s'", + name, new_name); + } else if (!os_has_said_disk_full) { + os_has_said_disk_full = true; + /* Disk full error is reported irrespective of the + on_error_silent setting. */ + ib_logf(IB_LOG_LEVEL_ERROR, + "Full disk prevents renaming file '%s' to '%s'", + name, new_name); + } +} + /***********************************************************************//** NOTE! Use the corresponding macro os_file_rename(), not directly this function! Renames a file (can also move it to another directory). It is safest that the @@ -1974,8 +1992,7 @@ os_file_rename_func( return(TRUE); } - os_file_handle_error_no_exit(oldpath, "rename", FALSE); - + os_file_handle_rename_error(oldpath, newpath); return(FALSE); #else int ret; @@ -1983,8 +2000,7 @@ os_file_rename_func( ret = rename(oldpath, newpath); if (ret != 0) { - os_file_handle_error_no_exit(oldpath, "rename", FALSE); - + os_file_handle_rename_error(oldpath, newpath); return(FALSE); } diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index c1ddc83d852..9df42470e5a 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2146,6 +2146,24 @@ loop: #endif } +/** Handle RENAME error. +@param name old name of the file +@param new_name new name of the file */ +static void os_file_handle_rename_error(const char* name, const char* new_name) +{ + if (os_file_get_last_error(true) != OS_FILE_DISK_FULL) { + ib_logf(IB_LOG_LEVEL_ERROR, "Cannot rename file '%s' to '%s'", + name, new_name); + } else if (!os_has_said_disk_full) { + os_has_said_disk_full = true; + /* Disk full error is reported irrespective of the + on_error_silent setting. */ + ib_logf(IB_LOG_LEVEL_ERROR, + "Full disk prevents renaming file '%s' to '%s'", + name, new_name); + } +} + /***********************************************************************//** NOTE! Use the corresponding macro os_file_rename(), not directly this function! Renames a file (can also move it to another directory). It is safest that the @@ -2181,7 +2199,7 @@ os_file_rename_func( return(TRUE); } - os_file_handle_error_no_exit(oldpath, "rename", FALSE); + os_file_handle_rename_error(oldpath, newpath); return(FALSE); #else @@ -2190,7 +2208,7 @@ os_file_rename_func( ret = rename(oldpath, newpath); if (ret != 0) { - os_file_handle_error_no_exit(oldpath, "rename", FALSE); + os_file_handle_rename_error(oldpath, newpath); return(FALSE); } From 6b8d34fe0d5e73a469383bcb0818d3ff91ed9a84 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Tue, 12 Jun 2018 12:36:51 +0400 Subject: [PATCH 119/203] MDEV-14668 ADD PRIMARY KEY IF NOT EXISTS on composite key. Check the name of the primary key to be 'PRIMARY'. Than differs it from any implicit primary keys created by an engine. --- mysql-test/r/alter_table.result | 23 +++++++++++++++++++++++ mysql-test/t/alter_table.test | 16 ++++++++++++++++ sql/sql_table.cc | 4 +++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index bb78df907ad..c6e3c7e31d9 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -2225,3 +2225,26 @@ t1 CREATE TABLE `t1` ( `b` int(11) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 DROP TABLE t1; +# +# MDEV-14668 ADD PRIMARY KEY IF NOT EXISTS on composite key +# +CREATE TABLE t1 ( +`ID` BIGINT(20) NOT NULL, +`RANK` MEDIUMINT(4) NOT NULL, +`CHECK_POINT` BIGINT(20) NOT NULL, +UNIQUE INDEX `HORIZON_UIDX01` (`ID`, `RANK`) +) ENGINE=InnoDB; +ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS (`ID`, `CHECK_POINT`); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `ID` bigint(20) NOT NULL, + `RANK` mediumint(4) NOT NULL, + `CHECK_POINT` bigint(20) NOT NULL, + PRIMARY KEY (`ID`,`CHECK_POINT`), + UNIQUE KEY `HORIZON_UIDX01` (`ID`,`RANK`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS (`ID`, `CHECK_POINT`); +Warnings: +Note 1061 Multiple primary key defined +DROP TABLE t1; diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index 585a272de9b..28d8c5bf5e9 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1841,3 +1841,19 @@ CREATE TABLE t1 (a INT, b INT) ENGINE=InnoDB; ALTER TABLE t1 DROP INDEX IF EXISTS fk, DROP COLUMN IF EXISTS c; SHOW CREATE TABLE t1; DROP TABLE t1; + +--echo # +--echo # MDEV-14668 ADD PRIMARY KEY IF NOT EXISTS on composite key +--echo # +CREATE TABLE t1 ( + `ID` BIGINT(20) NOT NULL, + `RANK` MEDIUMINT(4) NOT NULL, + `CHECK_POINT` BIGINT(20) NOT NULL, + UNIQUE INDEX `HORIZON_UIDX01` (`ID`, `RANK`) + ) ENGINE=InnoDB; + +ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS (`ID`, `CHECK_POINT`); +SHOW CREATE TABLE t1; +ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS (`ID`, `CHECK_POINT`); +DROP TABLE t1; + diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 376c1362cc7..6c71067d51b 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -5930,7 +5930,9 @@ drop_create_field: /* Check if the table already has a PRIMARY KEY */ if (key->type == Key::PRIMARY && - table->s->primary_key != MAX_KEY) + table->s->primary_key != MAX_KEY && + (keyname= table->s->key_info[table->s->primary_key].name) && + my_strcasecmp(system_charset_info, keyname, primary_key_name) == 0) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, ER_DUP_KEYNAME, ER(ER_MULTIPLE_PRI_KEY)); From 7bbe324fc17d9734833f717921629c36d1d0c996 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Fri, 8 Jun 2018 22:01:05 +0300 Subject: [PATCH 120/203] MDEV-13577 slave_parallel_mode=optimistic should not report the mode's specific temporary errors The optimistic parallel slave's worker thread could face a run-time error due to the algorithm's specifics which allows for conflicts like the reported "Can't find record in 'table'". A typical stack is like {noformat} #0 handler::print_error (this=0x61c00008f8a0, error=149, errflag=0) at handler.cc:3650 #1 0x0000555555e95361 in write_record (thd=thd@entry=0x62a0000a2208, table=table@entry=0x61f00008ce88, info=info@entry=0x7fffdee356d0) at sql_insert.cc:1944 #2 0x0000555555ea7767 in mysql_insert (thd=thd@entry=0x62a0000a2208, table_list=0x61b00012ada0, fields=..., values_list=..., update_fields=..., update_values=..., duplic=, ignore=) at sql_insert.cc:1039 #3 0x0000555555efda90 in mysql_execute_command (thd=thd@entry=0x62a0000a2208) at sql_parse.cc:3927 #4 0x0000555555f0cc50 in mysql_parse (thd=0x62a0000a2208, rawbuf=, length=, parser_state=) at sql_parse.cc:7449 #5 0x00005555566d4444 in Query_log_event::do_apply_event (this=0x61200005b9c8, rgi=, query_arg=, q_len_arg=) at log_event.cc:4508 #6 0x00005555566d639e in Query_log_event::do_apply_event (this=, rgi=) at log_event.cc:4185 #7 0x0000555555d738cf in Log_event::apply_event (rgi=0x61d0001ea080, this=0x61200005b9c8) at log_event.h:1343 #8 apply_event_and_update_pos_apply (ev=ev@entry=0x61200005b9c8, thd=thd@entry=0x62a0000a2208, rgi=rgi@entry=0x61d0001ea080, reason=) at slave.cc:3479 #9 0x0000555555d8596b in apply_event_and_update_pos_for_parallel (ev=ev@entry=0x61200005b9c8, thd=thd@entry=0x62a0000a2208, rgi=rgi@entry=0x61d0001ea080) at slave.cc:3623 #10 0x00005555562aca83 in rpt_handle_event (qev=qev@entry=0x6190000fa088, rpt=rpt@entry=0x62200002bd68) at rpl_parallel.cc:50 #11 0x00005555562bd04e in handle_rpl_parallel_thread (arg=arg@entry=0x62200002bd68) at rpl_parallel.cc:1258 {noformat} Here {{handler::print_error}} computes whether to error log the current error when --log-warnings > 1. The decision flag is consulted bu {{my_message_sql()}} which can be eventually called. In the bug case the decision is to log. However in the optimistic mode slave applier case any conflict is attempted to resolve with rollback and retry to success. Hence the logging is at least extraneous. The case is fixed with adding a new flag {{ME_LOG_AS_WARN}} which {{handler::print_error}} may propagate further on through {{my_error}} when the error comes from an optimistically running slave worker thread. The new flag effectively requests the warning level for the errlog record, while the thread's DA records the actual error (which is regarded as temporary one by the parallel slave error handler). --- include/my_sys.h | 1 + .../rpl/r/rpl_parallel_optimistic.result | 23 +++++++ .../suite/rpl/t/rpl_parallel_optimistic.test | 64 ++++++++++++++++++- sql/handler.cc | 5 +- sql/mysqld.cc | 10 +++ sql/sql_class.cc | 7 ++ sql/sql_class.h | 6 ++ 7 files changed, 114 insertions(+), 2 deletions(-) diff --git a/include/my_sys.h b/include/my_sys.h index 110a2ee9af3..1c5649812d1 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -112,6 +112,7 @@ typedef struct my_aio_result { #define ME_JUST_INFO 1024 /**< not error but just info */ #define ME_JUST_WARNING 2048 /**< not error but just warning */ #define ME_FATALERROR 4096 /* Fatal statement error */ +#define ME_LOG_AS_WARN 8192 /* is error but error-logged as warning */ /* Bits in last argument to fn_format */ #define MY_REPLACE_DIR 1 /* replace dir in name with 'dir' */ diff --git a/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result b/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result index 0177e65b10f..a6da3399fab 100644 --- a/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result +++ b/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result @@ -1,4 +1,6 @@ include/rpl_init.inc [topology=1->2] +call mtr.add_suppression("Warning.*Deadlock found when trying to get lock; try restarting transaction"); +call mtr.add_suppression("Warning.*mysqld: Can't find record in 't2'"); ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; @@ -543,6 +545,27 @@ a b 57 7 58 8 59 9 +DELETE FROM t1; +DELETE FROM t2; +include/save_master_gtid.inc +include/sync_with_master_gtid.inc +BEGIN; +INSERT INTO t1 SET a=1; +SET @save.binlog_format=@@session.binlog_format; +SET @@SESSION.binlog_format=row; +BEGIN; +INSERT INTO t1 SET a=1; +INSERT INTO t2 SET a=1; +COMMIT; +BEGIN; +DELETE FROM t2; +COMMIT; +ROLLBACK; +SET @@SESSION.binlog_format= @save.binlog_format; +DELETE FROM t1; +DELETE FROM t2; +include/save_master_gtid.inc +include/sync_with_master_gtid.inc include/stop_slave.inc SET GLOBAL slave_parallel_mode=@old_parallel_mode; SET GLOBAL slave_parallel_threads=@old_parallel_threads; diff --git a/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test b/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test index 41fb6ebb72e..28bf4a77fd4 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test @@ -4,6 +4,11 @@ --let $rpl_topology=1->2 --source include/rpl_init.inc +--connection server_2 +call mtr.add_suppression("Warning.*Deadlock found when trying to get lock; try restarting transaction"); +# The following instruction is a part of the proof of MDEV-13577 fixes, below. +call mtr.add_suppression("Warning.*mysqld: Can't find record in 't2'"); + --connection server_1 ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; @@ -480,8 +485,65 @@ SELECT * FROM t2 WHERE a >= 40 ORDER BY a; SELECT * FROM t1 WHERE a >= 40 ORDER BY a; SELECT * FROM t2 WHERE a >= 40 ORDER BY a; -# Clean up. +# partial cleanup to reuse the tables by following tests +--connection server_1 +DELETE FROM t1; +DELETE FROM t2; +--source include/save_master_gtid.inc +--connection server_2 +--source include/sync_with_master_gtid.inc + +# +# MDEV-13577 optimistic parallel slave errors out to error log unnecessary +# + +# The 1st of the following two trx:s a blocker on slave +--connection server_2 +BEGIN; +INSERT INTO t1 SET a=1; + +--connection server_1 +SET @save.binlog_format=@@session.binlog_format; +SET @@SESSION.binlog_format=row; + +BEGIN; + INSERT INTO t1 SET a=1; + INSERT INTO t2 SET a=1; +COMMIT; + +# This transaction is going to win optimistical race with above INSERT +# on slave while being depend on it. That means it will face a kind of temporary error +# and then will retry to succeed. +BEGIN; + DELETE FROM t2; +COMMIT; + +# First make sure DELETE raced indeed to get stuck at retrying stage +# where it runs "realistically" now. There is nomore optimistic error +# in the errorlog, which is downgraded to the warning level (when +# --log-warnings > 1), see above suppression. +--connection server_2 +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +# Next release the 1st trx to commit. +--connection server_2 +ROLLBACK; + +# MDEV-13577 local cleanup: +--connection server_1 +SET @@SESSION.binlog_format= @save.binlog_format; +DELETE FROM t1; +DELETE FROM t2; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc + +# +# Clean up. +# --connection server_2 --source include/stop_slave.inc SET GLOBAL slave_parallel_mode=@old_parallel_mode; diff --git a/sql/handler.cc b/sql/handler.cc index 397891ceec5..87f0926ad9a 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3639,12 +3639,15 @@ void handler::print_error(int error, myf errflag) if ((debug_assert_if_crashed_table || global_system_variables.log_warnings > 1)) { + THD *thd= ha_thd(); /* Log error to log before we crash or if extended warnings are requested */ errflag|= ME_NOREFRESH; + if (thd && thd->is_optimistic_slave_worker()) + errflag|= ME_LOG_AS_WARN; } - } + } /* if we got an OS error from a file-based engine, specify a path of error */ if (error < HA_ERR_FIRST && bas_ext()[0]) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5f954f7576d..5bf5e3f73fc 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3517,6 +3517,16 @@ void my_message_sql(uint error, const char *str, myf MyFlags) level= Sql_condition::WARN_LEVEL_WARN; func= sql_print_warning; } + else if (MyFlags & ME_LOG_AS_WARN) + { + /* + Typical use case is optimistic parallel slave where DA needs to hold + an error condition caused by the current error, but the error-log + level is relaxed to the warning one. + */ + level= Sql_condition::WARN_LEVEL_ERROR; + func= sql_print_warning; + } else { level= Sql_condition::WARN_LEVEL_ERROR; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 24140246b96..37f29115171 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -7100,6 +7100,13 @@ bool THD::rgi_have_temporary_tables() return rgi_slave->rli->save_temporary_tables != 0; } +bool THD::is_optimistic_slave_worker() +{ + DBUG_ASSERT(system_thread != SYSTEM_THREAD_SLAVE_SQL || rgi_slave); + + return system_thread == SYSTEM_THREAD_SLAVE_SQL && rgi_slave && + rgi_slave->speculation == rpl_group_info::SPECULATE_OPTIMISTIC; +} void wait_for_commit::reinit() diff --git a/sql/sql_class.h b/sql/sql_class.h index ca6155ec93f..0ced84791a0 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -4167,6 +4167,12 @@ public: (THD_TRANS::DID_WAIT | THD_TRANS::CREATED_TEMP_TABLE | THD_TRANS::DROPPED_TEMP_TABLE | THD_TRANS::DID_DDL)); } + + /* + Returns true when the thread handle belongs to a slave worker thread + running in the optimistic execution mode. + */ + bool is_optimistic_slave_worker(); }; From d2e1ed8b936a78ceeec0e64231997d7ed18a4daf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 13 Jun 2018 08:33:25 +0300 Subject: [PATCH 121/203] Fix innodb.rename_table for embedded An embedded run will output the full path name instead of a relative one. Update results to cover both cases. --- mysql-test/suite/innodb/r/rename_table.result | 2 +- mysql-test/suite/innodb/t/rename_table.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/rename_table.result b/mysql-test/suite/innodb/r/rename_table.result index b2e15c87348..49ce3254091 100644 --- a/mysql-test/suite/innodb/r/rename_table.result +++ b/mysql-test/suite/innodb/r/rename_table.result @@ -1,5 +1,5 @@ call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); CREATE TABLE t1 (a INT) ENGINE=InnoDB; RENAME TABLE t1 TO non_existing_db.t1; -ERROR HY000: Error on rename of './test/t1' to './non_existing_db/t1' (errno: -1 "Internal error < 0 (Not system error)") +ERROR HY000: Error on rename of '**path-to-t1**' to '**path-to-non-existing-db-t1**' (errno: -1 "Internal error < 0 (Not system error)") DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/rename_table.test b/mysql-test/suite/innodb/t/rename_table.test index 60df8b5d2ec..695c0915d14 100644 --- a/mysql-test/suite/innodb/t/rename_table.test +++ b/mysql-test/suite/innodb/t/rename_table.test @@ -3,7 +3,7 @@ call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); CREATE TABLE t1 (a INT) ENGINE=InnoDB; ---replace_result "\\" "/" +--replace_regex /\'.*t1\' to/'**path-to-t1**' to/ /to \'.*non.*t1\'/to '**path-to-non-existing-db-t1**'/ --error ER_ERROR_ON_RENAME RENAME TABLE t1 TO non_existing_db.t1; From edc1b8e1172abc35644f2b4824a78d49ac17a7d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 13 Jun 2018 08:57:15 +0300 Subject: [PATCH 122/203] Fix wsrep.variables test case Correctly port 09b25f85966f44aae933e86b84b4ebe59ded47c3 from 10.0. Expose check_is_super via include file to allow wsrep_on_check to use it. --- sql/set_var.h | 1 + sql/sys_vars.cc | 2 +- sql/wsrep_var.cc | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/set_var.h b/sql/set_var.h index fe66eaff775..203969d6169 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -426,6 +426,7 @@ CHARSET_INFO *get_old_charset_by_name(const char *old_name); int sys_var_init(); int sys_var_add_options(DYNAMIC_ARRAY *long_options, int parse_flags); void sys_var_end(void); +bool check_has_super(sys_var *self, THD *thd, set_var *var); #endif diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 9f87d66fef1..5470acc7892 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -428,7 +428,7 @@ error_if_in_trans_or_substatement(THD *thd, int in_substatement_error, return false; } -static bool check_has_super(sys_var *self, THD *thd, set_var *var) +bool check_has_super(sys_var *self, THD *thd, set_var *var) { DBUG_ASSERT(self->scope() != sys_var::GLOBAL);// don't abuse check_has_super() #ifndef NO_EMBEDDED_ACCESS_CHECKS diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 2d52396331d..216bab0cdcd 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -69,6 +69,9 @@ bool wsrep_on_check(sys_var *self, THD* thd, set_var* var) { bool new_wsrep_on= (bool)var->save_result.ulonglong_value; + if (check_has_super(self, thd, var)) + return true; + if (new_wsrep_on && innodb_lock_schedule_algorithm != 0) { my_message(ER_WRONG_ARGUMENTS, " WSREP (galera) can't be enabled " "if innodb_lock_schedule_algorithm=VATS. Please configure" From 92bd177fe9f8b297a464891a25bbbd6a992d655f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 11 Jun 2018 10:33:09 +0300 Subject: [PATCH 123/203] Correct a typo in a comment --- storage/innobase/include/fsp0fsp.h | 2 +- storage/xtradb/include/fsp0fsp.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 275a5785ce2..c11c48d58cf 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -922,7 +922,7 @@ fsp_flags_convert_from_101(ulint flags) /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they should contain one of the values 3,4,6,7, that is, be of the form - 0b0011 or 0b01xx (except 0b0110). + 0b0011 or 0b01xx (except 0b0101). In correct versions, these bits should be 0bc0se where c is the MariaDB COMPRESSED flag and e is the MySQL 5.7 ENCRYPTION flag diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 7395a248bed..bc81a9c126b 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -921,7 +921,7 @@ fsp_flags_convert_from_101(ulint flags) /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they should contain one of the values 3,4,6,7, that is, be of the form - 0b0011 or 0b01xx (except 0b0110). + 0b0011 or 0b01xx (except 0b0101). In correct versions, these bits should be 0bc0se where c is the MariaDB COMPRESSED flag and e is the MySQL 5.7 ENCRYPTION flag From 3fcc11fbb41aca43298e39c96525491699e686e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 11 Jun 2018 10:46:15 +0300 Subject: [PATCH 124/203] Remove traces of the non-working MDEV-6354 MariaDB never supported the MySQL 5.7 compression format. FIL_PAGE_TYPE_COMPRESSED: Remove. This was originally added as FIL_PAGE_COMPRESSED. --- storage/innobase/fil/fil0pagecompress.cc | 5 ++--- storage/innobase/include/fil0fil.h | 3 +-- storage/innobase/include/fil0fil.ic | 7 ++----- storage/xtradb/fil/fil0pagecompress.cc | 5 ++--- storage/xtradb/include/fil0fil.h | 3 +-- storage/xtradb/include/fil0fil.ic | 7 ++----- 6 files changed, 10 insertions(+), 20 deletions(-) diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index edc932f36f5..0ac764d5a32 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -501,8 +501,7 @@ fil_decompress_page( /* Do not try to uncompressed pages that are not compressed */ if (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && - ptype != FIL_PAGE_TYPE_COMPRESSED) { + ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { return; } diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9b6178bb8f1..ae2629b5b04 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -173,8 +173,7 @@ extern fil_addr_t fil_addr_null; #define FIL_PAGE_TYPE_BLOB 10 /*!< Uncompressed BLOB page */ #define FIL_PAGE_TYPE_ZBLOB 11 /*!< First compressed BLOB page */ #define FIL_PAGE_TYPE_ZBLOB2 12 /*!< Subsequent compressed BLOB page */ -#define FIL_PAGE_TYPE_COMPRESSED 13 /*!< Compressed page */ -#define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_COMPRESSED +#define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_ZBLOB2 /*!< Last page type */ /* @} */ diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index 6c2504c9f8c..49fdff4f3e5 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -65,8 +65,6 @@ fil_get_page_type_name( return "ZBLOB"; case FIL_PAGE_TYPE_ZBLOB2: return "ZBLOB2"; - case FIL_PAGE_TYPE_COMPRESSED: - return "ORACLE PAGE COMPRESSED"; } return "PAGE TYPE CORRUPTED"; @@ -112,8 +110,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_XDES || page_type == FIL_PAGE_TYPE_BLOB || page_type == FIL_PAGE_TYPE_ZBLOB || - page_type == FIL_PAGE_TYPE_ZBLOB2 || - page_type == FIL_PAGE_TYPE_COMPRESSED))) { + page_type == FIL_PAGE_TYPE_ZBLOB2))) { ulint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index edc932f36f5..0ac764d5a32 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -501,8 +501,7 @@ fil_decompress_page( /* Do not try to uncompressed pages that are not compressed */ if (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && - ptype != FIL_PAGE_TYPE_COMPRESSED) { + ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { return; } diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index a0db48f7037..91d46afe7a8 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -170,8 +170,7 @@ extern fil_addr_t fil_addr_null; #define FIL_PAGE_TYPE_BLOB 10 /*!< Uncompressed BLOB page */ #define FIL_PAGE_TYPE_ZBLOB 11 /*!< First compressed BLOB page */ #define FIL_PAGE_TYPE_ZBLOB2 12 /*!< Subsequent compressed BLOB page */ -#define FIL_PAGE_TYPE_COMPRESSED 13 /*!< Compressed page */ -#define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_COMPRESSED +#define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_ZBLOB2 /*!< Last page type */ /* @} */ diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic index 6c2504c9f8c..49fdff4f3e5 100644 --- a/storage/xtradb/include/fil0fil.ic +++ b/storage/xtradb/include/fil0fil.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -65,8 +65,6 @@ fil_get_page_type_name( return "ZBLOB"; case FIL_PAGE_TYPE_ZBLOB2: return "ZBLOB2"; - case FIL_PAGE_TYPE_COMPRESSED: - return "ORACLE PAGE COMPRESSED"; } return "PAGE TYPE CORRUPTED"; @@ -112,8 +110,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_XDES || page_type == FIL_PAGE_TYPE_BLOB || page_type == FIL_PAGE_TYPE_ZBLOB || - page_type == FIL_PAGE_TYPE_ZBLOB2 || - page_type == FIL_PAGE_TYPE_COMPRESSED))) { + page_type == FIL_PAGE_TYPE_ZBLOB2))) { ulint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); From 72005b7a1c42cb31294ead822b812909e5885f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Jun 2018 08:26:46 +0300 Subject: [PATCH 125/203] MDEV-13103: Improve 'cannot be decrypted' error message buf_page_check_corrupt(): Display the file name. --- .../encryption/r/innodb-bad-key-change.result | 8 ++++---- .../encryption/r/innodb-bad-key-change2.result | 5 ++--- .../encryption/r/innodb-bad-key-change4.result | 2 +- .../encryption/r/innodb-compressed-blob.result | 3 +-- .../r/innodb-encryption-disable.result | 3 +-- .../encryption/r/innodb-force-corrupt.result | 8 ++------ .../encryption/r/innodb-missing-key.result | 4 +--- .../encryption/r/innodb-redo-badkey.result | 17 +++++++---------- .../encryption/r/innodb-redo-nokeys.result | 13 +++++-------- .../encryption/t/innodb-bad-key-change.test | 9 +++++---- .../encryption/t/innodb-bad-key-change2.test | 5 ++--- .../encryption/t/innodb-bad-key-change4.test | 2 +- .../encryption/t/innodb-compressed-blob.test | 3 +-- .../encryption/t/innodb-encryption-disable.test | 3 +-- .../encryption/t/innodb-force-corrupt.test | 8 ++------ .../suite/encryption/t/innodb-missing-key.test | 4 +--- .../suite/encryption/t/innodb-redo-badkey.test | 17 +++++++---------- .../suite/encryption/t/innodb-redo-nokeys.test | 13 +++++-------- storage/innobase/buf/buf0buf.cc | 4 ++-- storage/xtradb/buf/buf0buf.cc | 4 ++-- 20 files changed, 53 insertions(+), 82 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index d581b98513d..51fc7a8cbc4 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -1,7 +1,7 @@ -call mtr.add_suppression("InnoDB: The page .*"); -call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* "); -call mtr.add_suppression("Plugin 'file_key_management' .*"); -call mtr.add_suppression("mysqld: File .*"); +call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); +call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t[12]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("mysqld: File .*keysbad3.txt' not found "); # Start server with keys2.txt SET GLOBAL innodb_file_format = `Barracuda`; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result index 94ed922a0ec..1ed78a9efd2 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result @@ -1,6 +1,5 @@ -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1new cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("Couldn't load plugins from 'file_key_management.*"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t1(new)?\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Table \'\"test\".\"t1\"\' tablespace is set as discarded."); SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result index 227b8cc3deb..a491b084764 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t1\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Cannot open table .*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management.*"); diff --git a/mysql-test/suite/encryption/r/innodb-compressed-blob.result b/mysql-test/suite/encryption/r/innodb-compressed-blob.result index 15c2a410948..ce73b80820f 100644 --- a/mysql-test/suite/encryption/r/innodb-compressed-blob.result +++ b/mysql-test/suite/encryption/r/innodb-compressed-blob.result @@ -1,5 +1,4 @@ -call mtr.add_suppression("InnoDB: However key management plugin or used key_version .*"); -call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file test/t[1-3] cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '..test.t[1-3]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("Unable to decompress space ..test.t[1-3].ibd \\[[1-9][0-9]*:[0-9]+\\]"); # Restart mysqld --file-key-management-filename=keys2.txt SET GLOBAL innodb_file_format = `Barracuda`; diff --git a/mysql-test/suite/encryption/r/innodb-encryption-disable.result b/mysql-test/suite/encryption/r/innodb-encryption-disable.result index dd2b025553e..4d15098d936 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-disable.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-disable.result @@ -1,5 +1,4 @@ -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t5 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("Couldn't load plugins from 'file_key_management*"); create table t5 ( `intcol1` int(32) DEFAULT NULL, diff --git a/mysql-test/suite/encryption/r/innodb-force-corrupt.result b/mysql-test/suite/encryption/r/innodb-force-corrupt.result index 3f3a2afb02d..7d63f47d17e 100644 --- a/mysql-test/suite/encryption/r/innodb-force-corrupt.result +++ b/mysql-test/suite/encryption/r/innodb-force-corrupt.result @@ -1,9 +1,5 @@ -CALL mtr.add_suppression("InnoDB: Database page corruption on disk or a failed .*"); -CALL mtr.add_suppression("InnoDB: Corruption: Block in space_id .*"); -CALL mtr.add_suppression("InnoDB: However key management plugin or used key_version .*"); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t[0-9]+ page \[page id: space=[0-9]+, page number=[0-9]+\]. You may have to recover from a backup."); SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; diff --git a/mysql-test/suite/encryption/r/innodb-missing-key.result b/mysql-test/suite/encryption/r/innodb-missing-key.result index 2f2cc025973..53ca0845c25 100644 --- a/mysql-test/suite/encryption/r/innodb-missing-key.result +++ b/mysql-test/suite/encryption/r/innodb-missing-key.result @@ -1,6 +1,4 @@ -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); # Start server with keys2.txt CREATE TABLE t1(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=19; diff --git a/mysql-test/suite/encryption/r/innodb-redo-badkey.result b/mysql-test/suite/encryption/r/innodb-redo-badkey.result index e12dad6d0d7..ed578d80b3a 100644 --- a/mysql-test/suite/encryption/r/innodb-redo-badkey.result +++ b/mysql-test/suite/encryption/r/innodb-redo-badkey.result @@ -1,16 +1,13 @@ call mtr.add_suppression("InnoDB: Block in space_id .* in file .* encrypted."); -call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("Plugin 'file_key_management' "); call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); -call mtr.add_suppression("InnoDB: Read operation failed for tablespace .*"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); -call mtr.add_suppression("InnoDB: Recovery read page .*"); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE .*"); -call mtr.add_suppression("InnoDB: Plugin initialization aborted .*"); +call mtr.add_suppression("InnoDB: Read operation failed for tablespace "); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed"); +call mtr.add_suppression("InnoDB: Recovery read page "); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[1234]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE "); +call mtr.add_suppression("InnoDB: Plugin initialization aborted "); call mtr.add_suppression("InnoDB: ############### CORRUPT LOG RECORD FOUND ##################"); # Restart mysqld --file-key-management-filename=keys2.txt # Wait max 10 min for key encryption threads to encrypt all spaces diff --git a/mysql-test/suite/encryption/r/innodb-redo-nokeys.result b/mysql-test/suite/encryption/r/innodb-redo-nokeys.result index dcbe1f5a395..1ea706dbde4 100644 --- a/mysql-test/suite/encryption/r/innodb-redo-nokeys.result +++ b/mysql-test/suite/encryption/r/innodb-redo-nokeys.result @@ -1,13 +1,10 @@ -call mtr.add_suppression("InnoDB: Block in space_id .*"); -call mtr.add_suppression("mysqld: File .*"); -call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("InnoDB: Block in space_id "); +call mtr.add_suppression("mysqld: File "); +call mtr.add_suppression("Plugin 'file_key_management' "); call mtr.add_suppression("InnoDB: cannot enable encryption, encryption plugin is not available"); -call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error\\."); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[1234]\\.ibd' cannot be decrypted\\."); # Restart mysqld --file-key-management-filename=keys2.txt SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index 6fa5fc9847f..9d61770f543 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -8,10 +8,11 @@ # table exists and encryption service is not available. # -call mtr.add_suppression("InnoDB: The page .*"); -call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* "); -call mtr.add_suppression("Plugin 'file_key_management' .*"); -call mtr.add_suppression("mysqld: File .*"); +call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); +call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t[12]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("mysqld: File .*keysbad3.txt' not found "); + --echo --echo # Start server with keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test index 52ee442c725..0c312f1f449 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test @@ -8,10 +8,9 @@ # MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys # MDEV-8727: Server/InnoDB hangs on shutdown after trying to read an encrypted table with a wrong key # -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1new cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t1(new)?\\.ibd' cannot be decrypted\\."); # Suppression for builds where file_key_management plugin is linked statically -call mtr.add_suppression("Couldn't load plugins from 'file_key_management.*"); +call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Table \'\"test\".\"t1\"\' tablespace is set as discarded."); --let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test index 14d88614f55..0c52bfe3fe6 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test @@ -7,7 +7,7 @@ # MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys # -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t1\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Cannot open table .*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); # Suppression for builds where file_key_management plugin is linked statically diff --git a/mysql-test/suite/encryption/t/innodb-compressed-blob.test b/mysql-test/suite/encryption/t/innodb-compressed-blob.test index cb69f22f745..4a81dae5a0f 100644 --- a/mysql-test/suite/encryption/t/innodb-compressed-blob.test +++ b/mysql-test/suite/encryption/t/innodb-compressed-blob.test @@ -4,8 +4,7 @@ # embedded does not support restart -- source include/not_embedded.inc -call mtr.add_suppression("InnoDB: However key management plugin or used key_version .*"); -call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file test/t[1-3] cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '..test.t[1-3]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("Unable to decompress space ..test.t[1-3].ibd \\[[1-9][0-9]*:[0-9]+\\]"); --echo # Restart mysqld --file-key-management-filename=keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-encryption-disable.test b/mysql-test/suite/encryption/t/innodb-encryption-disable.test index 38f36076c73..6ade8d6e3b2 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-disable.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-disable.test @@ -7,8 +7,7 @@ # MDEV-9559: Server without encryption configs crashes if selecting from an implicitly encrypted table # -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t5 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management*"); diff --git a/mysql-test/suite/encryption/t/innodb-force-corrupt.test b/mysql-test/suite/encryption/t/innodb-force-corrupt.test index dd4ee51b1eb..cb6440127cc 100644 --- a/mysql-test/suite/encryption/t/innodb-force-corrupt.test +++ b/mysql-test/suite/encryption/t/innodb-force-corrupt.test @@ -7,12 +7,8 @@ # Don't test under embedded -- source include/not_embedded.inc -CALL mtr.add_suppression("InnoDB: Database page corruption on disk or a failed .*"); -CALL mtr.add_suppression("InnoDB: Corruption: Block in space_id .*"); -CALL mtr.add_suppression("InnoDB: However key management plugin or used key_version .*"); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t[0-9]+ page \[page id: space=[0-9]+, page number=[0-9]+\]. You may have to recover from a backup."); --disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.test b/mysql-test/suite/encryption/t/innodb-missing-key.test index 07a2b16211c..6ea0528825b 100644 --- a/mysql-test/suite/encryption/t/innodb-missing-key.test +++ b/mysql-test/suite/encryption/t/innodb-missing-key.test @@ -7,9 +7,7 @@ # MDEV-11004: Unable to start (Segfault or os error 2) when encryption key missing # -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); --echo --echo # Start server with keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-redo-badkey.test b/mysql-test/suite/encryption/t/innodb-redo-badkey.test index 159646541c7..69de4f0f921 100644 --- a/mysql-test/suite/encryption/t/innodb-redo-badkey.test +++ b/mysql-test/suite/encryption/t/innodb-redo-badkey.test @@ -4,18 +4,15 @@ -- source include/not_embedded.inc call mtr.add_suppression("InnoDB: Block in space_id .* in file .* encrypted."); -call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("Plugin 'file_key_management' "); call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); -call mtr.add_suppression("InnoDB: Read operation failed for tablespace .*"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); -call mtr.add_suppression("InnoDB: Recovery read page .*"); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE .*"); -call mtr.add_suppression("InnoDB: Plugin initialization aborted .*"); +call mtr.add_suppression("InnoDB: Read operation failed for tablespace "); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed"); +call mtr.add_suppression("InnoDB: Recovery read page "); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[1234]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE "); +call mtr.add_suppression("InnoDB: Plugin initialization aborted "); call mtr.add_suppression("InnoDB: ############### CORRUPT LOG RECORD FOUND ##################"); --echo # Restart mysqld --file-key-management-filename=keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-redo-nokeys.test b/mysql-test/suite/encryption/t/innodb-redo-nokeys.test index e55e2ade153..68d831fcd17 100644 --- a/mysql-test/suite/encryption/t/innodb-redo-nokeys.test +++ b/mysql-test/suite/encryption/t/innodb-redo-nokeys.test @@ -3,16 +3,13 @@ # embedded does not support restart -- source include/not_embedded.inc -call mtr.add_suppression("InnoDB: Block in space_id .*"); -call mtr.add_suppression("mysqld: File .*"); -call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("InnoDB: Block in space_id "); +call mtr.add_suppression("mysqld: File "); +call mtr.add_suppression("Plugin 'file_key_management' "); call mtr.add_suppression("InnoDB: cannot enable encryption, encryption plugin is not available"); -call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error\\."); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[1234]\\.ibd' cannot be decrypted\\."); --echo # Restart mysqld --file-key-management-filename=keys2.txt -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 50c052fc690..ee8f40e7d39 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4746,9 +4746,9 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) ib_logf(IB_LOG_LEVEL_ERROR, "The page [page id: space=%u" ", page number=%u]" - " in file %s cannot be decrypted.", + " in file '%s' cannot be decrypted.", bpage->space, bpage->offset, - space->name); + space->chain.start->name); ib_logf(IB_LOG_LEVEL_INFO, "However key management plugin or used key_version " ULINTPF diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 3814d08f19a..5728b8a9fa3 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -4693,9 +4693,9 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) ib_logf(IB_LOG_LEVEL_ERROR, "The page [page id: space=%u" ", page number=%u]" - " in file %s cannot be decrypted.", + " in file '%s' cannot be decrypted.", bpage->space, bpage->offset, - space->name); + space->chain.start->name); ib_logf(IB_LOG_LEVEL_INFO, "However key management plugin or used key_version " ULINTPF From 51254da52c144781695250a98f365fbc090ef619 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 12 Jun 2018 12:37:28 +0200 Subject: [PATCH 126/203] MDEV-15359 Thread stay in "cleaning up" status after finishing make thd_get_error_context_description() to show not just thd->proc_info, but exactly the same thread state that SHOW PROCESSLIST shows. --- mysql-test/suite/plugins/r/processlist.result | 8 ++ mysql-test/suite/plugins/t/processlist.test | 18 ++++ sql/sql_class.cc | 97 ------------------- sql/sql_show.cc | 85 ++++++++++++++++ 4 files changed, 111 insertions(+), 97 deletions(-) create mode 100644 mysql-test/suite/plugins/r/processlist.result create mode 100644 mysql-test/suite/plugins/t/processlist.test diff --git a/mysql-test/suite/plugins/r/processlist.result b/mysql-test/suite/plugins/r/processlist.result new file mode 100644 index 00000000000..d08ea9d3523 --- /dev/null +++ b/mysql-test/suite/plugins/r/processlist.result @@ -0,0 +1,8 @@ +create table t1 (a int) engine=innodb; +start transaction; +insert t1 values (1); +state from show engine innodb status + +state from show processlist + +drop table t1; diff --git a/mysql-test/suite/plugins/t/processlist.test b/mysql-test/suite/plugins/t/processlist.test new file mode 100644 index 00000000000..5aacef317b9 --- /dev/null +++ b/mysql-test/suite/plugins/t/processlist.test @@ -0,0 +1,18 @@ +# +# MDEV-15359 Thread stay in "cleaning up" status after finishing +# +source include/have_innodb.inc; + +create table t1 (a int) engine=innodb; +start transaction; +insert t1 values (1); +let id=`select connection_id()`; +connect con2,localhost,root; +let s=query_get_value(show engine innodb status,Status,1); +disable_query_log; +eval select regexp_replace("$s", '(?s)^.*MySQL thread id $id,.*root([^\n]*)\n.*', '\\\\1') as `state from show engine innodb status`; +eval select state as `state from show processlist` from information_schema.processlist where id = $id; +enable_query_log; +disconnect con2; +connection default; +drop table t1; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index a644df9771c..3b293935048 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -717,103 +717,6 @@ extern "C" } -/** - Dumps a text description of a thread, its security context - (user, host) and the current query. - - @param thd thread context - @param buffer pointer to preferred result buffer - @param length length of buffer - @param max_query_len how many chars of query to copy (0 for all) - - @return Pointer to string -*/ - -extern "C" -char *thd_get_error_context_description(THD *thd, char *buffer, - unsigned int length, - unsigned int max_query_len) -{ - String str(buffer, length, &my_charset_latin1); - const Security_context *sctx= &thd->main_security_ctx; - char header[256]; - int len; - - mysql_mutex_lock(&LOCK_thread_count); - - /* - The pointers thd->query and thd->proc_info might change since they are - being modified concurrently. This is acceptable for proc_info since its - values doesn't have to very accurate and the memory it points to is static, - but we need to attempt a snapshot on the pointer values to avoid using NULL - values. The pointer to thd->query however, doesn't point to static memory - and has to be protected by thd->LOCK_thd_data or risk pointing to - uninitialized memory. - */ - const char *proc_info= thd->proc_info; - - len= my_snprintf(header, sizeof(header), - "MySQL thread id %lu, OS thread handle 0x%lx, query id %lu", - thd->thread_id, (ulong) thd->real_id, (ulong) thd->query_id); - str.length(0); - str.append(header, len); - - if (sctx->host) - { - str.append(' '); - str.append(sctx->host); - } - - if (sctx->ip) - { - str.append(' '); - str.append(sctx->ip); - } - - if (sctx->user) - { - str.append(' '); - str.append(sctx->user); - } - - if (proc_info) - { - str.append(' '); - str.append(proc_info); - } - - /* Don't wait if LOCK_thd_data is used as this could cause a deadlock */ - if (!mysql_mutex_trylock(&thd->LOCK_thd_data)) - { - if (thd->query()) - { - if (max_query_len < 1) - len= thd->query_length(); - else - len= MY_MIN(thd->query_length(), max_query_len); - str.append('\n'); - str.append(thd->query(), len); - } - mysql_mutex_unlock(&thd->LOCK_thd_data); - } - mysql_mutex_unlock(&LOCK_thread_count); - - if (str.c_ptr_safe() == buffer) - return buffer; - - /* - We have to copy the new string to the destination buffer because the string - was reallocated to a larger buffer to be able to fit. - */ - DBUG_ASSERT(buffer != NULL); - length= MY_MIN(str.length(), length-1); - memcpy(buffer, str.c_ptr_quick(), length); - /* Make sure that the new string is null terminated */ - buffer[length]= '\0'; - return buffer; -} - - #if MARIA_PLUGIN_INTERFACE_VERSION < 0x0200 /** TODO: This function is for API compatibility, remove it eventually. diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 965bed80148..d7641a6a96d 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -9773,3 +9773,88 @@ static void get_cs_converted_string_value(THD *thd, return; } #endif + +/** + Dumps a text description of a thread, its security context + (user, host) and the current query. + + @param thd thread context + @param buffer pointer to preferred result buffer + @param length length of buffer + @param max_query_len how many chars of query to copy (0 for all) + + @return Pointer to string +*/ + +extern "C" +char *thd_get_error_context_description(THD *thd, char *buffer, + unsigned int length, + unsigned int max_query_len) +{ + String str(buffer, length, &my_charset_latin1); + const Security_context *sctx= &thd->main_security_ctx; + char header[256]; + int len; + + mysql_mutex_lock(&LOCK_thread_count); + + len= my_snprintf(header, sizeof(header), + "MySQL thread id %lu, OS thread handle 0x%lx, query id %lu", + thd->thread_id, (ulong) thd->real_id, (ulong) thd->query_id); + str.length(0); + str.append(header, len); + + if (sctx->host) + { + str.append(' '); + str.append(sctx->host); + } + + if (sctx->ip) + { + str.append(' '); + str.append(sctx->ip); + } + + if (sctx->user) + { + str.append(' '); + str.append(sctx->user); + } + + /* Don't wait if LOCK_thd_data is used as this could cause a deadlock */ + if (!mysql_mutex_trylock(&thd->LOCK_thd_data)) + { + if (const char *info= thread_state_info(thd)) + { + str.append(' '); + str.append(info); + } + + if (thd->query()) + { + if (max_query_len < 1) + len= thd->query_length(); + else + len= MY_MIN(thd->query_length(), max_query_len); + str.append('\n'); + str.append(thd->query(), len); + } + mysql_mutex_unlock(&thd->LOCK_thd_data); + } + mysql_mutex_unlock(&LOCK_thread_count); + + if (str.c_ptr_safe() == buffer) + return buffer; + + /* + We have to copy the new string to the destination buffer because the string + was reallocated to a larger buffer to be able to fit. + */ + DBUG_ASSERT(buffer != NULL); + length= MY_MIN(str.length(), length-1); + memcpy(buffer, str.c_ptr_quick(), length); + /* Make sure that the new string is null terminated */ + buffer[length]= '\0'; + return buffer; +} From 3661d9882224dd485556ce937c1294eaeda02ef8 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 13 Jun 2018 20:31:40 +0200 Subject: [PATCH 127/203] fix SHOW PROCESSLIST for --embedded make it return the same Info values as for the standalone server. This fixes plugins.processlist for --embedded --- sql/sql_show.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/sql_show.cc b/sql/sql_show.cc index d7641a6a96d..af02c74c58b 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2267,16 +2267,16 @@ static const char *thread_state_info(THD *tmp) else return "Reading from net"; } - else +#else + if (tmp->get_command() == COM_SLEEP) + return ""; #endif - { - if (tmp->proc_info) - return tmp->proc_info; - else if (tmp->mysys_var && tmp->mysys_var->current_cond) - return "Waiting on cond"; - else - return NULL; - } + if (tmp->proc_info) + return tmp->proc_info; + else if (tmp->mysys_var && tmp->mysys_var->current_cond) + return "Waiting on cond"; + else + return NULL; } void mysqld_list_processes(THD *thd,const char *user, bool verbose) From f5eb37129f24893ab095e78c6fd2ef87e2c460cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Jun 2018 16:15:21 +0300 Subject: [PATCH 128/203] MDEV-13103 Deal with page_compressed page corruption fil_page_decompress(): Replaces fil_decompress_page(). Allow the caller detect errors. Remove duplicated code. Use the "safe" instead of "fast" variants of decompression routines. fil_page_compress(): Replaces fil_compress_page(). The length of the input buffer always was srv_page_size (innodb_page_size). Remove printouts, and remove the fil_space_t* parameter. buf_tmp_buffer_t::reserved: Make private; the accessors acquire() and release() will use atomic memory access. buf_pool_reserve_tmp_slot(): Make static. Remove the second parameter. Do not acquire any mutex. Remove the allocation of the buffers. buf_tmp_reserve_crypt_buf(), buf_tmp_reserve_compression_buf(): Refactored away from buf_pool_reserve_tmp_slot(). buf_page_decrypt_after_read(): Make static, and simplify the logic. Use the encryption buffer also for decompressing. buf_page_io_complete(), buf_dblwr_process(): Check more failures. fil_space_encrypt(): Simplify the debug checks. fil_space_t::printed_compression_failure: Remove. fil_get_compression_alg_name(): Remove. fil_iterate(): Allocate a buffer for compression and decompression only once, instead of allocating and freeing it for every page that uses compression, during IMPORT TABLESPACE. fil_node_get_space_id(), fil_page_is_index_page(), fil_page_is_lzo_compressed(): Remove (unused code). --- .../r/innodb-page_compression_default.result | 1 - .../r/innodb-page_compression_snappy.result | 1 - .../t/innodb-page_compression_default.test | 2 - .../t/innodb-page_compression_snappy.test | 2 - storage/innobase/buf/buf0buf.cc | 370 ++++------ storage/innobase/buf/buf0dblwr.cc | 37 +- storage/innobase/fil/fil0crypt.cc | 59 +- storage/innobase/fil/fil0fil.cc | 13 - storage/innobase/fil/fil0pagecompress.cc | 683 +++++------------- storage/innobase/ibuf/ibuf0ibuf.cc | 5 + storage/innobase/include/buf0buf.h | 53 +- storage/innobase/include/fil0fil.h | 3 - storage/innobase/include/fil0fil.ic | 1 - storage/innobase/include/fil0pagecompress.h | 93 +-- storage/innobase/include/fsp0pagecompress.ic | 70 +- storage/innobase/row/row0import.cc | 77 +- storage/xtradb/buf/buf0buf.cc | 380 +++++----- storage/xtradb/buf/buf0dblwr.cc | 40 +- storage/xtradb/fil/fil0crypt.cc | 59 +- storage/xtradb/fil/fil0fil.cc | 13 - storage/xtradb/fil/fil0pagecompress.cc | 683 +++++------------- storage/xtradb/ibuf/ibuf0ibuf.cc | 4 + storage/xtradb/include/buf0buf.h | 53 +- storage/xtradb/include/fil0fil.h | 3 - storage/xtradb/include/fil0fil.ic | 1 - storage/xtradb/include/fil0pagecompress.h | 93 +-- storage/xtradb/include/fsp0pagecompress.ic | 70 +- storage/xtradb/row/row0import.cc | 77 +- 28 files changed, 945 insertions(+), 2001 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-page_compression_default.result b/mysql-test/suite/innodb/r/innodb-page_compression_default.result index 18f7fb5e04a..19d0a875ce6 100644 --- a/mysql-test/suite/innodb/r/innodb-page_compression_default.result +++ b/mysql-test/suite/innodb/r/innodb-page_compression_default.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); set global innodb_file_format = `Barracuda`; set global innodb_file_per_table = on; create table innodb_normal (c1 int not null auto_increment primary key, b char(200)) engine=innodb; diff --git a/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result b/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result index 74cfa9bce3a..824b671eea3 100644 --- a/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result +++ b/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); set global innodb_compression_algorithm = snappy; set global innodb_file_format = `Barracuda`; set global innodb_file_per_table = on; diff --git a/mysql-test/suite/innodb/t/innodb-page_compression_default.test b/mysql-test/suite/innodb/t/innodb-page_compression_default.test index 1cc6c917548..34b1829485e 100644 --- a/mysql-test/suite/innodb/t/innodb-page_compression_default.test +++ b/mysql-test/suite/innodb/t/innodb-page_compression_default.test @@ -1,8 +1,6 @@ --source include/have_innodb.inc --source include/not_embedded.inc -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); - # All page compression test use the same --source include/innodb-page-compression.inc diff --git a/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test b/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test index 532ec294d28..0186c24ef2e 100644 --- a/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test +++ b/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test @@ -2,8 +2,6 @@ -- source include/have_innodb_snappy.inc --source include/not_embedded.inc -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); - # snappy set global innodb_compression_algorithm = snappy; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ee8f40e7d39..1386fd63cc3 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -354,14 +354,142 @@ on the io_type */ ? (counter##_READ) \ : (counter##_WRITTEN)) + +/** Reserve a buffer slot for encryption, decryption or page compression. +@param[in,out] buf_pool buffer pool +@return reserved buffer slot */ +static buf_tmp_buffer_t* buf_pool_reserve_tmp_slot(buf_pool_t* buf_pool) +{ + for (ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { + buf_tmp_buffer_t* slot = &buf_pool->tmp_arr->slots[i]; + if (slot->acquire()) { + return slot; + } + } + + /* We assume that free slot is found */ + ut_error; + return NULL; +} + +/** Reserve a buffer for encryption, decryption or decompression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_crypt_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->crypt_buf) { + slot->crypt_buf = static_cast( + aligned_malloc(srv_page_size, srv_page_size)); + } +} + +/** Reserve a buffer for compression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_compression_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->comp_buf) { + /* Both snappy and lzo compression methods require that + output buffer used for compression is bigger than input + buffer. Increase the allocated buffer size accordingly. */ + ulint size = srv_page_size; +#ifdef HAVE_LZO + size += LZO1X_1_15_MEM_COMPRESS; +#elif defined HAVE_SNAPPY + size = snappy_max_compressed_length(size); +#endif + slot->comp_buf = static_cast( + aligned_malloc(size, srv_page_size)); + } +} + /** Decrypt a page. @param[in,out] bpage Page control block @param[in,out] space tablespace @return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) - MY_ATTRIBUTE((nonnull)); +static bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) +{ + ut_ad(space->n_pending_ios > 0); + ut_ad(space->id == bpage->space); + + byte* dst_frame = bpage->zip.data ? bpage->zip.data : + ((buf_block_t*) bpage)->frame; + bool page_compressed = fil_page_is_compressed(dst_frame); + buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); + + if (bpage->offset == 0) { + /* File header pages are not encrypted/compressed */ + return true; + } + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + + buf_tmp_buffer_t* slot; + + if (page_compressed) { + /* the page we read is unencrypted */ + /* Find free slot from temporary memory array */ +decompress: + slot = buf_pool_reserve_tmp_slot(buf_pool); + /* For decompression, use crypt_buf. */ + buf_tmp_reserve_crypt_buf(slot); +decompress_with_slot: + ut_d(fil_page_type_validate(dst_frame)); + + bpage->write_size = fil_page_decompress(slot->crypt_buf, + dst_frame); + slot->release(); + + ut_ad(!bpage->write_size || fil_page_type_validate(dst_frame)); + ut_ad(space->n_pending_ios > 0); + return bpage->write_size != 0; + } + + if (space->crypt_data + && mach_read_from_4(FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + dst_frame)) { + /* Verify encryption checksum before we even try to + decrypt. */ + if (!fil_space_verify_crypt_checksum( + dst_frame, buf_page_get_zip_size(bpage), NULL, + bpage->offset)) { +decrypt_failed: + /* Mark page encrypted in case it should be. */ + if (space->crypt_data->type + != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted = true; + } + + return false; + } + + /* Find free slot from temporary memory array */ + slot = buf_pool_reserve_tmp_slot(buf_pool); + buf_tmp_reserve_crypt_buf(slot); + + ut_d(fil_page_type_validate(dst_frame)); + + /* decrypt using crypt_buf to dst_frame */ + if (!fil_space_decrypt(space, slot->crypt_buf, + dst_frame, &bpage->encrypted)) { + slot->release(); + goto decrypt_failed; + } + + ut_d(fil_page_type_validate(dst_frame)); + + if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress_with_slot; + } + + slot->release(); + } else if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress; + } + + ut_ad(space->n_pending_ios > 0); + return true; +} /********************************************************************//** Mark a table with the specified space pointed by bpage->space corrupted. @@ -4784,7 +4912,6 @@ buf_page_io_complete(buf_page_t* bpage, bool evict) buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); - byte* frame = NULL; dberr_t err = DB_SUCCESS; ut_a(buf_page_in_file(bpage)); @@ -4802,19 +4929,18 @@ buf_page_io_complete(buf_page_t* bpage, bool evict) ulint read_page_no = 0; ulint read_space_id = 0; uint key_version = 0; - - ut_ad(bpage->zip.data || ((buf_block_t*)bpage)->frame); + byte* frame = bpage->zip.data + ? bpage->zip.data + : reinterpret_cast(bpage)->frame; + ut_ad(frame); fil_space_t* space = fil_space_acquire_for_io(bpage->space); if (!space) { return(DB_TABLESPACE_DELETED); } - buf_page_decrypt_after_read(bpage, space); - - if (buf_page_get_zip_size(bpage)) { - frame = bpage->zip.data; - } else { - frame = ((buf_block_t*) bpage)->frame; + if (!buf_page_decrypt_after_read(bpage, space)) { + err = DB_DECRYPTION_FAILED; + goto database_corrupted; } if (buf_page_get_zip_size(bpage)) { @@ -4978,7 +5104,7 @@ database_corrupted: /* io_type == BUF_IO_WRITE */ if (bpage->slot) { /* Mark slot free */ - bpage->slot->reserved = false; + bpage->slot->release(); bpage->slot = NULL; } } @@ -6233,66 +6359,6 @@ buf_page_init_for_backup_restore( } #endif /* !UNIV_HOTBACKUP */ -/********************************************************************//** -Reserve unused slot from temporary memory array and allocate necessary -temporary memory if not yet allocated. -@return reserved slot */ -UNIV_INTERN -buf_tmp_buffer_t* -buf_pool_reserve_tmp_slot( -/*======================*/ - buf_pool_t* buf_pool, /*!< in: buffer pool where to - reserve */ - bool compressed) /*!< in: is file space compressed */ -{ - buf_tmp_buffer_t *free_slot=NULL; - - /* Array is protected by buf_pool mutex */ - buf_pool_mutex_enter(buf_pool); - - for(ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { - buf_tmp_buffer_t *slot = &buf_pool->tmp_arr->slots[i]; - - if(slot->reserved == false) { - free_slot = slot; - break; - } - } - - /* We assume that free slot is found */ - ut_a(free_slot != NULL); - free_slot->reserved = true; - /* Now that we have reserved this slot we can release - buf_pool mutex */ - buf_pool_mutex_exit(buf_pool); - - /* Allocate temporary memory for encryption/decryption */ - if (free_slot->crypt_buf == NULL) { - free_slot->crypt_buf = static_cast(aligned_malloc(UNIV_PAGE_SIZE, UNIV_PAGE_SIZE)); - memset(free_slot->crypt_buf, 0, UNIV_PAGE_SIZE); - } - - /* For page compressed tables allocate temporary memory for - compression/decompression */ - if (compressed && free_slot->comp_buf == NULL) { - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - size = snappy_max_compressed_length(size); -#endif -#if HAVE_LZO - size += LZO1X_1_15_MEM_COMPRESS; -#endif - free_slot->comp_buf = static_cast(aligned_malloc(size, UNIV_PAGE_SIZE)); - memset(free_slot->comp_buf, 0, size); - } - - return (free_slot); -} - /** Encryption and page_compression hook that is called just before a page is written to disk. @param[in,out] space tablespace @@ -6342,16 +6408,18 @@ buf_page_encrypt_before_write( } ulint zip_size = buf_page_get_zip_size(bpage); - ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + ut_ad(!zip_size || !page_compressed); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool); slot->out_buf = NULL; bpage->slot = slot; + buf_tmp_reserve_crypt_buf(slot); byte *dst_frame = slot->crypt_buf; if (!page_compressed) { +not_compressed: /* Encrypt page content */ byte* tmp = fil_space_encrypt(space, bpage->offset, @@ -6359,32 +6427,28 @@ buf_page_encrypt_before_write( src_frame, dst_frame); - bpage->real_size = page_size; + bpage->real_size = UNIV_PAGE_SIZE; slot->out_buf = dst_frame = tmp; ut_d(fil_page_type_validate(tmp)); } else { /* First we compress the page content */ - ulint out_len = 0; - - byte *tmp = fil_compress_page( - space, - (byte *)src_frame, - slot->comp_buf, - page_size, + buf_tmp_reserve_compression_buf(slot); + byte* tmp = slot->comp_buf; + ulint out_len = fil_page_compress( + src_frame, tmp, fsp_flags_get_page_compression_level(space->flags), fil_space_get_block_size(space, bpage->offset), - encrypted, - &out_len); + encrypted); + if (!out_len) { + goto not_compressed; + } bpage->real_size = out_len; -#ifdef UNIV_DEBUG - fil_page_type_validate(tmp); -#endif - - if(encrypted) { + ut_d(fil_page_type_validate(tmp)); + if (encrypted) { /* And then we encrypt the page content */ tmp = fil_space_encrypt(space, bpage->offset, @@ -6396,131 +6460,9 @@ buf_page_encrypt_before_write( slot->out_buf = dst_frame = tmp; } -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif + ut_d(fil_page_type_validate(dst_frame)); // return dst_frame which will be written return dst_frame; } - -/** Decrypt a page. -@param[in,out] bpage Page control block -@param[in,out] space tablespace -@return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) -{ - ut_ad(space->n_pending_ios > 0); - ut_ad(space->id == bpage->space); - - ulint zip_size = buf_page_get_zip_size(bpage); - ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - - byte* dst_frame = (zip_size) ? bpage->zip.data : - ((buf_block_t*) bpage)->frame; - unsigned key_version = - mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed = fil_page_is_compressed(dst_frame); - bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); - buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); - bool success = true; - - if (bpage->offset == 0) { - /* File header pages are not encrypted/compressed */ - return (true); - } - - /* Page is encrypted if encryption information is found from - tablespace and page contains used key_version. This is true - also for pages first compressed and then encrypted. */ - if (!space->crypt_data) { - key_version = 0; - } - - if (page_compressed) { - /* the page we read is unencrypted */ - /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size), - &bpage->write_size); - - /* Mark this slot as free */ - slot->reserved = false; - key_version = 0; - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - } else { - buf_tmp_buffer_t* slot = NULL; - - if (key_version) { - /* Verify encryption checksum before we even try to - decrypt. */ - if (!fil_space_verify_crypt_checksum(dst_frame, - zip_size, NULL, bpage->offset)) { - - /* Mark page encrypted in case it should - be. */ - if (space->crypt_data->type - != CRYPT_SCHEME_UNENCRYPTED) { - bpage->encrypted = true; - } - - return (false); - } - - /* Find free slot from temporary memory array */ - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - - /* decrypt using crypt_buf to dst_frame */ - if (!fil_space_decrypt(space, slot->crypt_buf, - dst_frame, &bpage->encrypted)) { - success = false; - } - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - } - - if (page_compressed_encrypted && success) { - if (!slot) { - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - } - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size), - &bpage->write_size); - ut_d(fil_page_type_validate(dst_frame)); - } - - /* Mark this slot as free */ - if (slot) { - slot->reserved = false; - } - } - - ut_ad(space->n_pending_ios > 0); - return (success); -} #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 70422671190..5f126ebb2a9 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -510,10 +510,11 @@ buf_dblwr_process() "Restoring possible half-written data pages " "from the doublewrite buffer..."); - unaligned_read_buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + unaligned_read_buf = static_cast(ut_malloc(3 * UNIV_PAGE_SIZE)); read_buf = static_cast( ut_align(unaligned_read_buf, UNIV_PAGE_SIZE)); + byte* const buf = read_buf + UNIV_PAGE_SIZE; for (std::list::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) { @@ -562,24 +563,23 @@ buf_dblwr_process() ignore this page (there should be redo log records to initialize it). */ } else { - if (fil_page_is_compressed_encrypted(read_buf) || - fil_page_is_compressed(read_buf)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, read_buf, srv_page_size, - NULL, true); + /* Decompress the page before + validating the checksum. */ + ulint decomp = fil_page_decompress(buf, read_buf); + if (!decomp || (decomp != srv_page_size && zip_size)) { + goto bad; } if (fil_space_verify_crypt_checksum( - read_buf, zip_size, NULL, page_no) - || !buf_page_is_corrupted( - true, read_buf, zip_size, space())) { + read_buf, zip_size, NULL, page_no) + || !buf_page_is_corrupted( + true, read_buf, zip_size, space())) { /* The page is good; there is no need to consult the doublewrite buffer. */ continue; } +bad: /* We intentionally skip this message for is_all_zero pages. */ ib_logf(IB_LOG_LEVEL_INFO, @@ -588,18 +588,15 @@ buf_dblwr_process() space_id, page_no); } - /* Next, validate the doublewrite page. */ - if (fil_page_is_compressed_encrypted(page) || - fil_page_is_compressed(page)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, page, srv_page_size, NULL, true); + ulint decomp = fil_page_decompress(buf, page); + if (!decomp || (decomp != srv_page_size && zip_size)) { + goto bad_doublewrite; } - - if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no) + if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, + page_no) && buf_page_is_corrupted(true, page, zip_size, space)) { if (!is_all_zero) { +bad_doublewrite: ib_logf(IB_LOG_LEVEL_WARN, "A doublewrite copy of page " ULINTPF ":" ULINTPF " is corrupted.", diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 2d149f07433..932d7b9e312 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -708,60 +708,39 @@ fil_space_encrypt( #ifdef UNIV_DEBUG if (tmp) { /* Verify that encrypted buffer is not corrupted */ - byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE); dberr_t err = DB_SUCCESS; byte* src = src_frame; bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - byte* comp_mem = NULL; - byte* uncomp_mem = NULL; + byte uncomp_mem[UNIV_PAGE_SIZE_MAX]; + byte tmp_mem[UNIV_PAGE_SIZE_MAX]; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; if (page_compressed_encrypted) { - comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); - fil_decompress_page(uncomp_mem, comp_mem, - srv_page_size, NULL); - src = uncomp_mem; + memcpy(uncomp_mem, src, srv_page_size); + ulint unzipped1 = fil_page_decompress( + tmp_mem, uncomp_mem); + ut_ad(unzipped1); + if (unzipped1 != srv_page_size) { + src = uncomp_mem; + } } - bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, space); - bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); + ut_ad(!buf_page_is_corrupted(true, src, zip_size, space)); + ut_ad(fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err)); + ut_ad(err == DB_SUCCESS); /* Need to decompress the page if it was also compressed */ if (page_compressed_encrypted) { - memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); - fil_decompress_page(tmp_mem, comp_mem, - srv_page_size, NULL); + byte buf[UNIV_PAGE_SIZE_MAX]; + memcpy(buf, tmp_mem, srv_page_size); + ulint unzipped2 = fil_page_decompress(tmp_mem, buf); + ut_ad(unzipped2); } - bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space); - memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); - bool different = memcmp(src, tmp_mem, size); - - if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { - fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", - ok , corrupted, corrupted1, err, different); - fprintf(stderr, "src_frame\n"); - buf_page_print(src_frame, zip_size); - fprintf(stderr, "encrypted_frame\n"); - buf_page_print(tmp, zip_size); - fprintf(stderr, "decrypted_frame\n"); - buf_page_print(tmp_mem, zip_size); - ut_ad(0); - } - - free(tmp_mem); - - if (comp_mem) { - free(comp_mem); - } - - if (uncomp_mem) { - free(uncomp_mem); - } + memcpy(tmp_mem + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + src + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); + ut_ad(!memcmp(src, tmp_mem, size)); } - #endif /* UNIV_DEBUG */ return tmp; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 112320bd3f5..5891db62758 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -342,19 +342,6 @@ fil_space_get_by_id( return(space); } -/****************************************************************//** -Get space id from fil node */ -ulint -fil_node_get_space_id( -/*==================*/ - fil_node_t* node) /*!< in: Compressed node*/ -{ - ut_ad(node); - ut_ad(node->space); - - return (node->space->id); -} - /*******************************************************************//** Returns the table space by a given name, NULL if not found. */ UNIV_INLINE diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 0ac764d5a32..25cd8e28a91 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -80,73 +80,26 @@ static ulint srv_data_read, srv_data_written; #include "snappy-c.h" #endif -/* Used for debugging */ -//#define UNIV_PAGECOMPRESS_DEBUG 1 - -/****************************************************************//** -For page compressed pages compress the page before actual write -operation. -@return compressed page to be written*/ -UNIV_INTERN -byte* -fil_compress_page( -/*==============*/ - fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: compressed buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint level, /* in: compression level */ - ulint block_size, /*!< in: block size */ - bool encrypted, /*!< in: is page also encrypted */ - ulint* out_len) /*!< out: actual length of compressed - page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) { - int err = Z_OK; - int comp_level = level; + int comp_level = int(level); ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; - ulint write_size = 0; -#if HAVE_LZO - lzo_uint write_size_lzo = write_size; -#endif /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; - bool allocated = false; - - /* page_compression does not apply to tables or tablespaces - that use ROW_FORMAT=COMPRESSED */ - ut_ad(!space || !FSP_FLAGS_GET_ZIP_SSIZE(space->flags)); if (encrypted) { header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; } - if (!out_buf) { - allocated = true; - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - if (comp_method == PAGE_SNAPPY_ALGORITHM) { - size = snappy_max_compressed_length(size); - } -#endif -#if HAVE_LZO - if (comp_method == PAGE_LZO_ALGORITHM) { - size += LZO1X_1_15_MEM_COMPRESS; - } -#endif - - out_buf = static_cast(ut_malloc(size)); - } - - ut_ad(buf); - ut_ad(out_buf); - ut_ad(len); - ut_ad(out_len); - /* Let's not compress file space header or extent descriptor */ switch (fil_page_get_type(buf)) { @@ -154,8 +107,7 @@ fil_compress_page( case FIL_PAGE_TYPE_FSP_HDR: case FIL_PAGE_TYPE_XDES: case FIL_PAGE_PAGE_COMPRESSED: - *out_len = len; - goto err_exit; + return 0; } /* If no compression level was provided to this table, use system @@ -164,204 +116,113 @@ fil_compress_page( comp_level = page_zip_level; } - DBUG_PRINT("compress", - ("Preparing for space " ULINTPF " '%s' len " ULINTPF, - space ? space->id : 0, - space ? space->name : "(import)", - len)); + ulint write_size = srv_page_size - header_len; - write_size = UNIV_PAGE_SIZE - header_len; - - switch(comp_method) { + switch (comp_method) { + default: + ut_ad(!"unknown compression method"); + /* fall through */ + case PAGE_UNCOMPRESSED: + return 0; + case PAGE_ZLIB_ALGORITHM: + { + ulong len = uLong(write_size); + if (Z_OK == compress2( + out_buf + header_len, &len, + buf, uLong(srv_page_size), comp_level)) { + write_size = len; + goto success; + } + } + break; #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: +# ifdef HAVE_LZ4_COMPRESS_DEFAULT + write_size = LZ4_compress_default( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# else + write_size = LZ4_compress_limitedOutput( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# endif -#ifdef HAVE_LZ4_COMPRESS_DEFAULT - err = LZ4_compress_default((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#else - err = LZ4_compress_limitedOutput((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#endif /* HAVE_LZ4_COMPRESS_DEFAULT */ - write_size = err; - - if (err == 0) { - /* If error we leave the actual page as it was */ - -#ifndef UNIV_PAGECOMPRESS_DEBUG - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; -#endif - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); -#ifndef UNIV_PAGECOMPRESS_DEBUG - } -#endif - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (write_size) { + goto success; } break; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO - case PAGE_LZO_ALGORITHM: - err = lzo1x_1_15_compress( - buf, len, out_buf+header_len, &write_size_lzo, out_buf+UNIV_PAGE_SIZE); + case PAGE_LZO_ALGORITHM: { + lzo_uint len = write_size; - write_size = write_size_lzo; - - if (err != LZO_E_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (LZO_E_OK == lzo1x_1_15_compress( + buf, srv_page_size, + out_buf + header_len, &len, + out_buf + srv_page_size) + && len <= write_size) { + write_size = len; + goto success; } - break; + } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - size_t out_pos=0; + size_t out_pos = 0; - err = lzma_easy_buffer_encode( - comp_level, - LZMA_CHECK_NONE, - NULL, /* No custom allocator, use malloc/free */ - reinterpret_cast(buf), - len, - reinterpret_cast(out_buf + header_len), - &out_pos, - (size_t)write_size); - - if (err != LZMA_OK || out_pos > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, out_pos); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (LZMA_OK == lzma_easy_buffer_encode( + comp_level, LZMA_CHECK_NONE, NULL, + buf, srv_page_size, out_buf + header_len, + &out_pos, write_size) + && out_pos <= write_size) { + write_size = out_pos; + goto success; } - - write_size = out_pos; - break; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - - err = BZ2_bzBuffToBuffCompress( - (char *)(out_buf + header_len), - (unsigned int *)&write_size, - (char *)buf, - len, - 1, - 0, - 0); - - if (err != BZ_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + unsigned len = unsigned(write_size); + if (BZ_OK == BZ2_bzBuffToBuffCompress( + reinterpret_cast(out_buf + header_len), + &len, + const_cast( + reinterpret_cast(buf)), + unsigned(srv_page_size), 1, 0, 0) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - write_size = snappy_max_compressed_length(UNIV_PAGE_SIZE); + case PAGE_SNAPPY_ALGORITHM: { + size_t len = snappy_max_compressed_length(srv_page_size); - cstatus = snappy_compress( - (const char *)buf, - (size_t)len, - (char *)(out_buf+header_len), - (size_t*)&write_size); - - if (cstatus != SNAPPY_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - (int)cstatus, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (SNAPPY_OK == snappy_compress( + reinterpret_cast(buf), + srv_page_size, + reinterpret_cast(out_buf) + header_len, + &len) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_SNAPPY */ - - case PAGE_ZLIB_ALGORITHM: - err = compress2(out_buf+header_len, (ulong*)&write_size, buf, - uLong(len), comp_level); - - if (err != Z_OK) { - /* If error we leave the actual page as it was */ - - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " rt %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; - } - break; - - case PAGE_UNCOMPRESSED: - *out_len = len; - return (buf); - break; - default: - ut_error; - break; } + srv_stats.pages_page_compression_error.inc(); + return 0; +success: /* Set up the page header */ memcpy(out_buf, buf, FIL_PAGE_DATA); /* Set up the checksum */ @@ -392,22 +253,11 @@ fil_compress_page( /* Verify that page can be decompressed */ { - byte *comp_page; - byte *uncomp_page; - - comp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - uncomp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); - - fil_decompress_page(uncomp_page, comp_page, ulong(len), NULL); - - if (buf_page_is_corrupted(false, uncomp_page, 0, space)) { - buf_page_print(uncomp_page, 0); - ut_ad(0); - } - - ut_free(comp_page); - ut_free(uncomp_page); + page_t tmp_buf[UNIV_PAGE_SIZE_MAX]; + page_t page[UNIV_PAGE_SIZE_MAX]; + memcpy(page, out_buf, srv_page_size); + ut_ad(fil_page_decompress(tmp_buf, page)); + ut_ad(!buf_page_is_corrupted(false, page, 0, NULL)); } #endif /* UNIV_DEBUG */ @@ -431,323 +281,144 @@ fil_compress_page( #endif } - DBUG_PRINT("compress", - ("Succeeded for space " ULINTPF - " '%s' len " ULINTPF " out_len " ULINTPF, - space ? space->id : 0, - space ? space->name : "(import)", - len, write_size)); - - srv_stats.page_compression_saved.add((len - write_size)); + srv_stats.page_compression_saved.add(srv_page_size - write_size); srv_stats.pages_page_compressed.inc(); /* If we do not persistently trim rest of page, we need to write it all */ if (!srv_use_trim) { - memset(out_buf+write_size,0,len-write_size); - write_size = len; + memset(out_buf + write_size, 0, srv_page_size - write_size); } - *out_len = write_size; - - if (allocated) { - /* TODO: reduce number of memcpy's */ - memcpy(buf, out_buf, len); - } else { - return(out_buf); - } - -err_exit: - if (allocated) { - ut_free(out_buf); - } - - return (buf); - + return write_size; } -/****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -UNIV_INTERN -void -fil_decompress_page( -/*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< out: buffer from which to read; in aio - this must be appropriately aligned */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of - the compressed data. */ - bool return_error) /*!< in: true if only an error should - be produced when decompression fails. - By default this parameter is false. */ +/** Decompress a page that may be subject to page_compressed compression. +@param[in,out] tmp_buf temporary buffer (of innodb_page_size) +@param[in,out] buf possibly compressed page buffer +@return size of the compressed data +@retval 0 if decompression failed +@retval srv_page_size if the page was not compressed */ +UNIV_INTERN ulint fil_page_decompress(byte* tmp_buf, byte* buf) { - int err = 0; - ulint actual_size = 0; - ulint compression_alg = 0; - byte *in_buf; - ulint ptype; - ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; - - ut_ad(buf); - ut_ad(len); - - ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); - - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; + const unsigned ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); + ulint header_len; + ib_uint64_t compression_alg; + switch (ptype) { + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + + FIL_PAGE_COMPRESSION_METHOD_SIZE; + compression_alg = mach_read_from_2( + FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + buf); + break; + case FIL_PAGE_PAGE_COMPRESSED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; + compression_alg = mach_read_from_8( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + buf); + break; + default: + return srv_page_size; } - /* Do not try to uncompressed pages that are not compressed */ - if (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return; + if (mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM) + != BUF_NO_CHECKSUM_MAGIC) { + return 0; } - // If no buffer was given, we need to allocate temporal buffer - if (page_buf == NULL) { - in_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - memset(in_buf, 0, UNIV_PAGE_SIZE); - } else { - in_buf = page_buf; - } + ulint actual_size = mach_read_from_2(buf + FIL_PAGE_DATA); - /* Before actual decompress, make sure that page type is correct */ - - if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || - (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page" - " CRC " ULINTPF " type " ULINTPF " len " ULINTPF ".", - mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM), - mach_read_from_2(buf+FIL_PAGE_TYPE), len); - - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - } - - /* Get compression algorithm */ - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - compression_alg = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE); - } else { - compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } - - /* Get the actual size of compressed page */ - actual_size = mach_read_from_2(buf+FIL_PAGE_DATA); /* Check if payload size is corrupted */ - if (actual_size == 0 || actual_size > UNIV_PAGE_SIZE) { + if (actual_size == 0 || actual_size > srv_page_size - header_len) { + return 0; + } + + switch (compression_alg) { + default: ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page" - " actual size " ULINTPF " compression %s.", - actual_size, fil_get_compression_alg_name(compression_alg)); - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - } - - /* Store actual payload size of the compressed data. This pointer - points to buffer pool. */ - if (write_size) { - *write_size = actual_size; - } - - DBUG_PRINT("compress", - ("Preparing for decompress for len " ULINTPF ".", - actual_size)); - - switch(compression_alg) { + "Unknown compression algorithm " UINT64PF, + compression_alg); + return 0; case PAGE_ZLIB_ALGORITHM: - err= uncompress(in_buf, &len, buf+header_len, (unsigned long)actual_size); - - /* If uncompress fails it means that page is corrupted */ - if (err != Z_OK) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; + { + uLong len = srv_page_size; + if (Z_OK != uncompress(tmp_buf, &len, + buf + header_len, + uLong(actual_size)) + && len != srv_page_size) { + return 0; } - ut_error; } break; - #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: - err = LZ4_decompress_fast((const char *)buf+header_len, (char *)in_buf, len); - - if (err != (int)actual_size) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (LZ4_decompress_safe(reinterpret_cast(buf) + + header_len, + reinterpret_cast(tmp_buf), + actual_size, srv_page_size) + == int(srv_page_size)) { + break; } - break; + return 0; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO case PAGE_LZO_ALGORITHM: { - ulint olen = 0; - lzo_uint olen_lzo = olen; - err = lzo1x_decompress((const unsigned char *)buf+header_len, - actual_size,(unsigned char *)in_buf, &olen_lzo, NULL); - - olen = olen_lzo; - - if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + lzo_uint len_lzo = srv_page_size; + if (LZO_E_OK == lzo1x_decompress_safe( + buf + header_len, + actual_size, tmp_buf, &len_lzo, NULL) + && len_lzo == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - - lzma_ret ret; size_t src_pos = 0; size_t dst_pos = 0; uint64_t memlimit = UINT64_MAX; - ret = lzma_stream_buffer_decode( - &memlimit, - 0, - NULL, - buf+header_len, - &src_pos, - actual_size, - in_buf, - &dst_pos, - len); - - - if (ret != LZMA_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size " ULINTPF "len " ULINTPF ".", - dst_pos, actual_size, len); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (LZMA_OK == lzma_stream_buffer_decode( + &memlimit, 0, NULL, buf + header_len, + &src_pos, actual_size, tmp_buf, &dst_pos, + srv_page_size) + && dst_pos == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - unsigned int dst_pos = UNIV_PAGE_SIZE; - - err = BZ2_bzBuffToBuffDecompress( - (char *)in_buf, - &dst_pos, - (char *)(buf+header_len), - actual_size, - 1, - 0); - - if (err != BZ_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %du bytes" - " size " ULINTPF " len " ULINTPF " err %d.", - dst_pos, actual_size, len, err); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + unsigned int dst_pos = srv_page_size; + if (BZ_OK == BZ2_bzBuffToBuffDecompress( + reinterpret_cast(tmp_buf), + &dst_pos, + reinterpret_cast(buf) + header_len, + actual_size, 1, 0) + && dst_pos == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - ulint olen = UNIV_PAGE_SIZE; + case PAGE_SNAPPY_ALGORITHM: { + size_t olen = srv_page_size; - cstatus = snappy_uncompress( - (const char *)(buf+header_len), - (size_t)actual_size, - (char *)in_buf, - (size_t*)&olen); - - if (cstatus != SNAPPY_OK || olen != UNIV_PAGE_SIZE) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only " ULINTPF " bytes" - " size " ULINTPF " len " ULINTPF " err %d.", - olen, actual_size, len, (int)cstatus); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (SNAPPY_OK == snappy_uncompress( + reinterpret_cast(buf) + header_len, + actual_size, + reinterpret_cast(tmp_buf), &olen) + && olen == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_SNAPPY */ - default: - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but compression algorithm %s" - " is not known." - ,fil_get_compression_alg_name(compression_alg)); - - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - break; } srv_stats.pages_page_decompressed.inc(); - - /* Copy the uncompressed page to the buffer pool, not - really any other options. */ - memcpy(buf, in_buf, len); - -error_return: - if (page_buf != in_buf) { - ut_free(in_buf); - } + memcpy(buf, tmp_buf, srv_page_size); + return actual_size; } diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 2b406e116b0..17fe3ae7511 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -5177,6 +5177,11 @@ ibuf_check_bitmap_on_import( bitmap_page = ibuf_bitmap_get_map_page( space_id, page_no, zip_size, &mtr); + if (!bitmap_page) { + mutex_exit(&ibuf_mutex); + return DB_CORRUPTION; + } + for (i = FSP_IBUF_BITMAP_OFFSET + 1; i < page_size; i++) { const ulint offset = page_no + i; diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 857bdf9d2be..aca4e58810c 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -39,6 +39,7 @@ Created 11/5/1995 Heikki Tuuri #include "ut0rbt.h" #include "os0proc.h" #include "log0log.h" +#include "my_atomic.h" /** @name Modes for buf_page_get_gen */ /* @{ */ @@ -1506,45 +1507,16 @@ buf_page_encrypt_before_write( buf_page_t* bpage, byte* src_frame); -/********************************************************************** -The hook that is called after page is written to disk. -The function releases any resources needed for encryption that was allocated -in buf_page_encrypt_before_write */ -UNIV_INTERN -ibool -buf_page_encrypt_after_write( -/*=========================*/ - buf_page_t* page); /*!< in/out: buffer page that was flushed */ - -/********************************************************************//** -The hook that is called just before a page is read from disk. -The function allocates memory that is used to temporarily store disk content -before getting decrypted */ -UNIV_INTERN -byte* -buf_page_decrypt_before_read( -/*=========================*/ - buf_page_t* page, /*!< in/out: buffer page read from disk */ - ulint zip_size); /*!< in: compressed page size, or 0 */ - -/********************************************************************//** -The hook that is called just after a page is read from disk. -The function decrypt disk content into buf_page_t and releases the -temporary buffer that was allocated in buf_page_decrypt_before_read */ -UNIV_INTERN -bool -buf_page_decrypt_after_read( -/*========================*/ - buf_page_t* page); /*!< in/out: buffer page read from disk */ - /** @brief The temporary memory structure. NOTE! The definition appears here only for other modules of this directory (buf) to see it. Do not use from outside! */ typedef struct { - bool reserved; /*!< true if this slot is reserved +private: + int32 reserved; /*!< true if this slot is reserved */ +public: byte* crypt_buf; /*!< for encryption the data needs to be copied to a separate buffer before it's encrypted&written. this as a page can be @@ -1555,6 +1527,21 @@ typedef struct { byte* out_buf; /*!< resulting buffer after encryption/compression. This is a pointer and not allocated. */ + + /** Release the slot */ + void release() + { + my_atomic_store32_explicit(&reserved, false, + MY_MEMORY_ORDER_RELAXED); + } + + /** Acquire the slot + @return whether the slot was acquired */ + bool acquire() + { + return !my_atomic_fas32_explicit(&reserved, true, + MY_MEMORY_ORDER_RELAXED); + } } buf_tmp_buffer_t; /** The common buffer control block structure diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index ae2629b5b04..a8dc9e14381 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -341,9 +341,6 @@ struct fil_space_t { bool is_in_unflushed_spaces; /*!< true if this space is currently in unflushed_spaces */ - bool printed_compression_failure; - /*!< true if we have already printed - compression failure */ fil_space_crypt_t* crypt_data; /*!< tablespace crypt data or NULL */ ulint file_block_size; diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index 49fdff4f3e5..3f21c529308 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -68,7 +68,6 @@ fil_get_page_type_name( } return "PAGE TYPE CORRUPTED"; - } /****************************************************************//** diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index 03e16699ce3..934372c55b2 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018 MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -30,70 +30,26 @@ atomic writes information to table space. Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ -/*******************************************************************//** -Find out wheather the page is index page or not -@return true if page type index page, false if not */ -UNIV_INLINE -ibool -fil_page_is_index_page( -/*===================*/ - byte *buf); /*!< in: page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) + MY_ATTRIBUTE((nonnull, warn_unused_result)); -/****************************************************************//** -Get the name of the compression algorithm used for page -compression. -@return compression algorithm name or "UNKNOWN" if not known*/ -UNIV_INLINE -const char* -fil_get_compression_alg_name( -/*=========================*/ - ulint comp_alg); /*! @@ -3365,15 +3371,30 @@ fil_iterate( os_offset_t offset; ulint n_bytes = iter.n_io_buffers * iter.page_size; + const ulint buf_size = srv_page_size +#ifdef HAVE_LZO + + LZO1X_1_15_MEM_COMPRESS +#elif defined HAVE_SNAPPY + + snappy_max_compressed_length(srv_page_size) +#endif + ; + byte* page_compress_buf = static_cast( + ut_malloc_low(buf_size, false)); ut_ad(!srv_read_only_mode); + if (!page_compress_buf) { + return DB_OUT_OF_MEMORY; + } + /* TODO: For ROW_FORMAT=COMPRESSED tables we do a lot of useless copying for non-index pages. Unfortunately, it is required by buf_zip_decompress() */ + dberr_t err = DB_SUCCESS; for (offset = iter.start; offset < iter.end; offset += n_bytes) { if (callback.is_interrupted()) { - return DB_INTERRUPTED; + err = DB_INTERRUPTED; + goto func_exit; } byte* io_buffer = iter.io_buffer; @@ -3404,12 +3425,13 @@ fil_iterate( if (!os_file_read_no_error_handling(iter.file, readptr, offset, n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed"); - return DB_IO_ERROR; + err = DB_IO_ERROR; + goto func_exit; } bool updated = false; - ulint n_pages_read = (ulint) n_bytes / iter.page_size; const ulint size = iter.page_size; + ulint n_pages_read = ulint(n_bytes) / size; block->page.offset = offset / size; for (ulint i = 0; i < n_pages_read; @@ -3438,11 +3460,11 @@ page_corrupted: UINT64PF " looks corrupted.", callback.filename(), ulong(offset / size), offset); - return DB_CORRUPTION; + err = DB_CORRUPTION; + goto func_exit; } bool decrypted = false; - dberr_t err = DB_SUCCESS; byte* dst = io_buffer + (i * size); bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -3451,6 +3473,10 @@ page_corrupted: == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED || page_type == FIL_PAGE_PAGE_COMPRESSED; + if (page_compressed && block->page.zip.data) { + goto page_corrupted; + } + if (!encrypted) { } else if (!mach_read_from_4( FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION @@ -3476,7 +3502,7 @@ not_encrypted: iter.page_size, src, &err); if (err != DB_SUCCESS) { - return err; + goto func_exit; } if (!decrypted) { @@ -3489,8 +3515,12 @@ not_encrypted: /* If the original page is page_compressed, we need to decompress it before adjusting further. */ if (page_compressed) { - fil_decompress_page(NULL, dst, ulong(size), - NULL); + ulint compress_length = fil_page_decompress( + page_compress_buf, dst); + ut_ad(compress_length != srv_page_size); + if (compress_length == 0) { + goto page_corrupted; + } updated = true; } else if (buf_page_is_corrupted( false, @@ -3501,7 +3531,7 @@ not_encrypted: } if ((err = callback(block)) != DB_SUCCESS) { - return err; + goto func_exit; } else if (!updated) { updated = buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE; @@ -3551,19 +3581,17 @@ not_encrypted: src = io_buffer + (i * size); if (page_compressed) { - ulint len = 0; - - fil_compress_page( - NULL, - src, - NULL, - size, - 0,/* FIXME: compression level */ - 512,/* FIXME: use proper block size */ - encrypted, - &len); - updated = true; + if (fil_page_compress( + src, + page_compress_buf, + 0,/* FIXME: compression level */ + 512,/* FIXME: proper block size */ + encrypted)) { + /* FIXME: remove memcpy() */ + memcpy(src, page_compress_buf, + srv_page_size); + } } /* If tablespace is encrypted, encrypt page before we @@ -3600,11 +3628,14 @@ not_encrypted: offset, (ulint) n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed"); - return DB_IO_ERROR; + err = DB_IO_ERROR; + goto func_exit; } } - return DB_SUCCESS; +func_exit: + ut_free(page_compress_buf); + return err; } /********************************************************************//** diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 5728b8a9fa3..3ae6dd56586 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -77,15 +77,6 @@ Created 11/5/1995 Heikki Tuuri #include "snappy-c.h" #endif -/** Decrypt a page. -@param[in,out] bpage Page control block -@param[in,out] space tablespace -@return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) - MY_ATTRIBUTE((nonnull)); - /********************************************************************//** Mark a table with the specified space pointed by bpage->space corrupted. Also remove the bpage from LRU list. @@ -390,6 +381,143 @@ on the io_type */ ? (counter##_READ) \ : (counter##_WRITTEN)) + +/** Reserve a buffer slot for encryption, decryption or page compression. +@param[in,out] buf_pool buffer pool +@return reserved buffer slot */ +static buf_tmp_buffer_t* buf_pool_reserve_tmp_slot(buf_pool_t* buf_pool) +{ + for (ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { + buf_tmp_buffer_t* slot = &buf_pool->tmp_arr->slots[i]; + if (slot->acquire()) { + return slot; + } + } + + /* We assume that free slot is found */ + ut_error; + return NULL; +} + +/** Reserve a buffer for encryption, decryption or decompression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_crypt_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->crypt_buf) { + slot->crypt_buf = static_cast( + aligned_malloc(srv_page_size, srv_page_size)); + } +} + +/** Reserve a buffer for compression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_compression_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->comp_buf) { + /* Both snappy and lzo compression methods require that + output buffer used for compression is bigger than input + buffer. Increase the allocated buffer size accordingly. */ + ulint size = srv_page_size; +#ifdef HAVE_LZO + size += LZO1X_1_15_MEM_COMPRESS; +#elif defined HAVE_SNAPPY + size = snappy_max_compressed_length(size); +#endif + slot->comp_buf = static_cast( + aligned_malloc(size, srv_page_size)); + } +} + +/** Decrypt a page. +@param[in,out] bpage Page control block +@param[in,out] space tablespace +@return whether the operation was successful */ +static bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) +{ + ut_ad(space->n_pending_ios > 0); + ut_ad(space->id == bpage->space); + + byte* dst_frame = bpage->zip.data ? bpage->zip.data : + ((buf_block_t*) bpage)->frame; + bool page_compressed = fil_page_is_compressed(dst_frame); + buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); + + if (bpage->offset == 0) { + /* File header pages are not encrypted/compressed */ + return true; + } + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + + buf_tmp_buffer_t* slot; + + if (page_compressed) { + /* the page we read is unencrypted */ + /* Find free slot from temporary memory array */ +decompress: + slot = buf_pool_reserve_tmp_slot(buf_pool); + /* For decompression, use crypt_buf. */ + buf_tmp_reserve_crypt_buf(slot); +decompress_with_slot: + ut_d(fil_page_type_validate(dst_frame)); + + bpage->write_size = fil_page_decompress(slot->crypt_buf, + dst_frame); + slot->release(); + + ut_ad(!bpage->write_size || fil_page_type_validate(dst_frame)); + ut_ad(space->n_pending_ios > 0); + return bpage->write_size != 0; + } + + if (space->crypt_data + && mach_read_from_4(FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + dst_frame)) { + /* Verify encryption checksum before we even try to + decrypt. */ + if (!fil_space_verify_crypt_checksum( + dst_frame, buf_page_get_zip_size(bpage), NULL, + bpage->offset)) { +decrypt_failed: + /* Mark page encrypted in case it should be. */ + if (space->crypt_data->type + != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted = true; + } + + return false; + } + + /* Find free slot from temporary memory array */ + slot = buf_pool_reserve_tmp_slot(buf_pool); + buf_tmp_reserve_crypt_buf(slot); + + ut_d(fil_page_type_validate(dst_frame)); + + /* decrypt using crypt_buf to dst_frame */ + if (!fil_space_decrypt(space, slot->crypt_buf, + dst_frame, &bpage->encrypted)) { + slot->release(); + goto decrypt_failed; + } + + ut_d(fil_page_type_validate(dst_frame)); + + if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress_with_slot; + } + + slot->release(); + } else if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress; + } + + ut_ad(space->n_pending_ios > 0); + return true; +} + /********************************************************************//** Gets the smallest oldest_modification lsn for any page in the pool. Returns zero if all modified pages have been flushed to disk. @@ -4730,7 +4858,6 @@ buf_page_io_complete(buf_page_t* bpage) const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); bool have_LRU_mutex = false; - byte* frame = NULL; dberr_t err = DB_SUCCESS; ut_a(buf_page_in_file(bpage)); @@ -4748,19 +4875,18 @@ buf_page_io_complete(buf_page_t* bpage) ulint read_page_no = 0; ulint read_space_id = 0; uint key_version = 0; - - ut_ad(bpage->zip.data || ((buf_block_t*)bpage)->frame); + byte* frame = bpage->zip.data + ? bpage->zip.data + : reinterpret_cast(bpage)->frame; + ut_ad(frame); fil_space_t* space = fil_space_acquire_for_io(bpage->space); if (!space) { return(DB_TABLESPACE_DELETED); } - buf_page_decrypt_after_read(bpage, space); - - if (buf_page_get_zip_size(bpage)) { - frame = bpage->zip.data; - } else { - frame = ((buf_block_t*) bpage)->frame; + if (!buf_page_decrypt_after_read(bpage, space)) { + err = DB_DECRYPTION_FAILED; + goto database_corrupted; } if (buf_page_get_zip_size(bpage)) { @@ -4941,7 +5067,7 @@ database_corrupted: /* io_type == BUF_IO_WRITE */ if (bpage->slot) { /* Mark slot free */ - bpage->slot->reserved = false; + bpage->slot->release(); bpage->slot = NULL; } } @@ -6240,66 +6366,6 @@ buf_pool_mutex_exit( mutex_exit(&buf_pool->LRU_list_mutex); } -/********************************************************************//** -Reserve unused slot from temporary memory array and allocate necessary -temporary memory if not yet allocated. -@return reserved slot */ -UNIV_INTERN -buf_tmp_buffer_t* -buf_pool_reserve_tmp_slot( -/*======================*/ - buf_pool_t* buf_pool, /*!< in: buffer pool where to - reserve */ - bool compressed) /*!< in: is file space compressed */ -{ - buf_tmp_buffer_t *free_slot=NULL; - - /* Array is protected by buf_pool mutex */ - buf_pool_mutex_enter(buf_pool); - - for(ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { - buf_tmp_buffer_t *slot = &buf_pool->tmp_arr->slots[i]; - - if(slot->reserved == false) { - free_slot = slot; - break; - } - } - - /* We assume that free slot is found */ - ut_a(free_slot != NULL); - free_slot->reserved = true; - /* Now that we have reserved this slot we can release - buf_pool mutex */ - buf_pool_mutex_exit(buf_pool); - - /* Allocate temporary memory for encryption/decryption */ - if (free_slot->crypt_buf == NULL) { - free_slot->crypt_buf = static_cast(aligned_malloc(UNIV_PAGE_SIZE, UNIV_PAGE_SIZE)); - memset(free_slot->crypt_buf, 0, UNIV_PAGE_SIZE); - } - - /* For page compressed tables allocate temporary memory for - compression/decompression */ - if (compressed && free_slot->comp_buf == NULL) { - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - size = snappy_max_compressed_length(size); -#endif -#if HAVE_LZO - size += LZO1X_1_15_MEM_COMPRESS; -#endif - free_slot->comp_buf = static_cast(aligned_malloc(size, UNIV_PAGE_SIZE)); - memset(free_slot->comp_buf, 0, size); - } - - return (free_slot); -} - /** Encryption and page_compression hook that is called just before a page is written to disk. @param[in,out] space tablespace @@ -6349,16 +6415,18 @@ buf_page_encrypt_before_write( } ulint zip_size = buf_page_get_zip_size(bpage); - ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + ut_ad(!zip_size || !page_compressed); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool); slot->out_buf = NULL; bpage->slot = slot; + buf_tmp_reserve_crypt_buf(slot); byte *dst_frame = slot->crypt_buf; if (!page_compressed) { +not_compressed: /* Encrypt page content */ byte* tmp = fil_space_encrypt(space, bpage->offset, @@ -6366,32 +6434,28 @@ buf_page_encrypt_before_write( src_frame, dst_frame); - bpage->real_size = page_size; + bpage->real_size = UNIV_PAGE_SIZE; slot->out_buf = dst_frame = tmp; ut_d(fil_page_type_validate(tmp)); } else { /* First we compress the page content */ - ulint out_len = 0; - - byte *tmp = fil_compress_page( - space, - (byte *)src_frame, - slot->comp_buf, - page_size, + buf_tmp_reserve_compression_buf(slot); + byte* tmp = slot->comp_buf; + ulint out_len = fil_page_compress( + src_frame, tmp, fsp_flags_get_page_compression_level(space->flags), fil_space_get_block_size(space, bpage->offset), - encrypted, - &out_len); + encrypted); + if (!out_len) { + goto not_compressed; + } bpage->real_size = out_len; -#ifdef UNIV_DEBUG - fil_page_type_validate(tmp); -#endif - - if(encrypted) { + ut_d(fil_page_type_validate(tmp)); + if (encrypted) { /* And then we encrypt the page content */ tmp = fil_space_encrypt(space, bpage->offset, @@ -6403,130 +6467,8 @@ buf_page_encrypt_before_write( slot->out_buf = dst_frame = tmp; } -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif + ut_d(fil_page_type_validate(dst_frame)); // return dst_frame which will be written return dst_frame; } - -/** Decrypt a page. -@param[in,out] bpage Page control block -@param[in,out] space tablespace -@return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) -{ - ut_ad(space->n_pending_ios > 0); - ut_ad(space->id == bpage->space); - - ulint zip_size = buf_page_get_zip_size(bpage); - ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - - byte* dst_frame = (zip_size) ? bpage->zip.data : - ((buf_block_t*) bpage)->frame; - unsigned key_version = - mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed = fil_page_is_compressed(dst_frame); - bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); - buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); - bool success = true; - - if (bpage->offset == 0) { - /* File header pages are not encrypted/compressed */ - return (true); - } - - /* Page is encrypted if encryption information is found from - tablespace and page contains used key_version. This is true - also for pages first compressed and then encrypted. */ - if (!space->crypt_data) { - key_version = 0; - } - - if (page_compressed) { - /* the page we read is unencrypted */ - /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size), - &bpage->write_size); - - /* Mark this slot as free */ - slot->reserved = false; - key_version = 0; - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - } else { - buf_tmp_buffer_t* slot = NULL; - - if (key_version) { - /* Verify encryption checksum before we even try to - decrypt. */ - if (!fil_space_verify_crypt_checksum(dst_frame, - zip_size, NULL, bpage->offset)) { - - /* Mark page encrypted in case it should - be. */ - if (space->crypt_data->type - != CRYPT_SCHEME_UNENCRYPTED) { - bpage->encrypted = true; - } - - return (false); - } - - /* Find free slot from temporary memory array */ - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - - /* decrypt using crypt_buf to dst_frame */ - if (!fil_space_decrypt(space, slot->crypt_buf, - dst_frame, &bpage->encrypted)) { - success = false; - } - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - } - - if (page_compressed_encrypted && success) { - if (!slot) { - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - } - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size), - &bpage->write_size); - ut_d(fil_page_type_validate(dst_frame)); - } - - /* Mark this slot as free */ - if (slot) { - slot->reserved = false; - } - } - - ut_ad(space->n_pending_ios > 0); - return (success); -} diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index ee8c85446e9..0ec23cbdbe1 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -510,10 +510,11 @@ buf_dblwr_process() "Restoring possible half-written data pages " "from the doublewrite buffer..."); - unaligned_read_buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + unaligned_read_buf = static_cast(ut_malloc(3 * UNIV_PAGE_SIZE)); read_buf = static_cast( ut_align(unaligned_read_buf, UNIV_PAGE_SIZE)); + byte* const buf = read_buf + UNIV_PAGE_SIZE; for (std::list::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) { @@ -562,24 +563,26 @@ buf_dblwr_process() ignore this page (there should be redo log records to initialize it). */ } else { - if (fil_page_is_compressed_encrypted(read_buf) || - fil_page_is_compressed(read_buf)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, read_buf, srv_page_size, - NULL, true); + /* Decompress the page before + validating the checksum. */ + ulint decomp = fil_page_decompress(buf, read_buf); + if (!decomp) { + goto bad; + } + if (!decomp || (decomp != srv_page_size && zip_size)) { + goto bad; } if (fil_space_verify_crypt_checksum( - read_buf, zip_size, NULL, page_no) - || !buf_page_is_corrupted( - true, read_buf, zip_size, space())) { + read_buf, zip_size, NULL, page_no) + || !buf_page_is_corrupted( + true, read_buf, zip_size, space())) { /* The page is good; there is no need to consult the doublewrite buffer. */ continue; } +bad: /* We intentionally skip this message for is_all_zero pages. */ ib_logf(IB_LOG_LEVEL_INFO, @@ -588,18 +591,15 @@ buf_dblwr_process() space_id, page_no); } - /* Next, validate the doublewrite page. */ - if (fil_page_is_compressed_encrypted(page) || - fil_page_is_compressed(page)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, page, srv_page_size, NULL, true); + ulint decomp = fil_page_decompress(buf, page); + if (!decomp || (decomp != srv_page_size && zip_size)) { + goto bad_doublewrite; } - - if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no) + if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, + page_no) && buf_page_is_corrupted(true, page, zip_size, space)) { if (!is_all_zero) { +bad_doublewrite: ib_logf(IB_LOG_LEVEL_WARN, "A doublewrite copy of page " ULINTPF ":" ULINTPF " is corrupted.", diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 2d149f07433..932d7b9e312 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -708,60 +708,39 @@ fil_space_encrypt( #ifdef UNIV_DEBUG if (tmp) { /* Verify that encrypted buffer is not corrupted */ - byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE); dberr_t err = DB_SUCCESS; byte* src = src_frame; bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - byte* comp_mem = NULL; - byte* uncomp_mem = NULL; + byte uncomp_mem[UNIV_PAGE_SIZE_MAX]; + byte tmp_mem[UNIV_PAGE_SIZE_MAX]; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; if (page_compressed_encrypted) { - comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); - fil_decompress_page(uncomp_mem, comp_mem, - srv_page_size, NULL); - src = uncomp_mem; + memcpy(uncomp_mem, src, srv_page_size); + ulint unzipped1 = fil_page_decompress( + tmp_mem, uncomp_mem); + ut_ad(unzipped1); + if (unzipped1 != srv_page_size) { + src = uncomp_mem; + } } - bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, space); - bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); + ut_ad(!buf_page_is_corrupted(true, src, zip_size, space)); + ut_ad(fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err)); + ut_ad(err == DB_SUCCESS); /* Need to decompress the page if it was also compressed */ if (page_compressed_encrypted) { - memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); - fil_decompress_page(tmp_mem, comp_mem, - srv_page_size, NULL); + byte buf[UNIV_PAGE_SIZE_MAX]; + memcpy(buf, tmp_mem, srv_page_size); + ulint unzipped2 = fil_page_decompress(tmp_mem, buf); + ut_ad(unzipped2); } - bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space); - memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); - bool different = memcmp(src, tmp_mem, size); - - if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { - fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", - ok , corrupted, corrupted1, err, different); - fprintf(stderr, "src_frame\n"); - buf_page_print(src_frame, zip_size); - fprintf(stderr, "encrypted_frame\n"); - buf_page_print(tmp, zip_size); - fprintf(stderr, "decrypted_frame\n"); - buf_page_print(tmp_mem, zip_size); - ut_ad(0); - } - - free(tmp_mem); - - if (comp_mem) { - free(comp_mem); - } - - if (uncomp_mem) { - free(uncomp_mem); - } + memcpy(tmp_mem + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + src + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); + ut_ad(!memcmp(src, tmp_mem, size)); } - #endif /* UNIV_DEBUG */ return tmp; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index bd7dfdf5ac8..a8835083165 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -351,19 +351,6 @@ fil_space_get_by_id( return(space); } -/****************************************************************//** -Get space id from fil node */ -ulint -fil_node_get_space_id( -/*==================*/ - fil_node_t* node) /*!< in: Compressed node*/ -{ - ut_ad(node); - ut_ad(node->space); - - return (node->space->id); -} - /*******************************************************************//** Returns the table space by a given name, NULL if not found. */ fil_space_t* diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 0ac764d5a32..25cd8e28a91 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -80,73 +80,26 @@ static ulint srv_data_read, srv_data_written; #include "snappy-c.h" #endif -/* Used for debugging */ -//#define UNIV_PAGECOMPRESS_DEBUG 1 - -/****************************************************************//** -For page compressed pages compress the page before actual write -operation. -@return compressed page to be written*/ -UNIV_INTERN -byte* -fil_compress_page( -/*==============*/ - fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: compressed buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint level, /* in: compression level */ - ulint block_size, /*!< in: block size */ - bool encrypted, /*!< in: is page also encrypted */ - ulint* out_len) /*!< out: actual length of compressed - page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) { - int err = Z_OK; - int comp_level = level; + int comp_level = int(level); ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; - ulint write_size = 0; -#if HAVE_LZO - lzo_uint write_size_lzo = write_size; -#endif /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; - bool allocated = false; - - /* page_compression does not apply to tables or tablespaces - that use ROW_FORMAT=COMPRESSED */ - ut_ad(!space || !FSP_FLAGS_GET_ZIP_SSIZE(space->flags)); if (encrypted) { header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; } - if (!out_buf) { - allocated = true; - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - if (comp_method == PAGE_SNAPPY_ALGORITHM) { - size = snappy_max_compressed_length(size); - } -#endif -#if HAVE_LZO - if (comp_method == PAGE_LZO_ALGORITHM) { - size += LZO1X_1_15_MEM_COMPRESS; - } -#endif - - out_buf = static_cast(ut_malloc(size)); - } - - ut_ad(buf); - ut_ad(out_buf); - ut_ad(len); - ut_ad(out_len); - /* Let's not compress file space header or extent descriptor */ switch (fil_page_get_type(buf)) { @@ -154,8 +107,7 @@ fil_compress_page( case FIL_PAGE_TYPE_FSP_HDR: case FIL_PAGE_TYPE_XDES: case FIL_PAGE_PAGE_COMPRESSED: - *out_len = len; - goto err_exit; + return 0; } /* If no compression level was provided to this table, use system @@ -164,204 +116,113 @@ fil_compress_page( comp_level = page_zip_level; } - DBUG_PRINT("compress", - ("Preparing for space " ULINTPF " '%s' len " ULINTPF, - space ? space->id : 0, - space ? space->name : "(import)", - len)); + ulint write_size = srv_page_size - header_len; - write_size = UNIV_PAGE_SIZE - header_len; - - switch(comp_method) { + switch (comp_method) { + default: + ut_ad(!"unknown compression method"); + /* fall through */ + case PAGE_UNCOMPRESSED: + return 0; + case PAGE_ZLIB_ALGORITHM: + { + ulong len = uLong(write_size); + if (Z_OK == compress2( + out_buf + header_len, &len, + buf, uLong(srv_page_size), comp_level)) { + write_size = len; + goto success; + } + } + break; #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: +# ifdef HAVE_LZ4_COMPRESS_DEFAULT + write_size = LZ4_compress_default( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# else + write_size = LZ4_compress_limitedOutput( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# endif -#ifdef HAVE_LZ4_COMPRESS_DEFAULT - err = LZ4_compress_default((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#else - err = LZ4_compress_limitedOutput((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#endif /* HAVE_LZ4_COMPRESS_DEFAULT */ - write_size = err; - - if (err == 0) { - /* If error we leave the actual page as it was */ - -#ifndef UNIV_PAGECOMPRESS_DEBUG - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; -#endif - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); -#ifndef UNIV_PAGECOMPRESS_DEBUG - } -#endif - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (write_size) { + goto success; } break; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO - case PAGE_LZO_ALGORITHM: - err = lzo1x_1_15_compress( - buf, len, out_buf+header_len, &write_size_lzo, out_buf+UNIV_PAGE_SIZE); + case PAGE_LZO_ALGORITHM: { + lzo_uint len = write_size; - write_size = write_size_lzo; - - if (err != LZO_E_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (LZO_E_OK == lzo1x_1_15_compress( + buf, srv_page_size, + out_buf + header_len, &len, + out_buf + srv_page_size) + && len <= write_size) { + write_size = len; + goto success; } - break; + } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - size_t out_pos=0; + size_t out_pos = 0; - err = lzma_easy_buffer_encode( - comp_level, - LZMA_CHECK_NONE, - NULL, /* No custom allocator, use malloc/free */ - reinterpret_cast(buf), - len, - reinterpret_cast(out_buf + header_len), - &out_pos, - (size_t)write_size); - - if (err != LZMA_OK || out_pos > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, out_pos); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (LZMA_OK == lzma_easy_buffer_encode( + comp_level, LZMA_CHECK_NONE, NULL, + buf, srv_page_size, out_buf + header_len, + &out_pos, write_size) + && out_pos <= write_size) { + write_size = out_pos; + goto success; } - - write_size = out_pos; - break; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - - err = BZ2_bzBuffToBuffCompress( - (char *)(out_buf + header_len), - (unsigned int *)&write_size, - (char *)buf, - len, - 1, - 0, - 0); - - if (err != BZ_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + unsigned len = unsigned(write_size); + if (BZ_OK == BZ2_bzBuffToBuffCompress( + reinterpret_cast(out_buf + header_len), + &len, + const_cast( + reinterpret_cast(buf)), + unsigned(srv_page_size), 1, 0, 0) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - write_size = snappy_max_compressed_length(UNIV_PAGE_SIZE); + case PAGE_SNAPPY_ALGORITHM: { + size_t len = snappy_max_compressed_length(srv_page_size); - cstatus = snappy_compress( - (const char *)buf, - (size_t)len, - (char *)(out_buf+header_len), - (size_t*)&write_size); - - if (cstatus != SNAPPY_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - (int)cstatus, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (SNAPPY_OK == snappy_compress( + reinterpret_cast(buf), + srv_page_size, + reinterpret_cast(out_buf) + header_len, + &len) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_SNAPPY */ - - case PAGE_ZLIB_ALGORITHM: - err = compress2(out_buf+header_len, (ulong*)&write_size, buf, - uLong(len), comp_level); - - if (err != Z_OK) { - /* If error we leave the actual page as it was */ - - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " rt %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; - } - break; - - case PAGE_UNCOMPRESSED: - *out_len = len; - return (buf); - break; - default: - ut_error; - break; } + srv_stats.pages_page_compression_error.inc(); + return 0; +success: /* Set up the page header */ memcpy(out_buf, buf, FIL_PAGE_DATA); /* Set up the checksum */ @@ -392,22 +253,11 @@ fil_compress_page( /* Verify that page can be decompressed */ { - byte *comp_page; - byte *uncomp_page; - - comp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - uncomp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); - - fil_decompress_page(uncomp_page, comp_page, ulong(len), NULL); - - if (buf_page_is_corrupted(false, uncomp_page, 0, space)) { - buf_page_print(uncomp_page, 0); - ut_ad(0); - } - - ut_free(comp_page); - ut_free(uncomp_page); + page_t tmp_buf[UNIV_PAGE_SIZE_MAX]; + page_t page[UNIV_PAGE_SIZE_MAX]; + memcpy(page, out_buf, srv_page_size); + ut_ad(fil_page_decompress(tmp_buf, page)); + ut_ad(!buf_page_is_corrupted(false, page, 0, NULL)); } #endif /* UNIV_DEBUG */ @@ -431,323 +281,144 @@ fil_compress_page( #endif } - DBUG_PRINT("compress", - ("Succeeded for space " ULINTPF - " '%s' len " ULINTPF " out_len " ULINTPF, - space ? space->id : 0, - space ? space->name : "(import)", - len, write_size)); - - srv_stats.page_compression_saved.add((len - write_size)); + srv_stats.page_compression_saved.add(srv_page_size - write_size); srv_stats.pages_page_compressed.inc(); /* If we do not persistently trim rest of page, we need to write it all */ if (!srv_use_trim) { - memset(out_buf+write_size,0,len-write_size); - write_size = len; + memset(out_buf + write_size, 0, srv_page_size - write_size); } - *out_len = write_size; - - if (allocated) { - /* TODO: reduce number of memcpy's */ - memcpy(buf, out_buf, len); - } else { - return(out_buf); - } - -err_exit: - if (allocated) { - ut_free(out_buf); - } - - return (buf); - + return write_size; } -/****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -UNIV_INTERN -void -fil_decompress_page( -/*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< out: buffer from which to read; in aio - this must be appropriately aligned */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of - the compressed data. */ - bool return_error) /*!< in: true if only an error should - be produced when decompression fails. - By default this parameter is false. */ +/** Decompress a page that may be subject to page_compressed compression. +@param[in,out] tmp_buf temporary buffer (of innodb_page_size) +@param[in,out] buf possibly compressed page buffer +@return size of the compressed data +@retval 0 if decompression failed +@retval srv_page_size if the page was not compressed */ +UNIV_INTERN ulint fil_page_decompress(byte* tmp_buf, byte* buf) { - int err = 0; - ulint actual_size = 0; - ulint compression_alg = 0; - byte *in_buf; - ulint ptype; - ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; - - ut_ad(buf); - ut_ad(len); - - ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); - - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; + const unsigned ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); + ulint header_len; + ib_uint64_t compression_alg; + switch (ptype) { + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + + FIL_PAGE_COMPRESSION_METHOD_SIZE; + compression_alg = mach_read_from_2( + FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + buf); + break; + case FIL_PAGE_PAGE_COMPRESSED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; + compression_alg = mach_read_from_8( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + buf); + break; + default: + return srv_page_size; } - /* Do not try to uncompressed pages that are not compressed */ - if (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return; + if (mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM) + != BUF_NO_CHECKSUM_MAGIC) { + return 0; } - // If no buffer was given, we need to allocate temporal buffer - if (page_buf == NULL) { - in_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - memset(in_buf, 0, UNIV_PAGE_SIZE); - } else { - in_buf = page_buf; - } + ulint actual_size = mach_read_from_2(buf + FIL_PAGE_DATA); - /* Before actual decompress, make sure that page type is correct */ - - if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || - (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page" - " CRC " ULINTPF " type " ULINTPF " len " ULINTPF ".", - mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM), - mach_read_from_2(buf+FIL_PAGE_TYPE), len); - - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - } - - /* Get compression algorithm */ - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - compression_alg = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE); - } else { - compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } - - /* Get the actual size of compressed page */ - actual_size = mach_read_from_2(buf+FIL_PAGE_DATA); /* Check if payload size is corrupted */ - if (actual_size == 0 || actual_size > UNIV_PAGE_SIZE) { + if (actual_size == 0 || actual_size > srv_page_size - header_len) { + return 0; + } + + switch (compression_alg) { + default: ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page" - " actual size " ULINTPF " compression %s.", - actual_size, fil_get_compression_alg_name(compression_alg)); - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - } - - /* Store actual payload size of the compressed data. This pointer - points to buffer pool. */ - if (write_size) { - *write_size = actual_size; - } - - DBUG_PRINT("compress", - ("Preparing for decompress for len " ULINTPF ".", - actual_size)); - - switch(compression_alg) { + "Unknown compression algorithm " UINT64PF, + compression_alg); + return 0; case PAGE_ZLIB_ALGORITHM: - err= uncompress(in_buf, &len, buf+header_len, (unsigned long)actual_size); - - /* If uncompress fails it means that page is corrupted */ - if (err != Z_OK) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; + { + uLong len = srv_page_size; + if (Z_OK != uncompress(tmp_buf, &len, + buf + header_len, + uLong(actual_size)) + && len != srv_page_size) { + return 0; } - ut_error; } break; - #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: - err = LZ4_decompress_fast((const char *)buf+header_len, (char *)in_buf, len); - - if (err != (int)actual_size) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (LZ4_decompress_safe(reinterpret_cast(buf) + + header_len, + reinterpret_cast(tmp_buf), + actual_size, srv_page_size) + == int(srv_page_size)) { + break; } - break; + return 0; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO case PAGE_LZO_ALGORITHM: { - ulint olen = 0; - lzo_uint olen_lzo = olen; - err = lzo1x_decompress((const unsigned char *)buf+header_len, - actual_size,(unsigned char *)in_buf, &olen_lzo, NULL); - - olen = olen_lzo; - - if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + lzo_uint len_lzo = srv_page_size; + if (LZO_E_OK == lzo1x_decompress_safe( + buf + header_len, + actual_size, tmp_buf, &len_lzo, NULL) + && len_lzo == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - - lzma_ret ret; size_t src_pos = 0; size_t dst_pos = 0; uint64_t memlimit = UINT64_MAX; - ret = lzma_stream_buffer_decode( - &memlimit, - 0, - NULL, - buf+header_len, - &src_pos, - actual_size, - in_buf, - &dst_pos, - len); - - - if (ret != LZMA_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size " ULINTPF "len " ULINTPF ".", - dst_pos, actual_size, len); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (LZMA_OK == lzma_stream_buffer_decode( + &memlimit, 0, NULL, buf + header_len, + &src_pos, actual_size, tmp_buf, &dst_pos, + srv_page_size) + && dst_pos == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - unsigned int dst_pos = UNIV_PAGE_SIZE; - - err = BZ2_bzBuffToBuffDecompress( - (char *)in_buf, - &dst_pos, - (char *)(buf+header_len), - actual_size, - 1, - 0); - - if (err != BZ_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %du bytes" - " size " ULINTPF " len " ULINTPF " err %d.", - dst_pos, actual_size, len, err); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + unsigned int dst_pos = srv_page_size; + if (BZ_OK == BZ2_bzBuffToBuffDecompress( + reinterpret_cast(tmp_buf), + &dst_pos, + reinterpret_cast(buf) + header_len, + actual_size, 1, 0) + && dst_pos == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - ulint olen = UNIV_PAGE_SIZE; + case PAGE_SNAPPY_ALGORITHM: { + size_t olen = srv_page_size; - cstatus = snappy_uncompress( - (const char *)(buf+header_len), - (size_t)actual_size, - (char *)in_buf, - (size_t*)&olen); - - if (cstatus != SNAPPY_OK || olen != UNIV_PAGE_SIZE) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only " ULINTPF " bytes" - " size " ULINTPF " len " ULINTPF " err %d.", - olen, actual_size, len, (int)cstatus); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (SNAPPY_OK == snappy_uncompress( + reinterpret_cast(buf) + header_len, + actual_size, + reinterpret_cast(tmp_buf), &olen) + && olen == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_SNAPPY */ - default: - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but compression algorithm %s" - " is not known." - ,fil_get_compression_alg_name(compression_alg)); - - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - break; } srv_stats.pages_page_decompressed.inc(); - - /* Copy the uncompressed page to the buffer pool, not - really any other options. */ - memcpy(buf, in_buf, len); - -error_return: - if (page_buf != in_buf) { - ut_free(in_buf); - } + memcpy(buf, tmp_buf, srv_page_size); + return actual_size; } diff --git a/storage/xtradb/ibuf/ibuf0ibuf.cc b/storage/xtradb/ibuf/ibuf0ibuf.cc index b169916c34e..96a86eea4c1 100644 --- a/storage/xtradb/ibuf/ibuf0ibuf.cc +++ b/storage/xtradb/ibuf/ibuf0ibuf.cc @@ -5218,6 +5218,10 @@ ibuf_check_bitmap_on_import( bitmap_page = ibuf_bitmap_get_map_page( space_id, page_no, zip_size, &mtr); + if (!bitmap_page) { + mutex_exit(&ibuf_mutex); + return DB_CORRUPTION; + } for (i = FSP_IBUF_BITMAP_OFFSET + 1; i < page_size; i++) { const ulint offset = page_no + i; diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 7661ba1785d..0944b5d4067 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -38,6 +38,7 @@ Created 11/5/1995 Heikki Tuuri #include "ut0rbt.h" #include "os0proc.h" #include "log0log.h" +#include "my_atomic.h" /** @name Modes for buf_page_get_gen */ /* @{ */ @@ -1528,45 +1529,16 @@ buf_page_encrypt_before_write( buf_page_t* bpage, byte* src_frame); -/********************************************************************** -The hook that is called after page is written to disk. -The function releases any resources needed for encryption that was allocated -in buf_page_encrypt_before_write */ -UNIV_INTERN -ibool -buf_page_encrypt_after_write( -/*=========================*/ - buf_page_t* page); /*!< in/out: buffer page that was flushed */ - -/********************************************************************//** -The hook that is called just before a page is read from disk. -The function allocates memory that is used to temporarily store disk content -before getting decrypted */ -UNIV_INTERN -byte* -buf_page_decrypt_before_read( -/*=========================*/ - buf_page_t* page, /*!< in/out: buffer page read from disk */ - ulint zip_size); /*!< in: compressed page size, or 0 */ - -/********************************************************************//** -The hook that is called just after a page is read from disk. -The function decrypt disk content into buf_page_t and releases the -temporary buffer that was allocated in buf_page_decrypt_before_read */ -UNIV_INTERN -bool -buf_page_decrypt_after_read( -/*========================*/ - buf_page_t* page); /*!< in/out: buffer page read from disk */ - /** @brief The temporary memory structure. NOTE! The definition appears here only for other modules of this directory (buf) to see it. Do not use from outside! */ typedef struct { - bool reserved; /*!< true if this slot is reserved +private: + int32 reserved; /*!< true if this slot is reserved */ +public: byte* crypt_buf; /*!< for encryption the data needs to be copied to a separate buffer before it's encrypted&written. this as a page can be @@ -1577,6 +1549,21 @@ typedef struct { byte* out_buf; /*!< resulting buffer after encryption/compression. This is a pointer and not allocated. */ + + /** Release the slot */ + void release() + { + my_atomic_store32_explicit(&reserved, false, + MY_MEMORY_ORDER_RELAXED); + } + + /** Acquire the slot + @return whether the slot was acquired */ + bool acquire() + { + return !my_atomic_fas32_explicit(&reserved, true, + MY_MEMORY_ORDER_RELAXED); + } } buf_tmp_buffer_t; /** The common buffer control block structure diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 91d46afe7a8..7b8339fec7d 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -340,9 +340,6 @@ struct fil_space_t { corrupted page. */ bool is_corrupt; /*!< true if tablespace corrupted */ - bool printed_compression_failure; - /*!< true if we have already printed - compression failure */ fil_space_crypt_t* crypt_data; /*!< tablespace crypt data or NULL */ ulint file_block_size; diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic index 49fdff4f3e5..3f21c529308 100644 --- a/storage/xtradb/include/fil0fil.ic +++ b/storage/xtradb/include/fil0fil.ic @@ -68,7 +68,6 @@ fil_get_page_type_name( } return "PAGE TYPE CORRUPTED"; - } /****************************************************************//** diff --git a/storage/xtradb/include/fil0pagecompress.h b/storage/xtradb/include/fil0pagecompress.h index 03e16699ce3..934372c55b2 100644 --- a/storage/xtradb/include/fil0pagecompress.h +++ b/storage/xtradb/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018 MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -30,70 +30,26 @@ atomic writes information to table space. Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ -/*******************************************************************//** -Find out wheather the page is index page or not -@return true if page type index page, false if not */ -UNIV_INLINE -ibool -fil_page_is_index_page( -/*===================*/ - byte *buf); /*!< in: page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) + MY_ATTRIBUTE((nonnull, warn_unused_result)); -/****************************************************************//** -Get the name of the compression algorithm used for page -compression. -@return compression algorithm name or "UNKNOWN" if not known*/ -UNIV_INLINE -const char* -fil_get_compression_alg_name( -/*=========================*/ - ulint comp_alg); /*! @@ -3364,15 +3370,30 @@ fil_iterate( os_offset_t offset; ulint n_bytes = iter.n_io_buffers * iter.page_size; + const ulint buf_size = srv_page_size +#ifdef HAVE_LZO + + LZO1X_1_15_MEM_COMPRESS +#elif defined HAVE_SNAPPY + + snappy_max_compressed_length(srv_page_size) +#endif + ; + byte* page_compress_buf = static_cast( + ut_malloc_low(buf_size, false)); ut_ad(!srv_read_only_mode); + if (!page_compress_buf) { + return DB_OUT_OF_MEMORY; + } + /* TODO: For ROW_FORMAT=COMPRESSED tables we do a lot of useless copying for non-index pages. Unfortunately, it is required by buf_zip_decompress() */ + dberr_t err = DB_SUCCESS; for (offset = iter.start; offset < iter.end; offset += n_bytes) { if (callback.is_interrupted()) { - return DB_INTERRUPTED; + err = DB_INTERRUPTED; + goto func_exit; } byte* io_buffer = iter.io_buffer; @@ -3403,12 +3424,13 @@ fil_iterate( if (!os_file_read_no_error_handling(iter.file, readptr, offset, n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed"); - return DB_IO_ERROR; + err = DB_IO_ERROR; + goto func_exit; } bool updated = false; - ulint n_pages_read = (ulint) n_bytes / iter.page_size; const ulint size = iter.page_size; + ulint n_pages_read = ulint(n_bytes) / size; block->page.offset = offset / size; for (ulint i = 0; i < n_pages_read; @@ -3437,11 +3459,11 @@ page_corrupted: UINT64PF " looks corrupted.", callback.filename(), ulong(offset / size), offset); - return DB_CORRUPTION; + err = DB_CORRUPTION; + goto func_exit; } bool decrypted = false; - dberr_t err = DB_SUCCESS; byte* dst = io_buffer + (i * size); bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -3450,6 +3472,10 @@ page_corrupted: == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED || page_type == FIL_PAGE_PAGE_COMPRESSED; + if (page_compressed && block->page.zip.data) { + goto page_corrupted; + } + if (!encrypted) { } else if (!mach_read_from_4( FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION @@ -3475,7 +3501,7 @@ not_encrypted: iter.page_size, src, &err); if (err != DB_SUCCESS) { - return err; + goto func_exit; } if (!decrypted) { @@ -3488,8 +3514,12 @@ not_encrypted: /* If the original page is page_compressed, we need to decompress it before adjusting further. */ if (page_compressed) { - fil_decompress_page(NULL, dst, ulong(size), - NULL); + ulint compress_length = fil_page_decompress( + page_compress_buf, dst); + ut_ad(compress_length != srv_page_size); + if (compress_length == 0) { + goto page_corrupted; + } updated = true; } else if (buf_page_is_corrupted( false, @@ -3500,7 +3530,7 @@ not_encrypted: } if ((err = callback(block)) != DB_SUCCESS) { - return err; + goto func_exit; } else if (!updated) { updated = buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE; @@ -3550,19 +3580,17 @@ not_encrypted: src = io_buffer + (i * size); if (page_compressed) { - ulint len = 0; - - fil_compress_page( - NULL, - src, - NULL, - size, - 0,/* FIXME: compression level */ - 512,/* FIXME: use proper block size */ - encrypted, - &len); - updated = true; + if (fil_page_compress( + src, + page_compress_buf, + 0,/* FIXME: compression level */ + 512,/* FIXME: proper block size */ + encrypted)) { + /* FIXME: remove memcpy() */ + memcpy(src, page_compress_buf, + srv_page_size); + } } /* If tablespace is encrypted, encrypt page before we @@ -3599,11 +3627,14 @@ not_encrypted: offset, (ulint) n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed"); - return DB_IO_ERROR; + err = DB_IO_ERROR; + goto func_exit; } } - return DB_SUCCESS; +func_exit: + ut_free(page_compress_buf); + return err; } /********************************************************************//** From 776fc876862ccf6bc77938246bbab2b349d621a9 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 14 Jun 2018 09:08:41 +0200 Subject: [PATCH 129/203] fix compilation w/o partitioning followup for d8da920264a --- sql/sql_alter.cc | 1 - sql/sql_parse.cc | 1 - sql/sql_table.cc | 3 --- sql/table.cc | 2 ++ 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/sql/sql_alter.cc b/sql/sql_alter.cc index 6f21fb4b931..f17931ae919 100644 --- a/sql/sql_alter.cc +++ b/sql/sql_alter.cc @@ -303,7 +303,6 @@ bool Sql_cmd_alter_table::execute(THD *thd) create_info.data_file_name= create_info.index_file_name= NULL; thd->enable_slow_log= opt_log_slow_admin_statements; - thd->work_part_info= 0; #ifdef WITH_WSREP TABLE *find_temporary_table(THD *thd, const TABLE_LIST *tl); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index b2b2bfb5a27..89b5adff60c 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -3315,7 +3315,6 @@ mysql_execute_command(THD *thd) create_info.add(DDL_options_st::OPT_OR_REPLACE_SLAVE_GENERATED); } - thd->work_part_info= 0; #ifdef WITH_PARTITION_STORAGE_ENGINE { partition_info *part_info= thd->lex->part_info; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 9637702d836..3c9146354d6 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -6060,7 +6060,6 @@ remove_key: } } - DBUG_ASSERT(thd->work_part_info == 0); #ifdef WITH_PARTITION_STORAGE_ENGINE partition_info *tab_part_info= table->part_info; thd->work_part_info= thd->lex->part_info; @@ -8415,8 +8414,6 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, { DBUG_ENTER("mysql_alter_table"); - thd->work_part_info= 0; // Used by partitioning - /* Check if we attempt to alter mysql.slow_log or mysql.general_log table and return an error if diff --git a/sql/table.cc b/sql/table.cc index f6a27b1b1b3..4407d631db8 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -2233,7 +2233,9 @@ int TABLE_SHARE::init_from_sql_statement_string(THD *thd, bool write, goto ret; thd->lex->create_info.db_type= hton; +#ifdef WITH_PARTITION_STORAGE_ENGINE thd->work_part_info= 0; // For partitioning +#endif if (tabledef_version.str) thd->lex->create_info.tabledef_version= tabledef_version; From c22ab56f0d690feee471e173a3d95acb642cd6dc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 14 Jun 2018 15:12:13 +0200 Subject: [PATCH 130/203] fix galera sst tests note that ${A#foo} is $A if there's no prefix foo. That's why galera nodes tried to connect to 127.0.0.1:127.0.0.1 if there was no port in the address Followup for 2b35db5ac4e --- scripts/wsrep_sst_common.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 50264fe560a..e160bf5c776 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -41,15 +41,14 @@ case "$1" in addr_no_bracket=${WSREP_SST_OPT_ADDR#\[} readonly WSREP_SST_OPT_HOST_UNESCAPED=${addr_no_bracket%%\]*} readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" - remain=${WSREP_SST_OPT_ADDR#*\]} - remain=${remain#*:} ;; *) readonly WSREP_SST_OPT_HOST=${WSREP_SST_OPT_ADDR%%[:/]*} readonly WSREP_SST_OPT_HOST_UNESCAPED=$WSREP_SST_OPT_HOST - remain=${WSREP_SST_OPT_ADDR#*:} ;; esac + remain=${WSREP_SST_OPT_ADDR#${WSREP_SST_OPT_HOST}} + remain=${remain#:} readonly WSREP_SST_OPT_ADDR_PORT=${remain%%/*} remain=${remain#*/} readonly WSREP_SST_OPT_MODULE=${remain%%/*} From f4387288ab54837aa84ce728b2ce61e52850e4ec Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Fri, 15 Jun 2018 03:49:04 +0300 Subject: [PATCH 131/203] Updated list of unstable tests for 10.1.34 release --- mysql-test/unstable-tests | 137 ++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 65 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 5c795bf79c8..9b00579944c 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,74 +23,80 @@ # ############################################################################## -# Based on 10.1 0db66ab18ffef6d8920e2e6ff66e99516a458a4d +# Based on 10.1 c22ab56f0d690feee471e173a3d95acb642cd6dc +main.alter_table : Modified in 10.1.34 main.alter_table_trans : MDEV-12084 - timeout main.analyze_stmt_slow_query_log : MDEV-12237 - Wrong result +main.assign_key_cache : Added in 10.1.34 +main.assign_key_cache_debug : Added in 10.1.34 main.auth_named_pipe : MDEV-14724 - System error 2 main.connect : Modified in 10.1.33 main.connect_debug : Added in 10.1.33 main.create_delayed : MDEV-10605 - failed with timeout -main.ctype_latin1 : Modified in 10.1.32 +main.create_or_replace : Modified in 10.1.34 main.ctype_ucs : Modified in 10.1.33 main.ctype_utf16le : MDEV-10675: timeout or extra warnings -main.ctype_utf8 : Modified in 10.1.32 main.ctype_utf8mb4 : Modified in 10.1.33 -main.dyncol : Modified in 10.1.32 +main.grant : Modified in 10.1.34 +main.grant_not_windows : Added in 10.1.34 main.events_2 : MDEV-13277 - Server crash main.events_bugs : MDEV-12892 - Crash in fill_schema_processlist main.events_slowlog : MDEV-12821 - Wrong result main.events_restart : MDEV-12236 - Server shutdown problem -main.fast_prefix_index_fetch_innodb : Modified in 10.1.32 -main.func_date_add : Modified in 10.1.32 main.func_misc : Modified in 10.1.33 main.func_str : Modified in 10.1.33 -main.func_time : Modified in 10.1.32 -main.having : Modified in 10.1.32 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout main.index_merge_innodb : MDEV-7142 - Wrong execution plan, timeout with valgrind main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure -main.join_outer : Modified in 10.1.32 +main.insert_select : Modified in 10.1.34 main.kill_processlist-6619 : MDEV-10793 - wrong result in processlist +main.limit : Modified in 10.1.34 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect"; modified in 10.1.33 main.merge : MDEV-10607 - sporadic "can't connect" -main.myisam : Modified in 10.1.33 +main.myisam : Modified in 10.1.34 main.myisam_recover : Modified in 10.1.33 +main.mysql : Modified in 10.1.34 main.mysql_client_test_nonblock : MDEV-15096 - exec failed +main.mysql_cp932 : Modified in 10.1.34 main.mysql_upgrade_noengine : MDEV-14355 - Plugin is busy -main.mysqlslap : MDEV-11801 - timeout +main.mysqldump : Modified in 10.1.34 +main.mysqlslap : MDEV-11801 - timeout; modified in 10.1.34 main.mysqltest : MDEV-9269 - fails on Alpha +main.olap : Modified in 10.1.34 main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan main.parser : Modified in 10.1.33 main.partition_debug_sync : MDEV-15669 - Deadlock found when trying to get lock main.partition_innodb_plugin : MDEV-12901 - Valgrind warnings -main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count -main.ps_qc_innodb : Added in 10.1.32 +main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count; modified in 10.1.34 main.query_cache : MDEV-12895 - Wrong result main.query_cache_debug : MDEV-15281 - Resize or similar command in progress main.range_vs_index_merge_innodb : MDEV-15283 - Server has gone away -main.read_only_innodb : Modified in 10.1.33 +main.read_only_innodb : Modified in 10.1.34 +main.rename : Modified in 10.1.34 +main.selectivity : Modified in 10.1.34 main.set_statement : MDEV-13183 - Wrong result main.show_explain : MDEV-10674 - sporadic failure -main.sp : Modified in 10.1.33 +main.sp : Modified in 10.1.34 main.sp-destruct : Modified in 10.1.33 main.sp-innodb : Modified in 10.1.33 main.sp-security : MDEV-10607 - sporadic "can't connect" +main.stat_tables_par_innodb : MDEV-14155 - Wrong rounding main.statistics : Modified in 10.1.33 +main.statistics_close : Added in 10.1.34 main.status : MDEV-8510 - sporadic wrong result main.subselect4 : Modified in 10.1.33 main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.subselect_mat : Modified in 10.1.32 main.subselect_sj : Modified in 10.1.33 +main.subselect_sj2_mat : Modified in 10.1.34 main.symlink-aria-11902 : MDEV-15098 - error 40 from storage engine main.symlink-myisam-11902 : MDEV-15098 - error 40 from storage engine main.tc_heuristic_recover : MDEV-15200 - wrong error on mysqld_stub_cmd +main.trigger : Modified in 10.1.34 main.type_blob : MDEV-15195 - Wrong result main.type_datetime_hires : MDEV-10687 - timeout -main.type_time : Modified in 10.1.32 -main.type_time_6065 : Modified in 10.1.32 main.variables : Modified in 10.1.33 main.view : Modified in 10.1.33 @@ -138,29 +144,32 @@ connect.zip : MDEV-13884 - Wrong result #---------------------------------------------------------------- -disks.disks : Added in 10.1.32 - -#---------------------------------------------------------------- - encryption.create_or_replace : MDEV-16115 - Trying to access tablespace encryption.debug_key_management : MDEV-13841 - Timeout on wait condition encryption.encrypt_and_grep : MDEV-13765 - Wrong result -encryption.innodb-bad-key-change : Modified in 10.1.32 -encryption.innodb-compressed-blob : MDEV-14728 - Unable to get certificate +encryption.innodb-bad-key-change : Modified in 10.1.34 +encryption.innodb-bad-key-change2 : Modified in 10.1.34 +encryption.innodb-bad-key-change4 : Modified in 10.1.34 +encryption.innodb-compressed-blob : MDEV-14728 - Unable to get certificate; modified in 10.1.34 +encryption.innodb-discard-import : Modified in 10.1.34 +encryption.innodb-encryption-disable : Modified in 10.1.34 encryption.innodb_encryption_discard_import : MDEV-16116 - Wrong result; modified in 10.1.33 encryption.innodb_encryption_filekeys : Modified in 10.1.33 -encryption.innodb_encryption-page-compression : MDEV-12630 - crash or assertion failure; modified in 10.1.32 +encryption.innodb_encryption-page-compression : MDEV-12630 - crash or assertion failure encryption.innodb_encryption_row_compressed : MDEV-16113 - Crash encryption.innodb_first_page : MDEV-10689 - Crash encryption.innodb-first-page-read : MDEV-14356 - Timeout on wait condition +encryption.innodb-force-corrupt : Modified in 10.1.34 encryption.innodb_lotoftables : MDEV-16111 - Wrong result +encryption.innodb-missing-key : Modified in 10.1.34 encryption.innodb-page_encryption : MDEV-10641 - mutex problem -encryption.innodb-redo-badkey : MDEV-12898 - Server hang on startup -encryption.innodb-remove-encryption : Added in 10.1.33 +encryption.innodb-read-only : MDEV-14728 - Unable to get certificate +encryption.innodb-redo-badkey : MDEV-12898 - Server hang on startup; modified in 10.1.34 +encryption.innodb-redo-nokeys : Modified in 10.1.34 +encryption.innodb-remove-encryption : MDEV-16493 - Timeout in wait condition; a dded in 10.1.33 encryption.innodb_scrub : MDEV-8139 - scrubbing tests need fixing encryption.innodb_scrub_background : MDEV-8139 - scrubbing tests need fixing encryption.innodb_scrub_compressed : MDEV-8139 - scrubbing tests need fixing -encryption.tempfiles : Modified in 10.1.32 #---------------------------------------------------------------- @@ -202,56 +211,60 @@ galera.MW-328A : MDEV-13876 - Wrong result #---------------------------------------------------------------- +handler.ps : Added in 10.1.34 + +#---------------------------------------------------------------- + innodb.alter_partitioned_xa : Added in 10.1.33 innodb.binlog_consistent : MDEV-10618 - Server fails to start -innodb.default_row_format_alter : Added in 10.1.32 -innodb.default_row_format_compatibility : Added in 10.1.32 -innodb.default_row_format_create : Added in 10.1.32 innodb.doublewrite : MDEV-12905 - Lost connection to MySQL server -innodb.file_format_defaults : Added in 10.1.32 -innodb_fts.fulltext2 : MDEV-14727 - Long semaphore wait innodb.innodb-64k-crash : MDEV-13872 - Failure and crash on startup +innodb.innodb-alter : Added in 10.1.34 innodb.innodb-alter-debug : MDEV-13182 - InnoDB: adjusting FSP_SPACE_FLAGS innodb.innodb-alter-nullable : Modified in 10.1.33 innodb.innodb-alter-table : MDEV-10619 - Testcase timeout -innodb.innodb-blob : MDEV-12053 - Client crash +innodb.innodb-blob : MDEV-12053 - Client crash; modified in 10.1.34 innodb.innodb_bug14147491 : MDEV-11808 - wrong error codes -innodb.innodb_bug27216817 : Added in 10.1.33 innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan innodb.innodb_bug48024 : MDEV-14352 - Assertion failure +innodb.innodb_bug54044 : Modified in 10.1.34 +innodb.innodb_defragment_small : Modified in 10.1.34 innodb.innodb-fk : MDEV-13832 - Assertion failure on shutdown innodb.innodb_max_recordsize_64k : MDEV-15203 - wrong result +innodb.innodb-mdev7046 : Modified in 10.1.34 +innodb.innodb-page_compression_default : Modified in 10.1.34 innodb.innodb-page_compression_lzma : MDEV-14353 - wrong result on Fedora 25 +innodb.innodb-page_compression_snappy : Modified in 10.1.34 innodb.innodb-page_compression_zip : MDEV-10641 - mutex problem innodb.innodb_stats : MDEV-10682 - wrong result innodb.innodb_sys_semaphore_waits : MDEV-10331 - wrong result innodb.innodb_zip_innochecksum2 : MDEV-13882 - Warning: difficult to find free blocks +innodb.lock_deleted : Added in 10.1.34 innodb.log_file_size : MDEV-15668 - Not found pattern -innodb.mvcc : Added in 10.1.32 -innodb.read_only_recover_committed : Added in 10.1.32 -innodb.recovery_shutdown : MDEV-15671 - Warning: database page corruption; modified in 10.1.32 -innodb.restart : Added in 10.1.32 +innodb.recovery_shutdown : MDEV-15671 - Warning: database page corruption +innodb.rename_table : Added in 10.1.34 innodb.row_format_redundant : MDEV-15192 - Trying to access missing tablespace innodb.table_definition_cache_debug : MDEV-14206 - Extra warning innodb.table_flags : MDEV-14363 - Operating system error number 2 -innodb_fts.fulltext_misc : MDEV-12636 - Valgrind warnings +#---------------------------------------------------------------- -innodb_zip.innodb-create-options : Modified in 10.1.32 -innodb_zip.innodb-zip : Modified in 10.1.32 +innodb_fts.basic : Added in 10.1.34 #---------------------------------------------------------------- -maria.dynamic : Added in 10.1.32 +maria.alter : Modified in 10.1.34 maria.insert_select : MDEV-12757 - Timeout -maria.maria : MDEV-14430 - Wrong result +maria.insert_select-7314 : MDEV-16492 - Timeout +maria.lock : Modified in 10.1.34 +maria.maria : MDEV-14430 - Wrong result; modified in 10.1.34 #---------------------------------------------------------------- mariabackup.absolute_ibdata_paths : Added in 10.1.33 +mariabackup.backup_ssl : Added in 10.1.34 mariabackup.incremental_encrypted : MDEV-15667 - Timeout mariabackup.mdev-14447 : MDEV-15201 - Timeout -mariabackup.undo_space_id : Added in 10.1.32 mariabackup.xb_compressed_encrypted : MDEV-14812 - Segfault #---------------------------------------------------------------- @@ -282,8 +295,8 @@ parts.partition_alter_innodb : Include file modified in 10.1.33 parts.partition_alter_maria : Include file modified in 10.1.33 parts.partition_alter_myisam : Include file modified in 10.1.33 parts.partition_alter2_2_maria : MDEV-14364 - Lost connection to MySQL server during query +parts.partition_auto_increment_archive : MDEV-16491 - Table marked as crashed parts.partition_auto_increment_maria : MDEV-14430 - Wrong result -parts.partition_basic_symlink_innodb : Modified in 10.1.32 parts.partition_debug_innodb : MDEV-15095 - table does not exist parts.partition_innodb_status_file : MDEV-12901 - Valgrind @@ -293,8 +306,8 @@ perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.hostcache_ipv4_max_con : Modified in 10.1.33 perfschema.hostcache_ipv6_max_con : Modified in 10.1.33 +perfschema.partition : Added in 10.1.34 perfschema.privilege_table_io : MDEV-13184 - Extra lines -perfschema.setup_actors : MDEV-10679 - rare crash perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match perfschema.stage_mdl_global : MDEV-11803 - wrong result on slow builders perfschema.stage_mdl_table : MDEV-12638 - Wrong result @@ -303,12 +316,14 @@ perfschema.threads_mysql : MDEV-10677 - sporadic wrong resul #---------------------------------------------------------------- plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url -plugins.server_audit : MDEV-9562 - crashes on sol10-sparc; modified in 10.1.32 +plugins.processlist : Added in 10.1.34 +plugins.server_audit : MDEV-9562 - crashes on sol10-sparc; modified in 10.1.34 plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc #---------------------------------------------------------------- rpl.last_insert_id : MDEV-10625 - warnings in error log +rpl.rename : Added in 10.1.34 rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log @@ -331,15 +346,15 @@ rpl.rpl_insert_id : MDEV-15197 - Wrong result rpl.rpl_insert_ignore : MDEV-14365 - Lost connection to MySQL server during query rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips rpl.rpl_mariadb_slave_capability : MDEV-11018 - sporadic wrong events in binlog -rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips +rpl.rpl_mdev6020 : MDEV-10417 - Fails on Mips +rpl.rpl_mixed_implicit_commit_binlog : Modified in 10.1.34 rpl.rpl_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_non_direct_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed -rpl.rpl_non_direct_row_mixing_engines : MDEV-14491 - Long semaphore wait rpl.rpl_non_direct_stm_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_parallel : MDEV-10653 - Timeouts rpl.rpl_parallel_mdev6589 : MDEV-12979 - Assertion failure rpl.rpl_parallel_multilevel2 : MDEV-14723 - Timeout -rpl.rpl_parallel_optimistic : MDEV-10511 - Timeout +rpl.rpl_parallel_optimistic : MDEV-10511 - Timeout; modified in 10.1.34 rpl.rpl_parallel_retry : MDEV-11119 - Server crash rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips @@ -349,20 +364,20 @@ rpl.rpl_row_drop_create_temp_table : MDEV-14487 - Wrong result rpl.rpl_row_img_blobs : MDEV-13875 - command "diff_files" failed rpl.rpl_row_img_eng_min : MDEV-13875 - command "diff_files" failed rpl.rpl_row_img_eng_noblob : MDEV-13875 - command "diff_files" failed +rpl.rpl_row_implicit_commit_binlog : Modified in 10.1.34 rpl.rpl_row_index_choice : MDEV-15196 - Slave crash -rpl.rpl_row_mixing_engines : MDEV-14491 - Long semaphore wait rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_semi_sync : MDEV-11220 - Wrong result rpl.rpl_semi_sync_after_sync : MDEV-14366 - Wrong result rpl.rpl_semi_sync_after_sync_row : MDEV-14366 - Wrong result rpl.rpl_semi_sync_event_after_sync : MDEV-11806 - warnings -rpl.rpl_semi_sync_skip_repl : Added in 10.1.32 rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status rpl.rpl_semi_sync_wait_point : MDEV-11807 - timeout in wait condition rpl.rpl_show_slave_hosts : MDEV-10681 - server startup problem rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock rpl.rpl_start_stop_slave : MDEV-13567 - Replication failure +rpl.rpl_stm_implicit_commit_binlog : Modified in 10.1.34 rpl.rpl_stm_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_stm_relay_ign_space : MDEV-14360 - Test assertion rpl.rpl_sync : MDEV-10633 - Database page corruption @@ -373,16 +388,11 @@ rpl.sec_behind_master-5114 : MDEV-13878 - Wrong result spider.* : MDEV-9329 - tests are too memory-consuming -spider.spider_fixes : MDEV-12900 - Valgrind -spider.spider_fixes_part : MDEV-12900 - Valgrind - -spider/bg.basic_sql : MDEV-12900 - Valgrind spider/bg.direct_aggregate : MDEV-7098 - Trying to unlock mutex that wasn't locked spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't locked -spider/bg.function : MDEV-12900 - Valgrind -spider/bg.ha : MDEV-7914 - Crash (only fixed in 10.3), MDEV-9329 - failures on s390x -spider/bg.ha_part : MDEV-7914 - Crash (only fixed in 10.3), MDEV-9329 - Fails on Ubuntu/s390x -spider/bg.spider_fixes : MDEV-7098 -Mutex problem, MDEV-9329 - failures on s390x, MDEV-12900 - Valgrind +spider/bg.ha : MDEV-9329 - failures on s390x +spider/bg.ha_part : MDEV-9329 - Fails on Ubuntu/s390x +spider/bg.spider_fixes : MDEV-7098 -Mutex problem, MDEV-9329 - failures on s390x spider/bg.spider3_fixes : MDEV-12639 - Packets out of order spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x @@ -404,7 +414,6 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x sys_vars.keep_files_on_create_basic : MDEV-10676 - timeout -sys_vars.innodb_default_row_format_basic : Added in 10.1.32 sys_vars.log_slow_admin_statements_func : MDEV-12235 - Server crash sys_vars.max_prepared_stmt_count_basic : Modified in 10.1.33 sys_vars.rpl_init_slave_func : MDEV-10149 - wrong results @@ -413,7 +422,6 @@ sys_vars.sysvars_server_embedded : Opt file added in 10.1.33 sys_vars.sysvars_server_notembedded : Opt file added in 10.1.33 sys_vars.thread_cache_size_func : MDEV-11775 - Wrong result sys_vars.wait_timeout_func : MDEV-12896 - Wrong result -sys_vars.wsrep_sst_receive_address_basic : Modified in 10.1.32 #---------------------------------------------------------------- @@ -440,8 +448,6 @@ tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output tokudb_bugs.db917 : Modified in 10.1.33 tokudb_bugs.xa : MDEV-11804 - Lock wait timeout -tokudb_mariadb.mdev6657 : Modified in 10.1.32 - #---------------------------------------------------------------- unit.lf : MDEV-12897 - Signal 11 thrown @@ -459,5 +465,6 @@ wsrep.binlog_format : MDEV-11532 - WSREP has not yet prepared node wsrep.foreign_key : MDEV-14725 - WSREP has not yet prepared node wsrep.mdev_6832 : MDEV-14195 - Failure upon check-testcase wsrep.pool_of_threads : MDEV-12234 - Library problem on Power +wsrep.variables : Modified in 10.1.34 wsrep_info.plugin : MDEV-12909 - Wrong result From c69357d8d4592b746a2dd7074c8fb6a437098b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 14 Jun 2018 15:47:39 +0300 Subject: [PATCH 132/203] MDEV-15611 Due to the failure of foreign key detection, Galera slave node killed himself. Merge following change from 10.2 revision-id: d52cff9f10aeea208a1058f7b5527e602125584c (mariadb-10.2.14-25-gd52cff9) parent(s): bc2501453c3ab9a2cf3516bc3557de8665bc2776 author: Sachin Setiya committer: Sachin Setiya timestamp: 2018-04-04 12:26:06 +0530 message: MDEV-15611 Due to the failure of foreign key detection, Galera... slave node killed himself. Problem:- If we try to delete table with foreign key and table whom it is referring with wsrep_slave_threads>1 then galera tries to execute both Delete_rows_log-event in parallel, which should not happen. Solution:- This is happening because we do not have foreign key info in write set. Upto version 10.2.7 it used to work fine. Actually it happening because of issue in commit 2f342c4. wsrep_must_process_fk should be used with negation. --- .../suite/galera/r/galera_mdev_15611.result | 15 ++ .../suite/galera/t/galera_mdev_15611.cnf | 5 + .../suite/galera/t/galera_mdev_15611.test | 30 ++++ storage/innobase/row/row0upd.cc | 141 ++++++++++-------- storage/xtradb/row/row0upd.cc | 141 ++++++++++-------- 5 files changed, 200 insertions(+), 132 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_mdev_15611.result create mode 100644 mysql-test/suite/galera/t/galera_mdev_15611.cnf create mode 100644 mysql-test/suite/galera/t/galera_mdev_15611.test diff --git a/mysql-test/suite/galera/r/galera_mdev_15611.result b/mysql-test/suite/galera/r/galera_mdev_15611.result new file mode 100644 index 00000000000..677ed98202d --- /dev/null +++ b/mysql-test/suite/galera/r/galera_mdev_15611.result @@ -0,0 +1,15 @@ +CREATE TABLE t1 ( +id int primary key +); +CREATE TABLE t2 ( +id int primary key , +f_id int DEFAULT NULL, FOREIGN KEY(f_id) REFERENCES t1 (id) +); +insert into t1 select 1; +#Running 200 insert in t2 table +select count(*) from t2; +count(*) +200 +delete from t2; +delete from t1; +drop table t2,t1; diff --git a/mysql-test/suite/galera/t/galera_mdev_15611.cnf b/mysql-test/suite/galera/t/galera_mdev_15611.cnf new file mode 100644 index 00000000000..b6f601c56b1 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_mdev_15611.cnf @@ -0,0 +1,5 @@ +!include ../galera_2nodes.cnf +[mysqld.1] + +[mysqld.2] +wsrep_slave_threads=6 diff --git a/mysql-test/suite/galera/t/galera_mdev_15611.test b/mysql-test/suite/galera/t/galera_mdev_15611.test new file mode 100644 index 00000000000..d32d7e75262 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_mdev_15611.test @@ -0,0 +1,30 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +--connection node_1 +CREATE TABLE t1 ( + id int primary key +); + +CREATE TABLE t2 ( + id int primary key , + f_id int DEFAULT NULL, FOREIGN KEY(f_id) REFERENCES t1 (id) +); + +insert into t1 select 1; + +--disable_query_log +--let $count=200 +--echo #Running 200 insert in t2 table +while($count) +{ + #Repeatedly execute the following SQL until you generate thousands of data + --eval insert into t2 values ($count, 1); + --dec $count +} +--enable_query_log + +select count(*) from t2; +delete from t2; +delete from t1; +drop table t2,t1; diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index e2de47bf86a..33f46882651 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1803,6 +1803,23 @@ row_upd_store_row( } } +#ifdef WITH_WSREP +/** Determine if a FOREIGN KEY constraint needs to be processed. +@param[in] node query node +@param[in] trx transaction +@return whether the node cannot be ignored */ + +inline bool wsrep_must_process_fk(const upd_node_t* node, const trx_t* trx) +{ + if (!wsrep_on_trx(trx)) { + return false; + } + return que_node_get_type(node->common.parent) != QUE_NODE_UPDATE + || static_cast(node->common.parent)->cascade_node + != node; +} +#endif /* WITH_WSREP */ + /***********************************************************//** Updates a secondary index entry of a row. @return DB_SUCCESS if operation successfully completed, else error @@ -1833,7 +1850,7 @@ row_upd_sec_index_entry( referenced = row_upd_index_is_referenced(index, trx); #ifdef WITH_WSREP - ibool foreign = wsrep_row_upd_index_is_foreign(index, trx); + bool foreign = wsrep_row_upd_index_is_foreign(index, trx); #endif /* WITH_WSREP */ heap = mem_heap_create(1024); @@ -1962,61 +1979,61 @@ row_upd_sec_index_entry( row_ins_sec_index_entry() below */ if (!rec_get_deleted_flag( rec, dict_table_is_comp(index->table))) { -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ err = btr_cur_del_mark_set_sec_rec( 0, btr_cur, TRUE, thr, &mtr); - if (err == DB_SUCCESS && referenced) { - - ulint* offsets; - - offsets = rec_get_offsets( - rec, index, NULL, ULINT_UNDEFINED, - &heap); - - /* NOTE that the following call loses - the position of pcur ! */ - err = row_upd_check_references_constraints( - node, &pcur, index->table, - index, offsets, thr, &mtr); + if (err != DB_SUCCESS) { + break; } + #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && - wsrep_on_trx(trx) && - !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && - !(parent && que_node_get_type(parent) == - QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { - ulint* offsets = - rec_get_offsets( + if (!referenced && foreign + && wsrep_must_process_fk(node, trx) + && !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + ulint* offsets = rec_get_offsets( rec, index, NULL, ULINT_UNDEFINED, &heap); + err = wsrep_row_upd_check_foreign_constraints( node, &pcur, index->table, index, offsets, thr, &mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: sec index FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } } #endif /* WITH_WSREP */ } - break; + + if (referenced) { + + ulint* offsets; + + offsets = rec_get_offsets( + rec, index, NULL, ULINT_UNDEFINED, + &heap); + + /* NOTE that the following call loses + the position of pcur ! */ + err = row_upd_check_references_constraints( + node, &pcur, index->table, + index, offsets, thr, &mtr); + } } btr_pcur_close(&pcur); @@ -2185,9 +2202,6 @@ row_upd_clust_rec_by_insert( rec_t* rec; ulint* offsets = NULL; -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ ut_ad(node); ut_ad(dict_index_is_clust(index)); @@ -2269,35 +2283,31 @@ err_exit: if (err != DB_SUCCESS) { goto err_exit; } - } #ifdef WITH_WSREP - if (!referenced && wsrep_on_trx(trx) && - !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { + } else if ((foreign && wsrep_must_process_fk(node, trx))) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: insert FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } - if (err != DB_SUCCESS) { - goto err_exit; - } - } #endif /* WITH_WSREP */ + } } mtr_commit(mtr); @@ -2500,7 +2510,7 @@ row_upd_del_mark_clust_rec( dberr_t err; #ifdef WITH_WSREP rec_t* rec; - que_node_t *parent = que_node_get_parent(node); + trx_t* trx = thr_get_trx(thr) ; #endif /* WITH_WSREP */ ut_ad(node); @@ -2529,38 +2539,37 @@ row_upd_del_mark_clust_rec( btr_cur_get_block(btr_cur), btr_cur_get_rec(btr_cur), #endif /* WITH_WSREP */ index, offsets, thr, mtr); - if (err == DB_SUCCESS && referenced) { + if (err != DB_SUCCESS) { + } else if (referenced) { /* NOTE that the following call loses the position of pcur ! */ err = row_upd_check_references_constraints( node, pcur, index->table, index, offsets, thr, mtr); - } #ifdef WITH_WSREP - trx_t* trx = thr_get_trx(thr) ; - if (err == DB_SUCCESS && !referenced && wsrep_on_trx(trx) && - !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { + } else if (foreign && wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: clust rec FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: clust rec referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } - } #endif /* WITH_WSREP */ + } mtr_commit(mtr); diff --git a/storage/xtradb/row/row0upd.cc b/storage/xtradb/row/row0upd.cc index 9ac72f8d068..93ccc07c9af 100644 --- a/storage/xtradb/row/row0upd.cc +++ b/storage/xtradb/row/row0upd.cc @@ -1806,6 +1806,23 @@ row_upd_store_row( } } +#ifdef WITH_WSREP +/** Determine if a FOREIGN KEY constraint needs to be processed. +@param[in] node query node +@param[in] trx transaction +@return whether the node cannot be ignored */ + +inline bool wsrep_must_process_fk(const upd_node_t* node, const trx_t* trx) +{ + if (!wsrep_on_trx(trx)) { + return false; + } + return que_node_get_type(node->common.parent) != QUE_NODE_UPDATE + || static_cast(node->common.parent)->cascade_node + != node; +} +#endif /* WITH_WSREP */ + /***********************************************************//** Updates a secondary index entry of a row. @return DB_SUCCESS if operation successfully completed, else error @@ -1836,7 +1853,7 @@ row_upd_sec_index_entry( referenced = row_upd_index_is_referenced(index, trx); #ifdef WITH_WSREP - ibool foreign = wsrep_row_upd_index_is_foreign(index, trx); + bool foreign = wsrep_row_upd_index_is_foreign(index, trx); #endif /* WITH_WSREP */ heap = mem_heap_create(1024); @@ -1968,61 +1985,61 @@ row_upd_sec_index_entry( row_ins_sec_index_entry() below */ if (!rec_get_deleted_flag( rec, dict_table_is_comp(index->table))) { -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ err = btr_cur_del_mark_set_sec_rec( 0, btr_cur, TRUE, thr, &mtr); - if (err == DB_SUCCESS && referenced) { - - ulint* offsets; - - offsets = rec_get_offsets( - rec, index, NULL, ULINT_UNDEFINED, - &heap); - - /* NOTE that the following call loses - the position of pcur ! */ - err = row_upd_check_references_constraints( - node, &pcur, index->table, - index, offsets, thr, &mtr); + if (err != DB_SUCCESS) { + break; } + #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && - wsrep_on_trx(trx) && - !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && - !(parent && que_node_get_type(parent) == - QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { - ulint* offsets = - rec_get_offsets( + if (!referenced && foreign + && wsrep_must_process_fk(node, trx) + && !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + ulint* offsets = rec_get_offsets( rec, index, NULL, ULINT_UNDEFINED, &heap); + err = wsrep_row_upd_check_foreign_constraints( node, &pcur, index->table, index, offsets, thr, &mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: sec index FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } } #endif /* WITH_WSREP */ } - break; + + if (referenced) { + + ulint* offsets; + + offsets = rec_get_offsets( + rec, index, NULL, ULINT_UNDEFINED, + &heap); + + /* NOTE that the following call loses + the position of pcur ! */ + err = row_upd_check_references_constraints( + node, &pcur, index->table, + index, offsets, thr, &mtr); + } } btr_pcur_close(&pcur); @@ -2191,9 +2208,6 @@ row_upd_clust_rec_by_insert( rec_t* rec; ulint* offsets = NULL; -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ ut_ad(node); ut_ad(dict_index_is_clust(index)); @@ -2278,35 +2292,31 @@ err_exit: if (err != DB_SUCCESS) { goto err_exit; } - } #ifdef WITH_WSREP - if (!referenced && wsrep_on_trx(trx) && - !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { + } else if ((foreign && wsrep_must_process_fk(node, trx))) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: insert FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } - if (err != DB_SUCCESS) { - goto err_exit; - } - } #endif /* WITH_WSREP */ + } } mtr_commit(mtr); @@ -2512,7 +2522,7 @@ row_upd_del_mark_clust_rec( dberr_t err; #ifdef WITH_WSREP rec_t* rec; - que_node_t *parent = que_node_get_parent(node); + trx_t* trx = thr_get_trx(thr) ; #endif /* WITH_WSREP */ ut_ad(node); @@ -2541,38 +2551,37 @@ row_upd_del_mark_clust_rec( btr_cur_get_block(btr_cur), btr_cur_get_rec(btr_cur), #endif /* WITH_WSREP */ index, offsets, thr, mtr); - if (err == DB_SUCCESS && referenced) { + if (err != DB_SUCCESS) { + } else if (referenced) { /* NOTE that the following call loses the position of pcur ! */ err = row_upd_check_references_constraints( node, pcur, index->table, index, offsets, thr, mtr); - } #ifdef WITH_WSREP - trx_t* trx = thr_get_trx(thr) ; - if (err == DB_SUCCESS && !referenced && wsrep_on_trx(trx) && - !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { + } else if (foreign && wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: clust rec FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: clust rec referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } - } #endif /* WITH_WSREP */ + } mtr_commit(mtr); From 93ab0effd37c3a96ab41cba9afc899d966048f42 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 15 Jun 2018 10:14:18 +0200 Subject: [PATCH 133/203] MDEV-16187 Ubuntu Bionic MariaDB has epoch version that makes 10.1 and 10.2 installs fail backport cb21e117bab from 10.2 --- debian/autobake-deb.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 9de37138ade..648c95cb416 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -57,9 +57,12 @@ sed -i -e "s/\\\${LIBSSL}/${LIBSSL}/g" debian/control # echo "Incrementing changelog and starting build scripts" -dch -b -D ${CODENAME} -v "${UPSTREAM}${PATCHLEVEL}-${RELEASE_NAME}${RELEASE_EXTRA:+-${RELEASE_EXTRA}}1~${CODENAME}" "Automatic build with ${LOGSTRING}." +if [[ "$CODENAME" == bionic ]]; then + EPOCH="1:" +fi +dch -b -D ${CODENAME} -v "${EPOCH}${UPSTREAM}${PATCHLEVEL}-${RELEASE_NAME}${RELEASE_EXTRA:+-${RELEASE_EXTRA}}1~${CODENAME}" "Automatic build with ${LOGSTRING}." -echo "Creating package version ${UPSTREAM}${PATCHLEVEL}-${RELEASE_NAME}${RELEASE_EXTRA:+-${RELEASE_EXTRA}}1~${CODENAME} ... " +echo "Creating package version ${EPOCH}${UPSTREAM}${PATCHLEVEL}-${RELEASE_NAME}${RELEASE_EXTRA:+-${RELEASE_EXTRA}}1~${CODENAME} ... " # Build the package. # Pass -I so that .git and other unnecessary temporary and source control files From ff317fe08ea4d0e65ea421ef20786be0edf5423e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 15 Jun 2018 13:31:43 +0300 Subject: [PATCH 134/203] Follow-up to MDEV-16367 mariabackup: error: failed to copy enough redo log Commit dc9c555415b1d37b713839b9f8143a053f402c0e moved the final phase of the redo log copying to the background thread. This would sometimes cause too little redo log to be copied at the end of the backup. We would only guarantee copying up to the latest redo log checkpoint. This would produce a consistent backup, but it could refer to a too old point of time. xtrabackup_copy_log(), xtrabackup_copy_logfile(): Add the parameter 'last'. xtrabackup_backup_low(): Copy any remaining part of the log after the backup threads have terminated. --- extra/mariabackup/xtrabackup.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index bd5e28a4f5f..7a4442e6860 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2469,9 +2469,10 @@ skip: /** Copy redo log blocks to the data sink. @param start_lsn buffer start LSN @param end_lsn buffer end LSN +@param last whether we are copying the final part of the log @return last scanned LSN @retval 0 on failure */ -static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn) +static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last) { lsn_t scanned_lsn = start_lsn; const byte* log_block = log_sys->buf; @@ -2530,7 +2531,7 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn) log_sys->log.scanned_lsn = scanned_lsn; - end_lsn = metadata_to_lsn + end_lsn = last ? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE) : scanned_lsn & ~lsn_t(OS_FILE_LOG_BLOCK_SIZE - 1); @@ -2550,8 +2551,9 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn) } /** Copy redo log until the current end of the log is reached +@param last whether we are copying the final part of the log @return whether the operation failed */ -static bool xtrabackup_copy_logfile() +static bool xtrabackup_copy_logfile(bool last = false) { ut_a(dst_log_file != NULL); ut_ad(recv_sys != NULL); @@ -2582,7 +2584,7 @@ static bool xtrabackup_copy_logfile() } start_lsn = (lsn == start_lsn) - ? 0 : xtrabackup_copy_log(start_lsn, lsn); + ? 0 : xtrabackup_copy_log(start_lsn, lsn, last); log_mutex_exit(); @@ -3726,6 +3728,11 @@ static bool xtrabackup_backup_low() stop_backup_threads(); + if (metadata_to_lsn && xtrabackup_copy_logfile(true)) { + ds_close(dst_log_file); + return false; + } + if (ds_close(dst_log_file) || !metadata_to_lsn) { dst_log_file = NULL; return false; From b8514c94f61f37757a24df1c6b9a7dd530b3de12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 15 Jun 2018 14:09:15 +0300 Subject: [PATCH 135/203] MDEV-16496 Mariabackup: Implement --verbose option to instrument InnoDB log apply srv_print_verbose_log: Introduce the value 2 to refer to mariabackup --verbose. recv_recover_page(), recv_parse_log_recs(): Add output for mariabackup --verbose. --- extra/mariabackup/xtrabackup.cc | 9 ++++++-- .../mariabackup/apply-log-only-incr.test | 4 ++-- mysql-test/suite/mariabackup/mdev-14447.test | 4 ++-- storage/innobase/log/log0recv.cc | 22 +++++++++++++++++++ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 7a4442e6860..843a93cd251 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4,7 +4,7 @@ MariaBackup: hot backup tool for InnoDB Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. -(c) 2017, MariaDB Corporation. +(c) 2017, 2018, MariaDB Corporation. Portions written by Marko Mäkelä. This program is free software; you can redistribute it and/or modify @@ -109,6 +109,7 @@ int sys_var_init(); char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/"; char *xtrabackup_target_dir= xtrabackup_real_target_dir; static my_bool xtrabackup_version; +static my_bool verbose; my_bool xtrabackup_backup; my_bool xtrabackup_prepare; my_bool xtrabackup_copy_back; @@ -696,6 +697,9 @@ enum options_xtrabackup struct my_option xb_client_options[] = { + {"verbose", 'V', "display verbose output", + (G_PTR*) &verbose, (G_PTR*) &verbose, 0, GET_BOOL, NO_ARG, + FALSE, 0, 0, 0, 0, 0}, {"version", 'v', "print xtrabackup version information", (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -1756,7 +1760,7 @@ innodb_init_param(void) srv_max_n_open_files = (ulint) innobase_open_files; srv_innodb_status = (ibool) innobase_create_status_file; - srv_print_verbose_log = 1; + srv_print_verbose_log = verbose ? 2 : 1; /* Store the default charset-collation number of this MySQL installation */ @@ -3730,6 +3734,7 @@ static bool xtrabackup_backup_low() if (metadata_to_lsn && xtrabackup_copy_logfile(true)) { ds_close(dst_log_file); + dst_log_file = NULL; return false; } diff --git a/mysql-test/suite/mariabackup/apply-log-only-incr.test b/mysql-test/suite/mariabackup/apply-log-only-incr.test index 25cb3a82023..74ab680d840 100644 --- a/mysql-test/suite/mariabackup/apply-log-only-incr.test +++ b/mysql-test/suite/mariabackup/apply-log-only-incr.test @@ -36,7 +36,7 @@ connection default; --disable_result_log exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --ftwrl-wait-timeout=5 --ftwrl-wait-threshold=300 --ftwrl-wait-query-type=all --target-dir=$incremental_dir --incremental-basedir=$basedir ; -exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir ; +exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir ; --enable_result_log let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test; @@ -44,7 +44,7 @@ let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test; --source include/search_pattern_in_file.inc --echo # expect NOT FOUND -exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; +exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; --source include/search_pattern_in_file.inc --echo # expect NOT FOUND diff --git a/mysql-test/suite/mariabackup/mdev-14447.test b/mysql-test/suite/mariabackup/mdev-14447.test index 48f37646231..689b578f2ab 100644 --- a/mysql-test/suite/mariabackup/mdev-14447.test +++ b/mysql-test/suite/mariabackup/mdev-14447.test @@ -18,8 +18,8 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir --disable_result_log echo # Prepare full backup, apply incremental one; -exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir; -exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; +exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir; +exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; echo # Restore and check results; let $targetdir=$basedir; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index b7ab1311cb7..c8976b193e2 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1772,6 +1772,11 @@ recv_recover_page(bool just_read_in, buf_block_t* block) ut_ad(recv_needed_recovery); + if (UNIV_UNLIKELY(srv_print_verbose_log == 2)) { + fprintf(stderr, "Applying log to page %u:%u\n", + recv_addr->space, recv_addr->page_no); + } + DBUG_PRINT("ib_log", ("Applying log to page %u:%u", recv_addr->space, recv_addr->page_no)); @@ -1872,6 +1877,13 @@ recv_recover_page(bool just_read_in, buf_block_t* block) start_lsn = recv->start_lsn; } + if (UNIV_UNLIKELY(srv_print_verbose_log == 2)) { + fprintf(stderr, "apply " LSN_PF ":" + " %d len " ULINTPF " page %u:%u\n", + recv->start_lsn, recv->type, recv->len, + recv_addr->space, recv_addr->page_no); + } + DBUG_PRINT("ib_log", ("apply " LSN_PF ":" " %s len " ULINTPF " page %u:%u", @@ -2422,6 +2434,16 @@ loop: #endif lsn = mach_read_from_8(ptr + 1); + if (UNIV_UNLIKELY(srv_print_verbose_log == 2)) { + fprintf(stderr, + "MLOG_CHECKPOINT(" LSN_PF ") %s at " + LSN_PF "\n", lsn, + lsn != checkpoint_lsn ? "ignored" + : recv_sys->mlog_checkpoint_lsn + ? "reread" : "read", + recv_sys->recovered_lsn); + } + DBUG_PRINT("ib_log", ("MLOG_CHECKPOINT(" LSN_PF ") %s at " LSN_PF, From 7fdb7d4058e5a41687057a4b56c5a7698521ba99 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 15 Jun 2018 18:20:36 +0200 Subject: [PATCH 136/203] more sst test failures * xtrabackup no longer support --compact * wsrep_sst_mysqldump wasn't always using --default-file * wsrep_sst.cc overquoted --default-file for wsrep_sst_mysqldump also remove redundant lines config and test lines, compiler warnings, and mark big tests as big. --- .../r/galera_sst_mysqldump_with_key.result | 4 +-- .../galera/t/galera_sst_mariabackup.test | 1 + .../suite/galera/t/galera_sst_mysqldump.test | 1 + .../t/galera_sst_mysqldump_with_key.cnf | 4 --- .../t/galera_sst_mysqldump_with_key.test | 14 ++++---- .../suite/galera/t/galera_sst_rsync.test | 1 + .../t/galera_sst_xtrabackup-v2-options.cnf | 2 +- .../galera/t/galera_sst_xtrabackup-v2.test | 1 + scripts/wsrep_sst_mysqldump.sh | 13 ++++--- sql/wsrep_sst.cc | 34 +++++++++---------- 10 files changed, 36 insertions(+), 39 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result index 227e1c15444..074481afc2d 100644 --- a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result +++ b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result @@ -1,4 +1,6 @@ Setting SST method to mysqldump ... +call mtr.add_suppression("WSREP: wsrep_sst_method is set to 'mysqldump' yet mysqld bind_address is set to '127.0.0.1'"); +call mtr.add_suppression("Failed to load slave replication state from table mysql.gtid_slave_pos"); CREATE USER 'sst'; GRANT ALL PRIVILEGES ON *.* TO 'sst'; SET GLOBAL wsrep_sst_auth = 'sst:'; @@ -105,5 +107,3 @@ CALL mtr.add_suppression("Can't open and lock privilege tables"); CALL mtr.add_suppression("Info table is not ready to be used"); CALL mtr.add_suppression("Native table .* has the wrong structure"); DROP USER sslsst; -SET GLOBAL general_log = ON; -SET GLOBAL slow_query_log = ON; diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup.test b/mysql-test/suite/galera/t/galera_sst_mariabackup.test index 0e7ac487700..bcb9ade3a25 100644 --- a/mysql-test/suite/galera/t/galera_sst_mariabackup.test +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source include/have_innodb.inc --source include/have_mariabackup.inc diff --git a/mysql-test/suite/galera/t/galera_sst_mysqldump.test b/mysql-test/suite/galera/t/galera_sst_mysqldump.test index 390e9815b20..835fac94a68 100644 --- a/mysql-test/suite/galera/t/galera_sst_mysqldump.test +++ b/mysql-test/suite/galera/t/galera_sst_mysqldump.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source suite/galera/include/galera_sst_set_mysqldump.inc diff --git a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.cnf b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.cnf index e108484b248..44e5573b3e6 100644 --- a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.cnf +++ b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.cnf @@ -12,10 +12,6 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore [mysqld] wsrep_debug=ON -ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/galera-cert.pem -ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/galera-cert.pem -ssl-key=@ENV.MYSQL_TEST_DIR/std_data/galera-key.pem - [client] ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem diff --git a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test index c813e04169f..0dbc63b531c 100644 --- a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test +++ b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test @@ -5,9 +5,12 @@ --source include/big_test.inc --source include/galera_cluster.inc --source include/have_innodb.inc ---source include/have_openssl.inc +--source include/have_ssl_communication.inc --source suite/galera/include/galera_sst_set_mysqldump.inc +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc --connection node_1 CREATE USER sslsst; @@ -18,12 +21,7 @@ SET GLOBAL wsrep_sst_auth = 'sslsst:'; --source suite/galera/include/galera_st_disconnect_slave.inc +--source include/auto_increment_offset_restore.inc --source suite/galera/include/galera_sst_restore.inc + DROP USER sslsst; - ---connection node_2 -# We have to manually restore global_log and slow_query_log due to mysql-wsrep#108 -# Otherwise MTR's check_testcases complains - -SET GLOBAL general_log = ON; -SET GLOBAL slow_query_log = ON; diff --git a/mysql-test/suite/galera/t/galera_sst_rsync.test b/mysql-test/suite/galera/t/galera_sst_rsync.test index f796356cac7..5c08707e870 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --let $node_1=node_1 diff --git a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf index 1e29673c0ff..3abf2549aae 100644 --- a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf +++ b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf @@ -8,7 +8,7 @@ wsrep_debug=ON [xtrabackup] backup-locks close-files -compact +#compact - disabled in xtrabackup 2.4, https://bugs.launchpad.net/percona-xtrabackup/+bug/1192834/comments/29 # compression requires qpress from the Percona repositories # compress # compress-threads=2 diff --git a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test index f1fd0f3ddf3..c270e4d0b19 100644 --- a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test +++ b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source include/have_innodb.inc --source include/have_xtrabackup.inc diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index fe49279a680..faa3f10639b 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -108,11 +108,6 @@ then DROP PREPARE stmt;" fi -# Retrieve the donor's @@global.gtid_binlog_state. -GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" |\ -$MYSQL_CLIENT $AUTH -S$WSREP_SST_OPT_SOCKET --disable-reconnect --connect_timeout=10 |\ -tail -1 | awk -F ' ' '{ print $2 }') - MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\ "$AUTH -h${WSREP_SST_OPT_HOST_UNESCAPED} "\ "-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10" @@ -126,12 +121,16 @@ tail -1 | awk -F ' ' '{ print $2 }') SERVER_VERSION=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'version'" | $MYSQL |\ tail -1 | awk -F ' ' '{ print $2 }') +# Retrieve the donor's @@global.gtid_binlog_state. +GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" | $MYSQL |\ +tail -1 | awk -F ' ' '{ print $2 }') + RESET_MASTER="" SET_GTID_BINLOG_STATE="" SQL_LOG_BIN_OFF="" # Safety check -if [ ${SERVER_VERSION%%.*} != '5' ] +if [ "${SERVER_VERSION%%.*}" != '5' ] then # If binary logging is enabled on the joiner node, we need to copy donor's # gtid_binlog_state to joiner. In order to do that, a RESET MASTER must be @@ -146,7 +145,7 @@ then fi # NOTE: we don't use --routines here because we're dumping mysql.proc table -MYSQLDUMP="$MYSQLDUMP $AUTH -S$WSREP_SST_OPT_SOCKET \ +MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF $AUTH -S$WSREP_SST_OPT_SOCKET \ --add-drop-database --add-drop-table --skip-add-locks --create-options \ --disable-keys --extended-insert --skip-lock-tables --quick --set-charset \ --skip-comments --flush-privileges --all-databases --events" diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index 260755d08a8..4df969496bc 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -556,11 +556,11 @@ static ssize_t sst_prepare_other (const char* method, ret= snprintf (cmd_str(), cmd_len, "wsrep_sst_%s " - WSREP_SST_OPT_ROLE" 'joiner' " - WSREP_SST_OPT_ADDR" '%s' " - WSREP_SST_OPT_DATA" '%s' " + WSREP_SST_OPT_ROLE " 'joiner' " + WSREP_SST_OPT_ADDR " '%s' " + WSREP_SST_OPT_DATA " '%s' " " %s " - WSREP_SST_OPT_PARENT" '%d'" + WSREP_SST_OPT_PARENT " '%d'" " %s '%s' ", method, addr_in, mysql_real_data_home, wsrep_defaults_file, @@ -842,13 +842,13 @@ static int sst_donate_mysqldump (const char* addr, int ret= snprintf (cmd_str(), cmd_len, "wsrep_sst_mysqldump " - WSREP_SST_OPT_ADDR" '%s' " - WSREP_SST_OPT_PORT" '%d' " - WSREP_SST_OPT_LPORT" '%u' " - WSREP_SST_OPT_SOCKET" '%s' " - " '%s' " - WSREP_SST_OPT_GTID" '%s:%lld' " - WSREP_SST_OPT_GTID_DOMAIN_ID" '%d'" + WSREP_SST_OPT_ADDR " '%s' " + WSREP_SST_OPT_PORT " '%d' " + WSREP_SST_OPT_LPORT " '%u' " + WSREP_SST_OPT_SOCKET " '%s' " + " %s " + WSREP_SST_OPT_GTID " '%s:%lld' " + WSREP_SST_OPT_GTID_DOMAIN_ID " '%d'" "%s", addr, port, mysqld_port, mysqld_unix_port, wsrep_defaults_file, uuid_str, @@ -1205,14 +1205,14 @@ static int sst_donate_other (const char* method, ret= snprintf (cmd_str(), cmd_len, "wsrep_sst_%s " - WSREP_SST_OPT_ROLE" 'donor' " - WSREP_SST_OPT_ADDR" '%s' " - WSREP_SST_OPT_SOCKET" '%s' " - WSREP_SST_OPT_DATA" '%s' " + WSREP_SST_OPT_ROLE " 'donor' " + WSREP_SST_OPT_ADDR " '%s' " + WSREP_SST_OPT_SOCKET " '%s' " + WSREP_SST_OPT_DATA " '%s' " " %s " " %s '%s' " - WSREP_SST_OPT_GTID" '%s:%lld' " - WSREP_SST_OPT_GTID_DOMAIN_ID" '%d'" + WSREP_SST_OPT_GTID " '%s:%lld' " + WSREP_SST_OPT_GTID_DOMAIN_ID " '%d'" "%s", method, addr, mysqld_unix_port, mysql_real_data_home, wsrep_defaults_file, From 9f848da640dd6c3f44d56eae18204370ae7f835c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 16 Jun 2018 01:20:44 +0200 Subject: [PATCH 137/203] fix dependencies on bionic --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 84fe9e07f88..3048ec78d62 100644 --- a/debian/control +++ b/debian/control @@ -316,7 +316,7 @@ Depends: bsdutils, galera-3 (>=25.3), gawk, grep, - iproute, + iproute | iproute2, libdbi-perl, lsb-base (>= 3.0-10), lsof, From 352c7e0dfaa0f121c5b35e1d9fafb9ec8897e768 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Sun, 17 Jun 2018 17:15:21 +0400 Subject: [PATCH 138/203] MDEV-15905 select json_value('{"b":true}','$.b')=1 --> false with "Truncated incorrect DOUBLE value: 'true'". JSON_VALUE_TRUE and JSON_VALUE_FALSE should be handled specifically in Item_json_value. --- mysql-test/r/func_json.result | 3 +++ mysql-test/t/func_json.test | 7 +++++++ sql/item_jsonfunc.cc | 21 ++++++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/func_json.result b/mysql-test/r/func_json.result index 2ffeb83b0de..77a424171b1 100644 --- a/mysql-test/r/func_json.result +++ b/mysql-test/r/func_json.result @@ -739,3 +739,6 @@ drop table t1; select json_extract('{"test":8.437e-5}','$.test'); json_extract('{"test":8.437e-5}','$.test') 8.437e-5 +select json_value('{"b":true}','$.b')=1; +json_value('{"b":true}','$.b')=1 +1 diff --git a/mysql-test/t/func_json.test b/mysql-test/t/func_json.test index a6ae934f7b4..b74915cd064 100644 --- a/mysql-test/t/func_json.test +++ b/mysql-test/t/func_json.test @@ -398,3 +398,10 @@ drop table t1; select json_extract('{"test":8.437e-5}','$.test'); +# +# MDEV-15905 select json_value('{"b":true}','$.b')=1 --> false with +# "Truncated incorrect DOUBLE value: 'true'" +# +select json_value('{"b":true}','$.b')=1; + + diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 72aeac5b930..c8cda66580c 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -513,6 +513,10 @@ err_return: bool Item_func_json_value::check_and_get_value(json_engine_t *je, String *res, int *error) { + CHARSET_INFO *json_cs; + const uchar *js; + uint js_len; + if (!json_value_scalar(je)) { /* We only look for scalar values! */ @@ -521,7 +525,22 @@ bool Item_func_json_value::check_and_get_value(json_engine_t *je, String *res, return true; } - return st_append_json(res, je->s.cs, je->value, je->value_len); + if (je->value_type == JSON_VALUE_TRUE || + je->value_type == JSON_VALUE_FALSE) + { + json_cs= &my_charset_utf8mb4_bin; + js= (const uchar *) ((je->value_type == JSON_VALUE_TRUE) ? "1" : "0"); + js_len= 1; + } + else + { + json_cs= je->s.cs; + js= je->value; + js_len= je->value_len; + } + + + return st_append_json(res, json_cs, js, js_len); } From eb77f8cf8de44ecaa7155afa7f55ece73b2b0497 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 18 Jun 2018 14:26:37 +0530 Subject: [PATCH 139/203] MDEV-16087 Inconsistent SELECT results when query cache is enabled The following conditions will decide the query cache retrieval or storing inside innodb: (1) There should not be any locks on the table. (2) Some other trx shouldn't invalidated the cache before the transaction started. (3) Read view shouldn't exist. If exists then the view low_limit_id should be greater than or equal to the transaction that invalidates the cache for the particular table. For read-only transaction: should satisfy the above (1) and (3) For read-write transaction: should satisfy the above (1), (2), (3). - Changed the variable from query_cache_inv_id to query_cache_inv_trx_id. - Moved the function row_search_check_if_query_cache_permitted from row0sel.h and made it as static function in ha_innodb.cc --- .../suite/innodb/r/innodb_query_cache.result | 39 +++++++++ .../suite/innodb/t/innodb_query_cache.test | 47 +++++++++++ storage/innobase/handler/ha_innodb.cc | 79 ++++++++++++++++++- storage/innobase/include/dict0mem.h | 4 +- storage/innobase/include/row0sel.h | 10 --- storage/innobase/lock/lock0lock.cc | 2 +- storage/innobase/row/row0sel.cc | 54 ------------- 7 files changed, 167 insertions(+), 68 deletions(-) create mode 100644 mysql-test/suite/innodb/r/innodb_query_cache.result create mode 100644 mysql-test/suite/innodb/t/innodb_query_cache.test diff --git a/mysql-test/suite/innodb/r/innodb_query_cache.result b/mysql-test/suite/innodb/r/innodb_query_cache.result new file mode 100644 index 00000000000..bbacbdfd748 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb_query_cache.result @@ -0,0 +1,39 @@ +# +# MDEV-16087 Inconsistent SELECT results when query cache is enabled +# +set GLOBAL query_cache_type=ON; +set LOCAL query_cache_type=ON; +create table t1 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +create table t2 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +create table t3 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +connect con1,localhost,root; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +select * from t2; +id +connection default; +insert into t3 () values (); +connection con1; +insert into t1 () values (); +select * from t3; +id +connect con2,localhost,root; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +select * from t3; +id +1 +select * from t3; +id +1 +select sql_no_cache * from t3; +id +1 +rollback; +connection con1; +rollback; +disconnect con1; +disconnect con2; +connection default; +drop table t1; +drop table t2; +drop table t3; +set GLOBAL query_cache_type=default; diff --git a/mysql-test/suite/innodb/t/innodb_query_cache.test b/mysql-test/suite/innodb/t/innodb_query_cache.test new file mode 100644 index 00000000000..7dbdf986946 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb_query_cache.test @@ -0,0 +1,47 @@ +-- source include/have_innodb.inc +-- source include/have_query_cache.inc +-- source include/not_embedded.inc + +--echo # +--echo # MDEV-16087 Inconsistent SELECT results when query cache is enabled +--echo # + +set GLOBAL query_cache_type=ON; +set LOCAL query_cache_type=ON; + +create table t1 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +create table t2 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +create table t3 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; + +connect (con1,localhost,root); +START TRANSACTION WITH CONSISTENT SNAPSHOT; +select * from t2; + +connection default; +insert into t3 () values (); + +connection con1; +insert into t1 () values (); +select * from t3; + +connect (con2,localhost,root); +START TRANSACTION WITH CONSISTENT SNAPSHOT; +select * from t3; +select * from t3; +select sql_no_cache * from t3; + +rollback; + +connection con1; + +rollback; + +disconnect con1; +disconnect con2; +connection default; + +drop table t1; +drop table t2; +drop table t3; + +set GLOBAL query_cache_type=default; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index ba7a476c754..2c8ea81e286 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3167,6 +3167,83 @@ AUTOCOMMIT==0 or we are inside BEGIN ... COMMIT. Thus transactions no longer put restrictions on the use of the query cache. */ +/** Check if mysql can allow the transaction to read from/store to +the query cache. +@param[in] table table object +@param[in] trx transaction object +@return whether the storing or retrieving from the query cache is permitted */ +static bool innobase_query_caching_table_check_low( + const dict_table_t* table, + trx_t* trx) +{ + /* The following conditions will decide the query cache + retrieval or storing into: + + (1) There should not be any locks on the table. + (2) Someother trx shouldn't invalidate the cache before this + transaction started. + (3) Read view shouldn't exist. If exists then the view + low_limit_id should be greater than or equal to the transaction that + invalidates the cache for the particular table. + + For read-only transaction: should satisfy (1) and (3) + For read-write transaction: should satisfy (1), (2), (3) */ + + if (lock_table_get_n_locks(table)) { + return false; + } + + if (trx->id && trx->id < table->query_cache_inv_trx_id) { + return false; + } + + return !MVCC::is_view_active(trx->read_view) + || trx->read_view->low_limit_id() + >= table->query_cache_inv_trx_id; +} + +/** Checks if MySQL at the moment is allowed for this table to retrieve a +consistent read result, or store it to the query cache. +@param[in,out] trx transaction +@param[in] norm_name concatenation of database name, + '/' char, table name +@return whether storing or retrieving from the query cache is permitted */ +static bool innobase_query_caching_table_check( + trx_t* trx, + const char* norm_name) +{ + dict_table_t* table = dict_table_open_on_name( + norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); + + if (table == NULL) { + return false; + } + + /* Start the transaction if it is not started yet */ + trx_start_if_not_started(trx, false); + + bool allow = innobase_query_caching_table_check_low(table, trx); + + dict_table_close(table, FALSE, FALSE); + + if (allow) { + /* If the isolation level is high, assign a read view for the + transaction if it does not yet have one */ + + if (trx->isolation_level >= TRX_ISO_REPEATABLE_READ + && !srv_read_only_mode + && !MVCC::is_view_active(trx->read_view)) { + + /* Start the transaction if it is not started yet */ + trx_start_if_not_started(trx, false); + + trx_sys->mvcc->view_open(trx->read_view, trx); + } + } + + return allow; +} + /******************************************************************//** The MySQL query cache uses this to check from InnoDB if the query cache at the moment is allowed to operate on an InnoDB table. The SQL query must @@ -3242,7 +3319,7 @@ innobase_query_caching_of_table_permitted( innobase_register_trx(innodb_hton_ptr, thd, trx); - return(row_search_check_if_query_cache_permitted(trx, norm_name)); + return innobase_query_caching_table_check(trx, norm_name); } /*****************************************************************//** diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index a18d6c1abdd..56f695f305d 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1546,8 +1546,8 @@ struct dict_table_t { /** Transactions whose view low limit is greater than this number are not allowed to store to the MySQL query cache or retrieve from it. When a trx with undo logs commits, it sets this to the value of the - current time. */ - trx_id_t query_cache_inv_id; + transaction id. */ + trx_id_t query_cache_inv_trx_id; /** Transaction id that last touched the table definition. Either when loading the definition or CREATE TABLE, or ALTER TABLE (prepare, diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index d73c186b12e..552680b16d1 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -213,16 +213,6 @@ row_count_rtree_recs( ulint* n_rows); /*!< out: number of entries seen in the consistent read */ -/*******************************************************************//** -Checks if MySQL at the moment is allowed for this table to retrieve a -consistent read result, or store it to the query cache. -@return whether storing or retrieving from the query cache is permitted */ -bool -row_search_check_if_query_cache_permitted( -/*======================================*/ - trx_t* trx, /*!< in: transaction object */ - const char* norm_name); /*!< in: concatenation of database name, - '/' char, table name */ /** Read the max AUTOINC value from an index. @param[in] index index starting with an AUTO_INCREMENT column @return the largest AUTO_INCREMENT value diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 240510ffa59..ee0dc5a9000 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -4456,7 +4456,7 @@ lock_release( block the use of the MySQL query cache for all currently active transactions. */ - table->query_cache_inv_id = max_trx_id; + table->query_cache_inv_trx_id = max_trx_id; } lock_table_dequeue(lock); diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index b5e1bab352d..7383d3d9510 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -5905,60 +5905,6 @@ func_exit: goto loop; } -/*******************************************************************//** -Checks if MySQL at the moment is allowed for this table to retrieve a -consistent read result, or store it to the query cache. -@return whether storing or retrieving from the query cache is permitted */ -bool -row_search_check_if_query_cache_permitted( -/*======================================*/ - trx_t* trx, /*!< in: transaction object */ - const char* norm_name) /*!< in: concatenation of database name, - '/' char, table name */ -{ - dict_table_t* table = dict_table_open_on_name( - norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); - - if (table == NULL) { - - return(false); - } - - /* Start the transaction if it is not started yet */ - - trx_start_if_not_started(trx, false); - - /* If there are locks on the table or some trx has invalidated the - cache before this transaction started then this transaction cannot - read/write from/to the cache. - - If a read view has not been created for the transaction then it doesn't - really matter what this transaction sees. If a read view was created - then the view low_limit_id is the max trx id that this transaction - saw at the time of the read view creation. */ - - const bool ret = lock_table_get_n_locks(table) == 0 - && ((trx->id != 0 && trx->id >= table->query_cache_inv_id) - || !MVCC::is_view_active(trx->read_view) - || trx->read_view->low_limit_id() - >= table->query_cache_inv_id); - if (ret) { - /* If the isolation level is high, assign a read view for the - transaction if it does not yet have one */ - - if (trx->isolation_level >= TRX_ISO_REPEATABLE_READ - && !srv_read_only_mode - && !MVCC::is_view_active(trx->read_view)) { - - trx_sys->mvcc->view_open(trx->read_view, trx); - } - } - - dict_table_close(table, FALSE, FALSE); - - return(ret); -} - /*******************************************************************//** Read the AUTOINC column from the current row. If the value is less than 0 and the type is not unsigned then we reset the value to 0. From f7b1b2bc5d6e1f923aca0ee8c5232cf095075c7b Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Mon, 18 Jun 2018 07:40:58 -0400 Subject: [PATCH 140/203] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f9ccf7c6188..c169837c460 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=1 -MYSQL_VERSION_PATCH=34 +MYSQL_VERSION_PATCH=35 From c69efab3963703688486287918dda105942d24e7 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 18 Jun 2018 17:54:30 +0200 Subject: [PATCH 141/203] - Fix MDEV-16167 Cannot insert unsigned values into a VEC table modified: storage/connect/filamvct.cpp modified: storage/connect/tabvct.cpp _ Typo modified: storage/connect/CMakeLists.txt --- storage/connect/CMakeLists.txt | 17 ----------------- storage/connect/filamvct.cpp | 22 ++++++++++++++-------- storage/connect/tabvct.cpp | 6 ++---- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 58d9ab1f1c3..408f6d223a1 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -346,23 +346,6 @@ IF(WIN32) DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) ENDIF(WIN32) -IF(NOT TARGET connect) - RETURN() -ENDIF() - -# Install some extra files that belong to connect engine -IF(WIN32) - # install ha_connect.lib - GET_TARGET_PROPERTY(CONNECT_LOCATION connect LOCATION) - STRING(REPLACE "dll" "lib" CONNECT_LIB ${CONNECT_LOCATION}) - IF(CMAKE_CONFIGURATION_TYPES) - STRING(REPLACE "${CMAKE_CFG_INTDIR}" "\${CMAKE_INSTALL_CONFIG_NAME}" - CONNECT_LIB ${CONNECT_LIB}) - ENDIF() - INSTALL(FILES ${CONNECT_LIB} - DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) -ENDIF(WIN32) - IF(CONNECT_WITH_JDBC AND JAVA_FOUND AND JNI_FOUND) # TODO: Find how to compile and install the java wrapper classes # Find required libraries and include directories diff --git a/storage/connect/filamvct.cpp b/storage/connect/filamvct.cpp index 244acfdc5c8..a660461e9ee 100755 --- a/storage/connect/filamvct.cpp +++ b/storage/connect/filamvct.cpp @@ -515,7 +515,8 @@ bool VCTFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) cp->Blk = AllocValBlock(g, NewBlock + Nrec * cp->Deplac, cp->Buf_Type, Nrec, cp->Format.Length, - cp->Format.Prec, chk); + cp->Format.Prec, chk, true, + cp->IsUnsigned()); return InitInsert(g); // Initialize inserting } else { @@ -549,7 +550,8 @@ bool VCTFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) // Not a pseudo column cp->Blk = AllocValBlock(g, NULL, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); } //endif mode @@ -1516,7 +1518,8 @@ bool VCMFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)Tdbp->GetColumns(); cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) { // Not a pseudo column cp->Blk = AllocValBlock(g, (void*)1, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); cp->AddStatus(BUF_MAPPED); } // endif IsSpecial @@ -2067,7 +2070,7 @@ bool VECFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)tdbp->Columns; cp; cp = (PVCTCOL)cp->Next) cp->Blk = AllocValBlock(g, To_Bufs[cp->Index - 1], cp->Buf_Type, Nrec, cp->Format.Length, - cp->Format.Prec, chk); + cp->Format.Prec, chk, true, cp->IsUnsigned()); return InitInsert(g); } else { @@ -2116,7 +2119,8 @@ bool VECFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)tdbp->Columns; cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) // Not a pseudo column cp->Blk = AllocValBlock(g, NULL, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); } // endif mode @@ -2887,7 +2891,8 @@ bool VMPFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)Tdbp->GetColumns(); cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) { // Not a pseudo column cp->Blk = AllocValBlock(g, (void*)1, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); cp->AddStatus(BUF_MAPPED); } // endif IsSpecial @@ -3669,7 +3674,7 @@ bool BGVFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) cp->Blk = AllocValBlock(g, NewBlock + Nrec * cp->Deplac, cp->Buf_Type, Nrec, cp->Format.Length, - cp->Format.Prec, chk); + cp->Format.Prec, chk, true, cp->IsUnsigned()); InitInsert(g); // Initialize inserting @@ -3717,7 +3722,8 @@ bool BGVFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) // Not a pseudo column cp->Blk = AllocValBlock(g, NULL, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); } //endif mode diff --git a/storage/connect/tabvct.cpp b/storage/connect/tabvct.cpp index 11b344ef652..40d020202ea 100644 --- a/storage/connect/tabvct.cpp +++ b/storage/connect/tabvct.cpp @@ -456,13 +456,11 @@ bool VCTCOL::SetBuffer(PGLOBAL g, PVAL value, bool ok, bool check) if (tdbp->Txfp->GetAmType() == TYPE_AM_VMP && ok) { Blk = AllocValBlock(g, (void*)1, Buf_Type, tdbp->Txfp->Nrec, - Format.Length, - Format.Prec, check); + Format.Length, Format.Prec, check, true, Unsigned); Status |= BUF_MAPPED; // Will point into mapped file } else Blk = AllocValBlock(g, NULL, Buf_Type, tdbp->Txfp->Nrec, - Format.Length, - Format.Prec, check); + Format.Length, Format.Prec, check, true, Unsigned); } // endif Mode return false; From 5ba6cee01255186ea77a69f122d3a15c07f44f6d Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 18 Jun 2018 23:00:34 +0400 Subject: [PATCH 142/203] MDEV-16209 JSON_EXTRACT in query crashes server. The optimizer can create various item's over the original one, so we can't count on the exact item's type inside the comparison. --- mysql-test/r/func_json.result | 5 +++ mysql-test/t/func_json.test | 8 +++++ sql/item_jsonfunc.cc | 58 ++++++++++++++++++++--------------- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/mysql-test/r/func_json.result b/mysql-test/r/func_json.result index 77a424171b1..564442a2880 100644 --- a/mysql-test/r/func_json.result +++ b/mysql-test/r/func_json.result @@ -742,3 +742,8 @@ json_extract('{"test":8.437e-5}','$.test') select json_value('{"b":true}','$.b')=1; json_value('{"b":true}','$.b')=1 1 +CREATE TABLE t1 (c VARCHAR(8)); +INSERT INTO t1 VALUES ('foo'),('bar'); +SELECT * FROM t1 WHERE c IN (JSON_EXTRACT('{"a":"b"}', '$.*')); +c +DROP TABLE t1; diff --git a/mysql-test/t/func_json.test b/mysql-test/t/func_json.test index b74915cd064..1b91061b9a3 100644 --- a/mysql-test/t/func_json.test +++ b/mysql-test/t/func_json.test @@ -404,4 +404,12 @@ select json_extract('{"test":8.437e-5}','$.test'); # select json_value('{"b":true}','$.b')=1; +# +# MDEV-16209 JSON_EXTRACT in query crashes server. +# + +CREATE TABLE t1 (c VARCHAR(8)); +INSERT INTO t1 VALUES ('foo'),('bar'); +SELECT * FROM t1 WHERE c IN (JSON_EXTRACT('{"a":"b"}', '$.*')); +DROP TABLE t1; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index c8cda66580c..8103aa906ff 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -3229,34 +3229,44 @@ String *Item_func_json_format::val_json(String *str) int Arg_comparator::compare_json_str_basic(Item *j, Item *s) { - String *res1,*res2; - json_value_types type; - char *value; - int value_len, c_len; - Item_func_json_extract *e= (Item_func_json_extract *) j; + String *js,*str; + int c_len; + json_engine_t je; - if ((res1= e->read_json(&value1, &type, &value, &value_len))) + if ((js= j->val_str(&value1))) { - if ((res2= s->val_str(&value2))) - { - if (type == JSON_VALUE_STRING) - { - if (value1.realloc_with_extra_if_needed(value_len) || - (c_len= json_unescape(value1.charset(), (uchar *) value, - (uchar *) value+value_len, - &my_charset_utf8_general_ci, - (uchar *) value1.ptr(), - (uchar *) (value1.ptr() + value_len))) < 0) - goto error; - value1.length(c_len); - res1= &value1; - } + json_scan_start(&je, js->charset(), (const uchar *) js->ptr(), + (const uchar *) js->ptr()+js->length()); + if (json_read_value(&je)) + goto error; + if (je.value_type == JSON_VALUE_STRING) + { + if (value2.realloc_with_extra_if_needed(je.value_len) || + (c_len= json_unescape(js->charset(), je.value, + je.value + je.value_len, + &my_charset_utf8_general_ci, + (uchar *) value2.ptr(), + (uchar *) (value2.ptr() + je.value_len))) < 0) + goto error; - if (set_null) - owner->null_value= 0; - return sortcmp(res1, res2, compare_collation()); - } + value2.length(c_len); + js= &value2; + str= &value1; + } + else + { + str= &value2; + } + + + if ((str= s->val_str(str))) + { + if (set_null) + owner->null_value= 0; + return sortcmp(js, str, compare_collation()); + } } + error: if (set_null) owner->null_value= 1; From 15b92915ed93661a56f40430204d18bf7b7cf1fc Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 19 Jun 2018 13:02:02 +0400 Subject: [PATCH 143/203] MDEV-15834 The code in TABLE_SHARE::init_from_binary_frm_image() is not safe --- mysql-test/std_data/frm/t1.frm | Bin 0 -> 8584 bytes mysql-test/suite/vcol/r/vcol_misc.result | 10 ++++++++++ mysql-test/suite/vcol/t/vcol_misc.test | 16 ++++++++++++++++ sql/table.cc | 6 +++++- 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 mysql-test/std_data/frm/t1.frm diff --git a/mysql-test/std_data/frm/t1.frm b/mysql-test/std_data/frm/t1.frm new file mode 100644 index 0000000000000000000000000000000000000000..a998f54ec67d650e93535a513951727f378b10de GIT binary patch literal 8584 zcmeI&p$@_@5XSMZS0FSq2!g?22?DSLgGY2`^tUD!}Vsh{VbpE2@OU-E;mST3bmJ0T;000IagfB*srAb(Z<0m=97W1MYHjI_F Date: Sat, 16 Jun 2018 12:03:15 +0300 Subject: [PATCH 144/203] Add PART_INDIRECT_KEY_FLAG This is to mark that a field is indirectly part of a key, which simplifes checking if we need to have this field up to date to evaluate a key. For example: CREATE TABLE t1 (a int, b int as (a) virtual, c int as (b) virtual, index(c)) would mark a and b with PART_INDIRECT_KEY_FLAG. c is marked with PART_KEY_FLAG as before. --- include/mysql_com.h | 1 + mysql-test/suite/vcol/r/index.result | 38 ++++++++++++++ mysql-test/suite/vcol/t/index.test | 25 ++++++++++ sql/sql_union.cc | 6 +-- sql/table.cc | 75 +++++++++++++++++++++------- sql/table.h | 2 +- 6 files changed, 126 insertions(+), 21 deletions(-) create mode 100644 mysql-test/suite/vcol/r/index.result create mode 100644 mysql-test/suite/vcol/t/index.test diff --git a/include/mysql_com.h b/include/mysql_com.h index 06c934bf9bd..17366e22da8 100644 --- a/include/mysql_com.h +++ b/include/mysql_com.h @@ -179,6 +179,7 @@ enum enum_indicator_type #define BINCMP_FLAG 131072U /* Intern: Used by sql_yacc */ #define GET_FIXED_FIELDS_FLAG (1U << 18) /* Used to get fields in item tree */ #define FIELD_IN_PART_FUNC_FLAG (1U << 19)/* Field part of partition func */ +#define PART_INDIRECT_KEY_FLAG (1U << 20) /** Intern: Field in TABLE object for new version of altered table, diff --git a/mysql-test/suite/vcol/r/index.result b/mysql-test/suite/vcol/r/index.result new file mode 100644 index 00000000000..cd4ebc96024 --- /dev/null +++ b/mysql-test/suite/vcol/r/index.result @@ -0,0 +1,38 @@ +CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=myisam; +insert into t1 (a) values (1),(2),(3); +update t1 set a=5 where a=3; +delete from t1 where a=1; +select * from t1; +a b c +2 3 4 +5 6 7 +select * from t1 where c=7; +a b c +5 6 7 +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select * from t1; +a b c +2 3 4 +5 6 7 +drop table t1; +CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=innodb; +insert into t1 (a) values (1),(2),(3); +update t1 set a=5 where a=3; +delete from t1 where a=1; +select * from t1; +a b c +2 3 4 +5 6 7 +select * from t1 where c=7; +a b c +5 6 7 +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select * from t1; +a b c +2 3 4 +5 6 7 +drop table t1; diff --git a/mysql-test/suite/vcol/t/index.test b/mysql-test/suite/vcol/t/index.test new file mode 100644 index 00000000000..55d5b68f26b --- /dev/null +++ b/mysql-test/suite/vcol/t/index.test @@ -0,0 +1,25 @@ +--source include/have_innodb.inc + +# +# Test creating table with a key that consists of indirect virtual fields +# + +CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=myisam; +insert into t1 (a) values (1),(2),(3); +update t1 set a=5 where a=3; +delete from t1 where a=1; +select * from t1; +select * from t1 where c=7; +check table t1; +select * from t1; +drop table t1; + +CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=innodb; +insert into t1 (a) values (1),(2),(3); +update t1 set a=5 where a=3; +delete from t1 where a=1; +select * from t1; +select * from t1 where c=7; +check table t1; +select * from t1; +drop table t1; diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 178d7393878..b409790c044 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -185,7 +185,7 @@ select_union::create_result_table(THD *thd_arg, List *column_types, table->keys_in_use_for_query.clear_all(); for (uint i=0; i < table->s->fields; i++) - table->field[i]->flags &= ~PART_KEY_FLAG; + table->field[i]->flags &= ~(PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG); if (create_table) { @@ -219,7 +219,7 @@ select_union_recursive::create_result_table(THD *thd_arg, incr_table->keys_in_use_for_query.clear_all(); for (uint i=0; i < table->s->fields; i++) - incr_table->field[i]->flags &= ~PART_KEY_FLAG; + incr_table->field[i]->flags &= ~(PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG); TABLE *rec_table= 0; if (! (rec_table= create_tmp_table(thd_arg, &tmp_table_param, *column_types, @@ -230,7 +230,7 @@ select_union_recursive::create_result_table(THD *thd_arg, rec_table->keys_in_use_for_query.clear_all(); for (uint i=0; i < table->s->fields; i++) - rec_table->field[i]->flags &= ~PART_KEY_FLAG; + rec_table->field[i]->flags &= ~(PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG); if (rec_tables.push_back(rec_table)) return true; diff --git a/sql/table.cc b/sql/table.cc index 12fc10c3259..198e77f1872 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3357,7 +3357,7 @@ partititon_err: } } - outparam->mark_columns_used_by_check_constraints(); + outparam->mark_columns_used_by_virtual_fields(); if (share->table_category == TABLE_CATEGORY_LOG) { @@ -6293,11 +6293,12 @@ void TABLE::mark_columns_needed_for_delete() Field **reg_field; for (reg_field= field ; *reg_field ; reg_field++) { - if ((*reg_field)->flags & PART_KEY_FLAG) + Field *cur_field= *reg_field; + if (cur_field->flags & (PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG)) { - bitmap_set_bit(read_set, (*reg_field)->field_index); - if ((*reg_field)->vcol_info) - mark_virtual_col(*reg_field); + bitmap_set_bit(read_set, cur_field->field_index); + if (cur_field->vcol_info) + bitmap_set_bit(vcol_set, cur_field->field_index); } } need_signal= true; @@ -6655,7 +6656,8 @@ bool TABLE::mark_virtual_columns_for_write(bool insert_fl if (bitmap_is_set(write_set, tmp_vfield->field_index)) bitmap_updated|= mark_virtual_col(tmp_vfield); else if (tmp_vfield->vcol_info->stored_in_db || - (tmp_vfield->flags & (PART_KEY_FLAG | FIELD_IN_PART_FUNC_FLAG))) + (tmp_vfield->flags & (PART_KEY_FLAG | FIELD_IN_PART_FUNC_FLAG | + PART_INDIRECT_KEY_FLAG))) { bitmap_set_bit(write_set, tmp_vfield->field_index); mark_virtual_col(tmp_vfield); @@ -6668,27 +6670,64 @@ bool TABLE::mark_virtual_columns_for_write(bool insert_fl } /* - Mark fields used by check constraints. + Mark fields used by check constraints into s->check_set. + Mark all fields used in an expression that is part of an index + with PART_INDIRECT_KEY_FLAG + This is done once for the TABLE_SHARE the first time the table is opened. The marking must be done non-destructively to handle the case when this could be run in parallely by two threads */ -void TABLE::mark_columns_used_by_check_constraints(void) +void TABLE::mark_columns_used_by_virtual_fields(void) { MY_BITMAP *save_read_set; - /* If there is no check constraints or if check_set is already initialized */ - if (!s->check_set || s->check_set_initialized) + Field **vfield_ptr; + + /* If there is virtual fields are already initialized */ + if (s->check_set_initialized) return; - save_read_set= read_set; - read_set= s->check_set; + if (s->tmp_table == NO_TMP_TABLE) + mysql_mutex_lock(&s->LOCK_share); + if (s->check_set) + { + /* Mark fields used by check constraint */ + save_read_set= read_set; + read_set= s->check_set; - for (Virtual_column_info **chk= check_constraints ; *chk ; chk++) - (*chk)->expr->walk(&Item::register_field_in_read_map, 1, 0); + for (Virtual_column_info **chk= check_constraints ; *chk ; chk++) + (*chk)->expr->walk(&Item::register_field_in_read_map, 1, 0); + read_set= save_read_set; + } - read_set= save_read_set; + /* + mark all fields that part of a virtual indexed field with + PART_INDIRECT_KEY_FLAG. This is used to ensure that all fields + that are part of an index exits before write/delete/update. + + As this code is only executed once per open share, it's reusing + existing functionality instead of adding an extra argument to + add_field_to_set_processor or adding another processor. + */ + if (vfield) + { + for (vfield_ptr= vfield; *vfield_ptr; vfield_ptr++) + { + if ((*vfield_ptr)->flags & PART_KEY_FLAG) + (*vfield_ptr)->vcol_info->expr->walk(&Item::add_field_to_set_processor, + 1, this); + } + for (uint i= 0 ; i < s->fields ; i++) + { + if (bitmap_is_set(&tmp_set, i)) + field[i]->flags|= PART_INDIRECT_KEY_FLAG; + } + bitmap_clear_all(&tmp_set); + } s->check_set_initialized= 1; + if (s->tmp_table == NO_TMP_TABLE) + mysql_mutex_unlock(&s->LOCK_share); } /* Add fields used by CHECK CONSTRAINT to read map */ @@ -7465,7 +7504,8 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) update= bitmap_is_set(vcol_set, vf->field_index); break; case VCOL_UPDATE_FOR_REPLACE: - update= ((!vcol_info->stored_in_db && (vf->flags & PART_KEY_FLAG) && + update= ((!vcol_info->stored_in_db && + (vf->flags & (PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG)) && bitmap_is_set(vcol_set, vf->field_index)) || update_all_columns); if (update && (vf->flags & BLOB_FLAG)) @@ -7484,7 +7524,8 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) case VCOL_UPDATE_INDEXED: case VCOL_UPDATE_INDEXED_FOR_UPDATE: /* Read indexed fields that was not updated in VCOL_UPDATE_FOR_READ */ - update= (!vcol_info->stored_in_db && (vf->flags & PART_KEY_FLAG) && + update= (!vcol_info->stored_in_db && + (vf->flags & (PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG)) && !bitmap_is_set(vcol_set, vf->field_index)); swap_values= 1; break; diff --git a/sql/table.h b/sql/table.h index 6febeb555f9..4c409342c27 100644 --- a/sql/table.h +++ b/sql/table.h @@ -1323,7 +1323,7 @@ public: bool mark_virtual_col(Field *field); bool mark_virtual_columns_for_write(bool insert_fl); void mark_default_fields_for_write(bool insert_fl); - void mark_columns_used_by_check_constraints(void); + void mark_columns_used_by_virtual_fields(void); void mark_check_constraint_columns_for_read(void); int verify_constraints(bool ignore_failure); inline void column_bitmaps_set(MY_BITMAP *read_set_arg) From ab194666564acab29a4bcb7ca763e0ae0e691d1f Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 17 Jun 2018 14:19:51 +0300 Subject: [PATCH 145/203] MDEV-15114 ASAN heap-use-after-free in mem_heap_dup or dfield_data_is_binary_equal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bug was that innobase_get_computed_value() trashed record[0] and data in Field_blob::value Fixed by using a record on the heap for innobase_get_computed_value() Reviewer: Marko Mäkelä --- mysql-test/suite/vcol/r/index.result | 53 +++++++++++++++++ mysql-test/suite/vcol/t/index.test | 62 +++++++++++++++++++- sql/field.h | 4 ++ sql/table.cc | 48 ++++++++++++++++ sql/table.h | 7 ++- storage/innobase/handler/ha_innodb.cc | 82 ++++++++++++++++++++++++--- storage/innobase/include/row0mysql.h | 39 +++++++++++++ storage/innobase/row/row0ins.cc | 20 ++++++- storage/innobase/row/row0merge.cc | 27 +++++++-- storage/innobase/row/row0sel.cc | 19 ++++++- storage/innobase/row/row0upd.cc | 23 +++++++- storage/innobase/row/row0vers.cc | 18 +++++- 12 files changed, 376 insertions(+), 26 deletions(-) diff --git a/mysql-test/suite/vcol/r/index.result b/mysql-test/suite/vcol/r/index.result index cd4ebc96024..8860a728bd1 100644 --- a/mysql-test/suite/vcol/r/index.result +++ b/mysql-test/suite/vcol/r/index.result @@ -1,3 +1,6 @@ +# +# Test table with a key that consists of indirect virtual fields +# CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=myisam; insert into t1 (a) values (1),(2),(3); update t1 set a=5 where a=3; @@ -36,3 +39,53 @@ a b c 2 3 4 5 6 7 drop table t1; +# +# MDEV-15114 +# ASAN heap-use-after-free in mem_heap_dup or +# dfield_data_is_binary_equal +# +CREATE TABLE t1 ( +pk INT, +extra tinyint, +c TEXT, +vc LONGTEXT AS (c) VIRTUAL, +i INT, +PRIMARY KEY(pk), +UNIQUE(i), +INDEX(vc(64)) +) ENGINE=InnoDB; +INSERT INTO t1 (pk,extra, c, i) VALUES (1, 10, REPEAT('foo ',15000),0); +REPLACE INTO t1 (pk,extra, c,i) SELECT pk,extra+10, c,i FROM t1; +select pk, extra, left(c, 10), length(c), left(vc,10), length(vc), extra from t1; +pk extra left(c, 10) length(c) left(vc,10) length(vc) extra +1 20 foo foo fo 60000 foo foo fo 60000 20 +DROP TABLE t1; +# +# Update of deleted row with binary logging enabled +# +SET BINLOG_FORMAT=row; +CREATE TABLE t1 ( +pk INT, +c TEXT, +vc LONGTEXT AS (c) VIRTUAL, +i INT, +PRIMARY KEY(pk), +UNIQUE(i), +INDEX(vc(64)) +) ENGINE=InnoDB; +INSERT INTO t1 (pk,c,i) VALUES (1,REPEAT('foo ',15000),10); +INSERT INTO t1 (pk,c,i) VALUES (2,REPEAT('bar ',15000),11); +connect c1,localhost,root,,; +connection c1; +begin; +DELETE from t1 WHERE pk=1; +connection default; +update t1 set pk=1 where pk=2; +connection c1; +commit; +connection default; +select pk, left(c, 10), length(c), i from t1; +pk left(c, 10) length(c) i +1 bar bar ba 60000 11 +drop table t1; +disconnect c1; diff --git a/mysql-test/suite/vcol/t/index.test b/mysql-test/suite/vcol/t/index.test index 55d5b68f26b..72eed0a8a40 100644 --- a/mysql-test/suite/vcol/t/index.test +++ b/mysql-test/suite/vcol/t/index.test @@ -1,8 +1,9 @@ --source include/have_innodb.inc +--source include/have_log_bin.inc -# -# Test creating table with a key that consists of indirect virtual fields -# +--echo # +--echo # Test table with a key that consists of indirect virtual fields +--echo # CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=myisam; insert into t1 (a) values (1),(2),(3); @@ -23,3 +24,58 @@ select * from t1 where c=7; check table t1; select * from t1; drop table t1; + +--echo # +--echo # MDEV-15114 +--echo # ASAN heap-use-after-free in mem_heap_dup or +--echo # dfield_data_is_binary_equal +--echo # + +CREATE TABLE t1 ( + pk INT, + extra tinyint, + c TEXT, + vc LONGTEXT AS (c) VIRTUAL, + i INT, + PRIMARY KEY(pk), + UNIQUE(i), + INDEX(vc(64)) +) ENGINE=InnoDB; + +INSERT INTO t1 (pk,extra, c, i) VALUES (1, 10, REPEAT('foo ',15000),0); +REPLACE INTO t1 (pk,extra, c,i) SELECT pk,extra+10, c,i FROM t1; +select pk, extra, left(c, 10), length(c), left(vc,10), length(vc), extra from t1; +DROP TABLE t1; + +--echo # +--echo # Update of deleted row with binary logging enabled +--echo # + +SET BINLOG_FORMAT=row; + +CREATE TABLE t1 ( + pk INT, + c TEXT, + vc LONGTEXT AS (c) VIRTUAL, + i INT, + PRIMARY KEY(pk), + UNIQUE(i), + INDEX(vc(64)) +) ENGINE=InnoDB; + +INSERT INTO t1 (pk,c,i) VALUES (1,REPEAT('foo ',15000),10); +INSERT INTO t1 (pk,c,i) VALUES (2,REPEAT('bar ',15000),11); + +--connect (c1,localhost,root,,) +--connection c1 +begin; +DELETE from t1 WHERE pk=1; +--connection default +--send update t1 set pk=1 where pk=2 +--connection c1 +commit; +--connection default +--reap +select pk, left(c, 10), length(c), i from t1; +drop table t1; +disconnect c1; diff --git a/sql/field.h b/sql/field.h index e554f92031c..820cc5f3a7a 100644 --- a/sql/field.h +++ b/sql/field.h @@ -3433,6 +3433,10 @@ public: uint32 max_display_length(); uint32 char_length() const; uint is_equal(Create_field *new_field); + + friend void TABLE::remember_blob_values(String *blob_storage); + friend void TABLE::restore_blob_values(String *blob_storage); + private: int do_save_field_metadata(uchar *first_byte); }; diff --git a/sql/table.cc b/sql/table.cc index 198e77f1872..f7aee3eae7d 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -2434,6 +2434,8 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, reg_field->vcol_info= vcol_info; share->virtual_fields++; share->stored_fields--; + if (reg_field->flags & BLOB_FLAG) + share->virtual_not_stored_blob_fields++; /* Correct stored_rec_length as non stored fields are last */ recpos= (uint) (reg_field->ptr - record); if (share->stored_rec_length >= recpos) @@ -6776,6 +6778,52 @@ void TABLE::move_fields(Field **ptr, const uchar *to, const uchar *from) } +/* + Store all allocated virtual fields blob values + Used by InnoDB when calculating virtual fields for it's own internal + records +*/ + +void TABLE::remember_blob_values(String *blob_storage) +{ + Field **vfield_ptr; + for (vfield_ptr= vfield; *vfield_ptr; vfield_ptr++) + { + if ((*vfield_ptr)->type() == MYSQL_TYPE_BLOB && + !(*vfield_ptr)->vcol_info->stored_in_db) + { + Field_blob *blob= ((Field_blob*) *vfield_ptr); + memcpy((void*) blob_storage, (void*) &blob->value, sizeof(blob->value)); + blob_storage++; + blob->value.release(); + } + } +} + + +/* + Restore all allocated virtual fields blob values + Used by InnoDB when calculating virtual fields for it's own internal + records +*/ + +void TABLE::restore_blob_values(String *blob_storage) +{ + Field **vfield_ptr; + for (vfield_ptr= vfield; *vfield_ptr; vfield_ptr++) + { + if ((*vfield_ptr)->type() == MYSQL_TYPE_BLOB && + !(*vfield_ptr)->vcol_info->stored_in_db) + { + Field_blob *blob= ((Field_blob*) *vfield_ptr); + blob->value.free(); + memcpy((void*) &blob->value, (void*) blob_storage, sizeof(blob->value)); + blob_storage++; + } + } +} + + /** @brief Allocate space for keys diff --git a/sql/table.h b/sql/table.h index 4c409342c27..c0cca1026ea 100644 --- a/sql/table.h +++ b/sql/table.h @@ -663,8 +663,11 @@ struct TABLE_SHARE */ uint null_bytes_for_compare; uint fields; /* number of fields */ - uint stored_fields; /* number of stored fields, purely virtual not included */ + /* number of stored fields, purely virtual not included */ + uint stored_fields; uint virtual_fields; /* number of purely virtual fields */ + /* number of purely virtual not stored blobs */ + uint virtual_not_stored_blob_fields; uint null_fields; /* number of null fields */ uint blob_fields; /* number of blob fields */ uint varchar_fields; /* number of varchar fields */ @@ -1423,6 +1426,8 @@ public: { return (my_ptrdiff_t) (s->default_values - record[0]); } void move_fields(Field **ptr, const uchar *to, const uchar *from); + void remember_blob_values(String *blob_storage); + void restore_blob_values(String *blob_storage); uint actual_n_key_parts(KEY *keyinfo); ulong actual_key_flags(KEY *keyinfo); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 2c8ea81e286..7ac7495e221 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -21760,6 +21760,77 @@ innobase_get_field_from_update_vector( return (NULL); } + +/** + Allocate a heap and record for calculating virtual fields + Used mainly for virtual fields in indexes + +@param[in] thd MariaDB THD +@param[in] index Index in use +@param[out] heap Heap that holds temporary row +@param[in,out] mysql_table MariaDB table +@param[out] rec Pointer to allocated MariaDB record +@param[out] storage Internal storage for blobs etc + +@return FALSE ok +@return TRUE malloc failure +*/ + +bool innobase_allocate_row_for_vcol( + THD * thd, + dict_index_t* index, + mem_heap_t** heap, + TABLE** table, + byte** record, + VCOL_STORAGE** storage) +{ + TABLE *maria_table; + String *blob_value_storage; + if (!*table) + *table= innobase_find_mysql_table_for_vc(thd, index->table); + maria_table= *table; + if (!*heap && !(*heap= mem_heap_create(srv_page_size))) + { + *storage= 0; + return TRUE; + } + *record= static_cast(mem_heap_alloc(*heap, + maria_table->s->reclength)); + *storage= static_cast + (mem_heap_alloc(*heap, sizeof(**storage))); + blob_value_storage= static_cast + (mem_heap_alloc(*heap, + maria_table->s->virtual_not_stored_blob_fields * + sizeof(String))); + if (!*record || !*storage || !blob_value_storage) + { + *storage= 0; + return TRUE; + } + (*storage)->maria_table= maria_table; + (*storage)->innobase_record= *record; + (*storage)->maria_record= maria_table->field[0]->record_ptr(); + (*storage)->blob_value_storage= blob_value_storage; + + maria_table->move_fields(maria_table->field, *record, + (*storage)->maria_record); + maria_table->remember_blob_values(blob_value_storage); + + return FALSE; +} + + +/** Free memory allocated by innobase_allocate_row_for_vcol() */ + +void innobase_free_row_for_vcol(VCOL_STORAGE *storage) +{ + TABLE *maria_table= storage->maria_table; + maria_table->move_fields(maria_table->field, storage->maria_record, + storage->innobase_record); + maria_table->restore_blob_values(storage->blob_value_storage); +} + + /** Get the computed value by supplying the base column values. @param[in,out] row the data row @param[in] col virtual column @@ -21785,12 +21856,12 @@ innobase_get_computed_value( const dict_field_t* ifield, THD* thd, TABLE* mysql_table, + byte* mysql_rec, const dict_table_t* old_table, upd_t* parent_update, dict_foreign_t* foreign) { byte rec_buf2[REC_VERSION_56_MAX_INDEX_COL_LEN]; - byte* mysql_rec; byte* buf; dfield_t* field; ulint len; @@ -21803,6 +21874,7 @@ innobase_get_computed_value( ut_ad(index->table->vc_templ); ut_ad(thd != NULL); + ut_ad(mysql_table); const mysql_row_templ_t* vctempl = index->table->vc_templ->vtempl[ @@ -21820,14 +21892,6 @@ innobase_get_computed_value( buf = rec_buf2; } - if (!mysql_table) { - mysql_table = innobase_find_mysql_table_for_vc(thd, index->table); - } - - ut_ad(mysql_table); - - mysql_rec = mysql_table->record[0]; - for (ulint i = 0; i < col->num_base; i++) { dict_col_t* base_col = col->base_col[i]; const dfield_t* row_field = NULL; diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 7a34c025dab..8a573a23652 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -846,6 +846,44 @@ struct SysIndexCallback { virtual void operator()(mtr_t* mtr, btr_pcur_t* pcur) throw() = 0; }; + +/** Storage for calculating virtual columns */ + +class String; +struct VCOL_STORAGE +{ + TABLE *maria_table; + byte *innobase_record; + byte *maria_record; + String *blob_value_storage; +}; + +/** + Allocate a heap and record for calculating virtual fields + Used mainly for virtual fields in indexes + +@param[in] thd MariaDB THD +@param[in] index Index in use +@param[out] heap Heap that holds temporary row +@param[in,out] mysql_table MariaDB table +@param[out] rec Pointer to allocated MariaDB record +@param[out] storage Internal storage for blobs etc + +@return FALSE ok +@return TRUE malloc failure +*/ + +bool innobase_allocate_row_for_vcol( + THD * thd, + dict_index_t* index, + mem_heap_t** heap, + TABLE** table, + byte** record, + VCOL_STORAGE** storage); + +/** Free memory allocated by innobase_allocate_row_for_vcol() */ +void innobase_free_row_for_vcol(VCOL_STORAGE *storage); + /** Get the computed value by supplying the base column values. @param[in,out] row the data row @param[in] col virtual column @@ -870,6 +908,7 @@ innobase_get_computed_value( const dict_field_t* ifield, THD* thd, TABLE* mysql_table, + byte* mysql_rec, const dict_table_t* old_table, upd_t* parent_update, dict_foreign_t* foreign); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 1aa97bc34c5..57796939be1 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -944,6 +944,9 @@ row_ins_foreign_fill_virtual( rec_get_offsets(rec, index, offsets_, true, ULINT_UNDEFINED, &cascade->heap); mem_heap_t* v_heap = NULL; + TABLE* mysql_table= NULL; + VCOL_STORAGE* vcol_storage= NULL; + byte* record; upd_t* update = cascade->update; ulint n_v_fld = index->table->n_v_def; ulint n_diff; @@ -963,6 +966,14 @@ row_ins_foreign_fill_virtual( innobase_init_vc_templ(index->table); } + if (innobase_allocate_row_for_vcol(thd, index, &v_heap, + &mysql_table, + &record, &vcol_storage)) + { + *err = DB_OUT_OF_MEMORY; + goto func_exit; + } + for (ulint i = 0; i < n_v_fld; i++) { dict_v_col_t* col = dict_table_get_nth_v_col( @@ -976,8 +987,8 @@ row_ins_foreign_fill_virtual( dfield_t* vfield = innobase_get_computed_value( update->old_vrow, col, index, - &v_heap, update->heap, NULL, thd, NULL, - NULL, NULL, NULL); + &v_heap, update->heap, NULL, thd, mysql_table, + record, NULL, NULL, NULL); if (vfield == NULL) { *err = DB_COMPUTE_VALUE_FAILED; @@ -1007,7 +1018,8 @@ row_ins_foreign_fill_virtual( dfield_t* new_vfield = innobase_get_computed_value( update->old_vrow, col, index, &v_heap, update->heap, NULL, thd, - NULL, NULL, node->update, foreign); + mysql_table, record, NULL, + node->update, foreign); if (new_vfield == NULL) { *err = DB_COMPUTE_VALUE_FAILED; @@ -1025,6 +1037,8 @@ row_ins_foreign_fill_virtual( func_exit: if (v_heap) { + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(v_heap); } } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index bed15941638..66f3b3936af 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -534,6 +534,8 @@ row_merge_buf_add( ulint bucket = 0; doc_id_t write_doc_id; ulint n_row_added = 0; + VCOL_STORAGE* vcol_storage= 0; + byte* record; DBUG_ENTER("row_merge_buf_add"); if (buf->n_tuples >= buf->max_tuples) { @@ -606,14 +608,21 @@ row_merge_buf_add( dict_index_t* clust_index = dict_table_get_first_index(new_table); + if (!vcol_storage && + innobase_allocate_row_for_vcol(trx->mysql_thd, clust_index, v_heap, &my_table, &record, &vcol_storage)) { + *err = DB_OUT_OF_MEMORY; + goto error; + } + row_field = innobase_get_computed_value( row, v_col, clust_index, v_heap, NULL, ifield, trx->mysql_thd, - my_table, old_table, NULL, NULL); + my_table, record, old_table, NULL, + NULL); if (row_field == NULL) { *err = DB_COMPUTE_VALUE_FAILED; - DBUG_RETURN(0); + goto error; } dfield_copy(field, row_field); } else { @@ -649,7 +658,7 @@ row_merge_buf_add( ib::warn() << "FTS Doc ID is" " zero. Record" " skipped"; - DBUG_RETURN(0); + goto error; } } @@ -797,7 +806,7 @@ row_merge_buf_add( /* If this is FTS index, we already populated the sort buffer, return here */ if (index->type & DICT_FTS) { - DBUG_RETURN(n_row_added); + goto end; } #ifdef UNIV_DEBUG @@ -831,7 +840,7 @@ row_merge_buf_add( /* Reserve bytes for the end marker of row_merge_block_t. */ if (buf->total_size + data_size >= srv_sort_buf_size) { - DBUG_RETURN(0); + goto error; } buf->total_size += data_size; @@ -850,7 +859,15 @@ row_merge_buf_add( mem_heap_empty(conv_heap); } +end: + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); DBUG_RETURN(n_row_added); + +error: + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); + DBUG_RETURN(0); } /*************************************************************//** diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 7383d3d9510..68a81615e88 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -179,6 +179,8 @@ row_sel_sec_rec_is_for_clust_rec( ulint* clust_offs = clust_offsets_; ulint* sec_offs = sec_offsets_; ibool is_equal = TRUE; + VCOL_STORAGE* vcol_storage= 0; + byte* record; rec_offs_init(clust_offsets_); rec_offs_init(sec_offsets_); @@ -226,6 +228,17 @@ row_sel_sec_rec_is_for_clust_rec( dfield_t* vfield; row_ext_t* ext; + if (!vcol_storage) + { + TABLE *mysql_table= thr->prebuilt->m_mysql_table; + innobase_allocate_row_for_vcol(thr_get_trx(thr)->mysql_thd, + clust_index, + &heap, + &mysql_table, + &record, + &vcol_storage); + } + v_col = reinterpret_cast(col); row = row_build(ROW_COPY_POINTERS, @@ -237,8 +250,8 @@ row_sel_sec_rec_is_for_clust_rec( row, v_col, clust_index, &heap, NULL, NULL, thr_get_trx(thr)->mysql_thd, - thr->prebuilt->m_mysql_table, NULL, - NULL, NULL); + thr->prebuilt->m_mysql_table, + record, NULL, NULL, NULL); clust_len = vfield->len; clust_field = static_cast(vfield->data); @@ -326,6 +339,8 @@ inequal: func_exit: if (UNIV_LIKELY_NULL(heap)) { + if (UNIV_LIKELY_NULL(vcol_storage)) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(heap); } return(is_equal); diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 18a6a26f3ce..171e06894ca 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1023,6 +1023,7 @@ row_upd_build_sec_rec_difference_binary( return(update); } + /** Builds an update vector from those fields, excluding the roll ptr and trx id fields, which in an index entry differ from a record that has the equal ordering fields. NOTE: we compare the fields as binary strings! @@ -1123,6 +1124,9 @@ row_upd_build_difference_binary( if (n_v_fld > 0) { row_ext_t* ext; mem_heap_t* v_heap = NULL; + byte* record; + VCOL_STORAGE* vcol_storage; + THD* thd; if (trx == NULL) { @@ -1133,6 +1137,10 @@ row_upd_build_difference_binary( ut_ad(!update->old_vrow); + innobase_allocate_row_for_vcol(thd, index, &v_heap, + &mysql_table, + &record, &vcol_storage); + for (i = 0; i < n_v_fld; i++) { const dict_v_col_t* col = dict_table_get_nth_v_col(index->table, i); @@ -1151,7 +1159,7 @@ row_upd_build_difference_binary( dfield_t* vfield = innobase_get_computed_value( update->old_vrow, col, index, - &v_heap, heap, NULL, thd, mysql_table, + &v_heap, heap, NULL, thd, mysql_table, record, NULL, NULL, NULL); if (!dfield_data_is_binary_equal( @@ -1177,6 +1185,8 @@ row_upd_build_difference_binary( } if (v_heap) { + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(v_heap); } } @@ -2116,6 +2126,12 @@ row_upd_store_v_row( { mem_heap_t* heap = NULL; dict_index_t* index = dict_table_get_first_index(node->table); + byte* record= 0; + VCOL_STORAGE *vcol_storage= 0; + + if (!update) + innobase_allocate_row_for_vcol(thd, index, &heap, &mysql_table, + &record, &vcol_storage); for (ulint col_no = 0; col_no < dict_table_get_n_v_cols(node->table); col_no++) { @@ -2168,7 +2184,7 @@ row_upd_store_v_row( innobase_get_computed_value( node->row, col, index, &heap, node->heap, NULL, - thd, mysql_table, NULL, + thd, mysql_table, record, NULL, NULL, NULL); } } @@ -2176,8 +2192,11 @@ row_upd_store_v_row( } if (heap) { + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(heap); } + } /** Stores to the heap the row on which the node->pcur is positioned. diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 31cef0bad2d..aebcf70715f 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -446,6 +446,19 @@ row_vers_build_clust_v_col( mem_heap_t* heap) { mem_heap_t* local_heap = NULL; + VCOL_STORAGE *vcol_storage= NULL; + THD* thd= current_thd; + TABLE* maria_table= 0; + byte* record= 0; + + ut_ad(dict_index_has_virtual(index)); + + innobase_allocate_row_for_vcol(thd, index, + &local_heap, + &maria_table, + &record, + &vcol_storage); + for (ulint i = 0; i < dict_index_get_n_fields(index); i++) { const dict_field_t* ind_field = dict_index_get_nth_field( index, i); @@ -458,15 +471,18 @@ row_vers_build_clust_v_col( innobase_get_computed_value( row, col, clust_index, &local_heap, - heap, NULL, current_thd, NULL, NULL, + heap, NULL, thd, maria_table, record, NULL, NULL, NULL); } } if (local_heap) { + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(local_heap); } } + /** Build latest virtual column data from undo log @param[in] in_purge whether this is the purge thread @param[in] rec clustered index record From 10d09a57f88cafaabcb6ba8475c1951fe329756e Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 18 Jun 2018 22:44:58 +0300 Subject: [PATCH 146/203] Fixed failing test acl_load_mutex-5170 Added flush tables to ensure that MyISAM tables are properly flushed before reboot --- mysql-test/suite/roles/acl_load_mutex-5170.result | 1 + mysql-test/suite/roles/acl_load_mutex-5170.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/roles/acl_load_mutex-5170.result b/mysql-test/suite/roles/acl_load_mutex-5170.result index 6f6688f446e..775541b94f7 100644 --- a/mysql-test/suite/roles/acl_load_mutex-5170.result +++ b/mysql-test/suite/roles/acl_load_mutex-5170.result @@ -1,6 +1,7 @@ create user user1@localhost; create role r1 with admin user1@localhost; grant all on test.* to r1; +flush tables; select 1; 1 1 diff --git a/mysql-test/suite/roles/acl_load_mutex-5170.test b/mysql-test/suite/roles/acl_load_mutex-5170.test index e77d191ea16..22b9dffb5fd 100644 --- a/mysql-test/suite/roles/acl_load_mutex-5170.test +++ b/mysql-test/suite/roles/acl_load_mutex-5170.test @@ -5,6 +5,7 @@ create user user1@localhost; create role r1 with admin user1@localhost; grant all on test.* to r1; +flush tables; --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect wait From 778df04661c72544baed4363a55b07362d5020ed Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Tue, 19 Jun 2018 19:19:40 +0200 Subject: [PATCH 147/203] MDEV-16517: Server crash in Item_func_in::val_int() when IN predicate defined with non-constant values is pushed down The problem appears because of wrong changes made in MDEV-16090 in the Item_func_in::build_clone() method. For the clone of the IN predicate it copied 'cmp_fields' array values that become dirty after Item::cleanup_excluding_const_fields_processor has worked in pushdown. That causes crash. There is no need to copy 'cmp_fields' field, the array values should be NULLs in order to fix_fields() for the cloned IN predicate can set them correctly. fix_fields() computes values for 'cmp_fields' array only if they were not set earlier. --- mysql-test/r/derived_cond_pushdown.result | 139 ++++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 39 ++++++ sql/item_cmpfunc.cc | 2 +- 3 files changed, 179 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 83e70dd634c..a2e84a1e419 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -9669,3 +9669,142 @@ EXPLAIN } } DROP TABLE t1; +# +# MDEV-16517: pushdown condition with the IN predicate defined +# with non-constant values +# +CREATE TABLE t1 (a INT, b INT); +INSERT INTO t1 VALUES (1,2),(1,3); +SELECT * FROM +( +SELECT t1.a +FROM t1 +WHERE 1 IN (0,t1.a) +GROUP BY t1.a +) AS dt1 +JOIN +( +SELECT t1.a +FROM t1 +WHERE 1 IN (0,t1.a) +) AS dt2 +ON dt1.a = dt2.a; +a a +1 1 +1 1 +EXPLAIN FORMAT=JSON SELECT * FROM +( +SELECT t1.a +FROM t1 +WHERE 1 IN (0,t1.a) +GROUP BY t1.a +) AS dt1 +JOIN +( +SELECT t1.a +FROM t1 +WHERE 1 IN (0,t1.a) +) AS dt2 +ON dt1.a = dt2.a; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 2, + "filtered": 100, + "attached_condition": "1 in (0,dt1.a)", + "materialized": { + "query_block": { + "select_id": 2, + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 2, + "filtered": 100, + "attached_condition": "1 in (0,t1.a) and 1 in (0,t1.a)" + } + } + } + } + } + }, + "block-nl-join": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 2, + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "t1.a = dt1.a" + } + } +} +SELECT * FROM +( +SELECT t1.a,MAX(t1.b) +FROM t1 +GROUP BY t1.a +) AS dt, t1 +WHERE dt.a=t1.a AND dt.a IN (1,t1.a); +a MAX(t1.b) a b +1 3 1 2 +1 3 1 3 +EXPLAIN FORMAT=JSON SELECT * FROM +( +SELECT t1.a,MAX(t1.b) +FROM t1 +GROUP BY t1.a +) AS dt, t1 +WHERE dt.a=t1.a AND dt.a IN (1,t1.a); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 2, + "filtered": 100, + "attached_condition": "dt.a in (1,dt.a)", + "materialized": { + "query_block": { + "select_id": 2, + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 2, + "filtered": 100, + "attached_condition": "t1.a in (1,t1.a)" + } + } + } + } + } + }, + "block-nl-join": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 2, + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "t1.a = dt.a" + } + } +} +DROP TABLE t1; diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index 718140d3a77..3103ddb5c55 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -1801,3 +1801,42 @@ EVAL $query; EVAL EXPLAIN FORMAT=JSON $query; DROP TABLE t1; + +--echo # +--echo # MDEV-16517: pushdown condition with the IN predicate defined +--echo # with non-constant values +--echo # + +CREATE TABLE t1 (a INT, b INT); +INSERT INTO t1 VALUES (1,2),(1,3); + +LET $query= +SELECT * FROM +( + SELECT t1.a + FROM t1 + WHERE 1 IN (0,t1.a) + GROUP BY t1.a +) AS dt1 +JOIN +( + SELECT t1.a + FROM t1 + WHERE 1 IN (0,t1.a) +) AS dt2 +ON dt1.a = dt2.a; +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +LET $query= +SELECT * FROM +( + SELECT t1.a,MAX(t1.b) + FROM t1 + GROUP BY t1.a +) AS dt, t1 +WHERE dt.a=t1.a AND dt.a IN (1,t1.a); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +DROP TABLE t1; diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index b86c0079bce..f176a0a8193 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4438,7 +4438,7 @@ Item *Item_func_in::build_clone(THD *thd, MEM_ROOT *mem_root) { if (array && clone->create_array(thd)) return NULL; - memcpy(&clone->cmp_items, &cmp_items, sizeof(cmp_items)); + bzero(&clone->cmp_items, sizeof(cmp_items)); } return clone; } From 5f2a67a6c35fd0d833024652ddc33eab8bcb1ed4 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Wed, 20 Jun 2018 02:36:00 +0530 Subject: [PATCH 148/203] MDEV-15247: Crash when SET NAMES 'utf8' is set In this case we are accessing incorrect memory when we have mergeable semi-joins. In the case when we have mergeable semi joins parent select will have a table count of all the tables in that select plus all the tables involved in the IN-subquery. But this table count does not include the "sjm table" (only includes the inner and outer tables) denotes as in explain. --- mysql-test/r/subselect_sj2_mat.result | 90 +++++++++++++++++++++++++++ mysql-test/t/subselect_sj2_mat.test | 87 ++++++++++++++++++++++++++ sql/sql_select.cc | 4 +- 3 files changed, 180 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/subselect_sj2_mat.result b/mysql-test/r/subselect_sj2_mat.result index c629c8196c7..c27beb295b8 100644 --- a/mysql-test/r/subselect_sj2_mat.result +++ b/mysql-test/r/subselect_sj2_mat.result @@ -1691,3 +1691,93 @@ id 12 13 drop table t1; +# +# MDEV-15247: Crash when SET NAMES 'utf8' is set +# +CREATE TABLE t1 ( +id_category int unsigned, +id_product int unsigned, +PRIMARY KEY (id_category,id_product) +) ENGINE=MyISAM; +INSERT INTO `t1` VALUES (31,216), (31,215), (31,214), (31,213), (31,212), (32,211), (32,210), (32,209), (32,208), (29,207), (30,315372), (2,161), (2,132), (33,315380), (31,315371), (29,315370), (29,315373), (29,315369), (29,315374), (29,315368), (29,315375), (29,315367), (29,183), (29,182), (30,177), (29,315376), (13,315365), (2,167), (2,315357), (2,164), (2,159), (2,131), (2,127), (14,315364), (27,315363), (29,205), (29,204), (29,203), (29,202), (29,201), (29,200), (29,199), (29,198), (29,197), (29,196), (29,195), (29,194), (29,193), (29,192), (29,191), (29,190), (29,189), (14,188), (29,187), (29,186), (29,185), (29,184), (29,315377), (29,315378), (29,181), (33,315379), (29,179), (30,178), (29,180), (30,176), (30,175), (30,174), (30,173), (30,172), (11,171), (27,315357), (23,108), (23,102); +CREATE TABLE t2 ( +id_product int, +id_t2 int, +KEY id_t2 (id_t2), +KEY id_product (id_product) +) ENGINE=MyISAM; +INSERT INTO `t2` VALUES (11,31), (11,31), (11,31), (11,32), (11,32), +(11,32), (10,26), (11,32), (10,28), (11,32), (10,29), (11,33), (10,26), +(11,33), (10,27), (9,23), (11,32), (10,26), (8,18), (7,15), (11,32), +(10,28), (11,32), (10,28), (11,32), (10,29), (11,32), (10,29), (8,19), +(7,16), (8,18), (7,16), (8,20), (7,16), (11,32), (10,28), (8,19), +(7,16), (8,20), (7,16), (11,32), (10,29), (8,19), (7,16), (8,20), +(7,16), (10,27), (9,23), (10,27), (9,23), (10,27), (9,23), (11,32), +(10,27), (11,32), (10,27), (8,18), (7,15), (10,26), (9,24), (8,19), +(7,16), (10,26), (9,23), (8,19), (7,16), (8,18), (7,16), (8,18), (7,16), +(9,23), (8,18), (9,23), (8,19), (7,16), (7,16), (8,19), (7,16), (11,31), +(10,27), (9,24), (11,31), (10,27), (9,23), (8,19), (11,31), (10,26), (9,24), +(8,19), (11,31), (10,26), (9,25), (8,18), (11,31), (10,26), (9,23), (8,19), +(11,31), (10,26), (9,23), (8,18), (11,31), (10,30), (9,23), (8,18), (11,31), +(10,30), (9,23), (8,19), (11,31), (10,26), (9,25), (8,19), (8,21), (11,32), +(10,26), (9,22), (8,19), (11,32), (10,26), (9,22), (8,18), (11,32), (10,26), +(9,22), (8,20), (11,33), (10,26), (9,22), (8,19), (11,33), (10,26), (9,22), +(8,18), (11,33), (10,26), (9,22), (8,20), (11,32), (10,26), (9,24), (8,19), +(11,32), (10,26), (9,25), (8,19), (11,32), (10,26), (9,25), (8,18), (11,32), +(10,26), (9,23), (8,18), (11,32), (10,30), (9,23), (8,18), (11,32), (10,30), +(9,23), (8,19), (11,32), (10,26), (9,23), (8,19), (11,32), (10,27), (9,23), +(11,32), (10,27), (9,23), (11,32), (10,27), (9,23), (10,26), (9,22), (8,19), +(7,15), (10,26), (9,22), (8,20), (7,15), (10,26), (9,22), (8,18), (7,15), +(8,19), (10,26), (10,26), (11,33), (10,26), (11,33), (10,26), (11,33), +(10,27), (11,33), (10,27), (11,31), (10,26), (11,31), (10,26), (8,18), +(7,15), (9,23), (9,23), (9,24), (8,21), (7,15), (7,15), (7,15), (7,15), +(7,15), (7,15), (7,15), (7,15), (7,15), (8,18), (7,17), (8,18), (7,17), (8,19), (8,19); +CREATE TABLE t3 ( +id_product int unsigned, +PRIMARY KEY (id_product) +) ENGINE=MyISAM; +INSERT INTO t3 VALUES +(102),(103),(104),(105),(106),(107),(108),(109),(110), +(315371),(315373),(315374),(315375),(315376),(315377), +(315378),(315379),(315380); +CREATE TABLE t4 ( +id_product int not null, +id_shop int, +PRIMARY KEY (id_product,id_shop) +) ENGINE=MyISAM ; +INSERT INTO t4 VALUES +(202,1),(201,1),(200,1),(199,1),(198,1),(197,1),(196,1),(195,1), +(194,1),(193,1),(192,1),(191,1),(190,1),(189,1),(188,1),(187,1), +(186,1),(185,1),(184,1),(183,1),(182,1),(181,1),(179,1),(178,1), +(177,1),(176,1),(126,1),(315380,1); +CREATE TABLE t5 (id_product int) ENGINE=MyISAM; +INSERT INTO `t5` VALUES +(652),(668),(669),(670),(671),(673),(674),(675),(676), +(677),(679),(680),(681),(682),(683),(684),(685),(686); +explain +SELECT * FROM t3 +JOIN t4 ON (t4.id_product = t3.id_product AND t4.id_shop = 1) +JOIN t1 ON (t1.id_product = t3.id_product) +LEFT JOIN t5 ON (t5.id_product = t3.id_product) +WHERE 1=1 +AND t3.id_product IN (SELECT id_product FROM t2 t2_1 WHERE t2_1.id_t2 = 32) +AND t3.id_product IN (SELECT id_product FROM t2 t2_2 WHERE t2_2.id_t2 = 15) +AND t3.id_product IN (SELECT id_product FROM t2 t2_3 WHERE t2_3.id_t2 = 18 OR t2_3.id_t2 = 19) +AND t3.id_product IN (SELECT id_product FROM t2 t2_4 WHERE t2_4.id_t2 = 34 OR t2_4.id_t2 = 23) +AND t3.id_product IN (SELECT id_product FROM t2 t2_5 WHERE t2_5.id_t2 = 29 OR t2_5.id_t2 = 28 OR t2_5.id_t2 = 26); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 index NULL PRIMARY 8 NULL 73 Using index +1 PRIMARY t3 eq_ref PRIMARY PRIMARY 4 test.t1.id_product 1 Using index +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY t4 eq_ref PRIMARY PRIMARY 8 test.t1.id_product,const 1 Using where; Using index +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY t5 ALL NULL NULL NULL NULL 18 Using where; Using join buffer (flat, BNL join) +5 MATERIALIZED t2_4 range id_t2,id_product id_t2 5 NULL 18 Using index condition; Using where +4 MATERIALIZED t2_3 range id_t2,id_product id_t2 5 NULL 32 Using index condition; Using where +3 MATERIALIZED t2_2 ref id_t2,id_product id_t2 5 const 12 +2 MATERIALIZED t2_1 ref id_t2,id_product id_t2 5 const 50 +6 MATERIALIZED t2_5 range id_t2,id_product id_t2 5 NULL 30 Using index condition; Using where +drop table t1,t2,t3,t4,t5; diff --git a/mysql-test/t/subselect_sj2_mat.test b/mysql-test/t/subselect_sj2_mat.test index 0665cdf68fe..68a888012f2 100644 --- a/mysql-test/t/subselect_sj2_mat.test +++ b/mysql-test/t/subselect_sj2_mat.test @@ -345,3 +345,90 @@ WHERE ( (t.id IN (0,4,12,13,1,10,3,11)) ); drop table t1; + +--echo # +--echo # MDEV-15247: Crash when SET NAMES 'utf8' is set +--echo # + +CREATE TABLE t1 ( + id_category int unsigned, + id_product int unsigned, + PRIMARY KEY (id_category,id_product) +) ENGINE=MyISAM; + +INSERT INTO `t1` VALUES (31,216), (31,215), (31,214), (31,213), (31,212), (32,211), (32,210), (32,209), (32,208), (29,207), (30,315372), (2,161), (2,132), (33,315380), (31,315371), (29,315370), (29,315373), (29,315369), (29,315374), (29,315368), (29,315375), (29,315367), (29,183), (29,182), (30,177), (29,315376), (13,315365), (2,167), (2,315357), (2,164), (2,159), (2,131), (2,127), (14,315364), (27,315363), (29,205), (29,204), (29,203), (29,202), (29,201), (29,200), (29,199), (29,198), (29,197), (29,196), (29,195), (29,194), (29,193), (29,192), (29,191), (29,190), (29,189), (14,188), (29,187), (29,186), (29,185), (29,184), (29,315377), (29,315378), (29,181), (33,315379), (29,179), (30,178), (29,180), (30,176), (30,175), (30,174), (30,173), (30,172), (11,171), (27,315357), (23,108), (23,102); + +CREATE TABLE t2 ( + id_product int, + id_t2 int, + KEY id_t2 (id_t2), + KEY id_product (id_product) +) ENGINE=MyISAM; + +INSERT INTO `t2` VALUES (11,31), (11,31), (11,31), (11,32), (11,32), +(11,32), (10,26), (11,32), (10,28), (11,32), (10,29), (11,33), (10,26), +(11,33), (10,27), (9,23), (11,32), (10,26), (8,18), (7,15), (11,32), +(10,28), (11,32), (10,28), (11,32), (10,29), (11,32), (10,29), (8,19), +(7,16), (8,18), (7,16), (8,20), (7,16), (11,32), (10,28), (8,19), +(7,16), (8,20), (7,16), (11,32), (10,29), (8,19), (7,16), (8,20), +(7,16), (10,27), (9,23), (10,27), (9,23), (10,27), (9,23), (11,32), +(10,27), (11,32), (10,27), (8,18), (7,15), (10,26), (9,24), (8,19), +(7,16), (10,26), (9,23), (8,19), (7,16), (8,18), (7,16), (8,18), (7,16), +(9,23), (8,18), (9,23), (8,19), (7,16), (7,16), (8,19), (7,16), (11,31), +(10,27), (9,24), (11,31), (10,27), (9,23), (8,19), (11,31), (10,26), (9,24), +(8,19), (11,31), (10,26), (9,25), (8,18), (11,31), (10,26), (9,23), (8,19), +(11,31), (10,26), (9,23), (8,18), (11,31), (10,30), (9,23), (8,18), (11,31), +(10,30), (9,23), (8,19), (11,31), (10,26), (9,25), (8,19), (8,21), (11,32), +(10,26), (9,22), (8,19), (11,32), (10,26), (9,22), (8,18), (11,32), (10,26), +(9,22), (8,20), (11,33), (10,26), (9,22), (8,19), (11,33), (10,26), (9,22), +(8,18), (11,33), (10,26), (9,22), (8,20), (11,32), (10,26), (9,24), (8,19), +(11,32), (10,26), (9,25), (8,19), (11,32), (10,26), (9,25), (8,18), (11,32), +(10,26), (9,23), (8,18), (11,32), (10,30), (9,23), (8,18), (11,32), (10,30), +(9,23), (8,19), (11,32), (10,26), (9,23), (8,19), (11,32), (10,27), (9,23), +(11,32), (10,27), (9,23), (11,32), (10,27), (9,23), (10,26), (9,22), (8,19), +(7,15), (10,26), (9,22), (8,20), (7,15), (10,26), (9,22), (8,18), (7,15), +(8,19), (10,26), (10,26), (11,33), (10,26), (11,33), (10,26), (11,33), +(10,27), (11,33), (10,27), (11,31), (10,26), (11,31), (10,26), (8,18), +(7,15), (9,23), (9,23), (9,24), (8,21), (7,15), (7,15), (7,15), (7,15), +(7,15), (7,15), (7,15), (7,15), (7,15), (8,18), (7,17), (8,18), (7,17), (8,19), (8,19); + +CREATE TABLE t3 ( + id_product int unsigned, + PRIMARY KEY (id_product) +) ENGINE=MyISAM; + +INSERT INTO t3 VALUES +(102),(103),(104),(105),(106),(107),(108),(109),(110), +(315371),(315373),(315374),(315375),(315376),(315377), +(315378),(315379),(315380); + +CREATE TABLE t4 ( + id_product int not null, + id_shop int, + PRIMARY KEY (id_product,id_shop) +) ENGINE=MyISAM ; + +INSERT INTO t4 VALUES +(202,1),(201,1),(200,1),(199,1),(198,1),(197,1),(196,1),(195,1), +(194,1),(193,1),(192,1),(191,1),(190,1),(189,1),(188,1),(187,1), +(186,1),(185,1),(184,1),(183,1),(182,1),(181,1),(179,1),(178,1), +(177,1),(176,1),(126,1),(315380,1); + +CREATE TABLE t5 (id_product int) ENGINE=MyISAM; +INSERT INTO `t5` VALUES +(652),(668),(669),(670),(671),(673),(674),(675),(676), +(677),(679),(680),(681),(682),(683),(684),(685),(686); + +explain +SELECT * FROM t3 + JOIN t4 ON (t4.id_product = t3.id_product AND t4.id_shop = 1) + JOIN t1 ON (t1.id_product = t3.id_product) +LEFT JOIN t5 ON (t5.id_product = t3.id_product) +WHERE 1=1 +AND t3.id_product IN (SELECT id_product FROM t2 t2_1 WHERE t2_1.id_t2 = 32) +AND t3.id_product IN (SELECT id_product FROM t2 t2_2 WHERE t2_2.id_t2 = 15) +AND t3.id_product IN (SELECT id_product FROM t2 t2_3 WHERE t2_3.id_t2 = 18 OR t2_3.id_t2 = 19) +AND t3.id_product IN (SELECT id_product FROM t2 t2_4 WHERE t2_4.id_t2 = 34 OR t2_4.id_t2 = 23) +AND t3.id_product IN (SELECT id_product FROM t2 t2_5 WHERE t2_5.id_t2 = 29 OR t2_5.id_t2 = 28 OR t2_5.id_t2 = 26); + +drop table t1,t2,t3,t4,t5; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 39209f04c9f..1ec6cca7c5b 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -9592,7 +9592,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) table_map current_map; i= join->const_tables; for (tab= first_depth_first_tab(join); tab; - tab= next_depth_first_tab(join, tab), i++) + tab= next_depth_first_tab(join, tab)) { bool is_hj; /* @@ -10063,6 +10063,8 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) } first_inner_tab= first_inner_tab->first_upper; } + if (!tab->bush_children) + i++; } } DBUG_RETURN(0); From d79bf0009a17f0020203003a97ce7e83449aeb3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 20 Jun 2018 01:23:07 +0300 Subject: [PATCH 149/203] MDEV-16525: MyRocks linking fails with: Undefined reference to `ZDICT_trainFromBuffer' RocksDB will only build with libzstd support if libzstd version is >=1.1.13. Unfortunately CMake's FindPackage claims it found version 1.1.13 when we have 1.1.12-1 installed, so a workaround with CheckFunctionExists is used to properly check for correct libzstd support. --- storage/rocksdb/build_rocksdb.cmake | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index c76f711463e..8f01024be63 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -64,10 +64,20 @@ if(SNAPPY_FOUND AND (NOT WITH_ROCKSDB_SNAPPY STREQUAL "OFF")) list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES}) endif() +include(CheckFunctionExists) if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_ZSTD STREQUAL "OFF")) - add_definitions(-DZSTD) - include_directories(${ZSTD_INCLUDE_DIR}) - list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARY}) + SET(CMAKE_REQUIRED_LIBRARIES zstd) + CHECK_FUNCTION_EXISTS(ZDICT_trainFromBuffer ZSTD_VALID) + UNSET(CMAKE_REQUIRED_LIBRARIES) + if (WITH_ROCKSDB_ZSTD STREQUAL "ON" AND NOT ZSTD_VALID) + MESSAGE(FATAL_ERROR + "WITH_ROCKSDB_ZSTD is ON and ZSTD library was found, but the version needs to be >= 1.1.3") + endif() + if (ZSTD_VALID) + add_definitions(-DZSTD) + include_directories(${ZSTD_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARY}) + endif() endif() add_definitions(-DZLIB) @@ -119,7 +129,6 @@ int main() { endif() endif() -include(CheckFunctionExists) CHECK_FUNCTION_EXISTS(malloc_usable_size HAVE_MALLOC_USABLE_SIZE) if(HAVE_MALLOC_USABLE_SIZE) add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE) From 04c47454781533408deb10bf770744327415aa59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 20 Jun 2018 01:28:59 +0300 Subject: [PATCH 150/203] Fix double WSREP_ISOLATION_BEGIN merge error --- sql/sql_plugin.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 7e4be9aed16..270466fa0f7 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2122,8 +2122,6 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, DBUG_RETURN(TRUE); WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) - /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table = open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT))) From 44682962e3fe591d16c6fa966c39e9c738427712 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Jun 2018 11:10:15 +0200 Subject: [PATCH 151/203] Fix another double WSREP_ISOLATION_BEGIN merge error --- sql/sql_plugin.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 270466fa0f7..8d055fd612f 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2263,7 +2263,6 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, if (!opt_noacl && check_table_access(thd, DELETE_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) From bb24663f5ad955a615172512c04779d219bb6645 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Jun 2018 10:45:57 +0200 Subject: [PATCH 152/203] MDEV-13577 slave_parallel_mode=optimistic should not report the mode's specific temporary errors Revert 7bbe324fc17 Fix the bug differently (in log_event.cc) Fix the test case to actually fail without the bug fix --- include/my_sys.h | 1 - .../suite/rpl/r/rpl_parallel_optimistic.result | 4 ++-- .../suite/rpl/t/rpl_parallel_optimistic.test | 7 ++----- sql/handler.cc | 3 --- sql/log_event.cc | 18 ++++++++++++++---- sql/mysqld.cc | 10 ---------- sql/sql_class.cc | 7 ------- sql/sql_class.h | 6 ------ 8 files changed, 18 insertions(+), 38 deletions(-) diff --git a/include/my_sys.h b/include/my_sys.h index 1c5649812d1..110a2ee9af3 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -112,7 +112,6 @@ typedef struct my_aio_result { #define ME_JUST_INFO 1024 /**< not error but just info */ #define ME_JUST_WARNING 2048 /**< not error but just warning */ #define ME_FATALERROR 4096 /* Fatal statement error */ -#define ME_LOG_AS_WARN 8192 /* is error but error-logged as warning */ /* Bits in last argument to fn_format */ #define MY_REPLACE_DIR 1 /* replace dir in name with 'dir' */ diff --git a/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result b/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result index a6da3399fab..3cd4f8231bf 100644 --- a/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result +++ b/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result @@ -1,6 +1,4 @@ include/rpl_init.inc [topology=1->2] -call mtr.add_suppression("Warning.*Deadlock found when trying to get lock; try restarting transaction"); -call mtr.add_suppression("Warning.*mysqld: Can't find record in 't2'"); ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; @@ -549,6 +547,7 @@ DELETE FROM t1; DELETE FROM t2; include/save_master_gtid.inc include/sync_with_master_gtid.inc +set global log_warnings=2; BEGIN; INSERT INTO t1 SET a=1; SET @save.binlog_format=@@session.binlog_format; @@ -567,6 +566,7 @@ DELETE FROM t2; include/save_master_gtid.inc include/sync_with_master_gtid.inc include/stop_slave.inc +set global log_warnings=default; SET GLOBAL slave_parallel_mode=@old_parallel_mode; SET GLOBAL slave_parallel_threads=@old_parallel_threads; include/start_slave.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test b/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test index 28bf4a77fd4..9f6669279db 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test @@ -4,11 +4,6 @@ --let $rpl_topology=1->2 --source include/rpl_init.inc ---connection server_2 -call mtr.add_suppression("Warning.*Deadlock found when trying to get lock; try restarting transaction"); -# The following instruction is a part of the proof of MDEV-13577 fixes, below. -call mtr.add_suppression("Warning.*mysqld: Can't find record in 't2'"); - --connection server_1 ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; @@ -500,6 +495,7 @@ DELETE FROM t2; # The 1st of the following two trx:s a blocker on slave --connection server_2 +set global log_warnings=2; BEGIN; INSERT INTO t1 SET a=1; @@ -546,6 +542,7 @@ DELETE FROM t2; # --connection server_2 --source include/stop_slave.inc +set global log_warnings=default; SET GLOBAL slave_parallel_mode=@old_parallel_mode; SET GLOBAL slave_parallel_threads=@old_parallel_threads; --source include/start_slave.inc diff --git a/sql/handler.cc b/sql/handler.cc index 1027a8b102d..35b0814ef79 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3639,13 +3639,10 @@ void handler::print_error(int error, myf errflag) if ((debug_assert_if_crashed_table || global_system_variables.log_warnings > 1)) { - THD *thd= ha_thd(); /* Log error to log before we crash or if extended warnings are requested */ errflag|= ME_NOREFRESH; - if (thd && thd->is_optimistic_slave_worker()) - errflag|= ME_LOG_AS_WARN; } } diff --git a/sql/log_event.cc b/sql/log_event.cc index e70d97c54c0..3ac7ac5a20f 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -12080,6 +12080,16 @@ void issue_long_find_row_warning(Log_event_type type, } +/* + HA_ERR_KEY_NOT_FOUND is a fatal error normally, but it's an expected + error in speculate optimistic mode, so use something non-fatal instead +*/ +static int row_not_found_error(rpl_group_info *rgi) +{ + return rgi->speculation != rpl_group_info::SPECULATE_OPTIMISTIC + ? HA_ERR_KEY_NOT_FOUND : HA_ERR_RECORD_CHANGED; +} + /** Locate the current row in event's table. @@ -12167,8 +12177,8 @@ int Rows_log_event::find_row(rpl_group_info *rgi) if (error) { DBUG_PRINT("info",("rnd_pos returns error %d",error)); - if (error == HA_ERR_RECORD_DELETED) - error= HA_ERR_KEY_NOT_FOUND; + if (error == HA_ERR_RECORD_DELETED || error == HA_ERR_KEY_NOT_FOUND) + error= row_not_found_error(rgi); table->file->print_error(error, MYF(0)); } DBUG_RETURN(error); @@ -12233,8 +12243,8 @@ int Rows_log_event::find_row(rpl_group_info *rgi) HA_READ_KEY_EXACT))) { DBUG_PRINT("info",("no record matching the key found in the table")); - if (error == HA_ERR_RECORD_DELETED) - error= HA_ERR_KEY_NOT_FOUND; + if (error == HA_ERR_RECORD_DELETED || error == HA_ERR_KEY_NOT_FOUND) + error= row_not_found_error(rgi); table->file->print_error(error, MYF(0)); table->file->ha_index_end(); goto end; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5bf5e3f73fc..5f954f7576d 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3517,16 +3517,6 @@ void my_message_sql(uint error, const char *str, myf MyFlags) level= Sql_condition::WARN_LEVEL_WARN; func= sql_print_warning; } - else if (MyFlags & ME_LOG_AS_WARN) - { - /* - Typical use case is optimistic parallel slave where DA needs to hold - an error condition caused by the current error, but the error-log - level is relaxed to the warning one. - */ - level= Sql_condition::WARN_LEVEL_ERROR; - func= sql_print_warning; - } else { level= Sql_condition::WARN_LEVEL_ERROR; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 0a95d2e2cc5..0f629a2c995 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -7003,13 +7003,6 @@ bool THD::rgi_have_temporary_tables() return rgi_slave->rli->save_temporary_tables != 0; } -bool THD::is_optimistic_slave_worker() -{ - DBUG_ASSERT(system_thread != SYSTEM_THREAD_SLAVE_SQL || rgi_slave); - - return system_thread == SYSTEM_THREAD_SLAVE_SQL && rgi_slave && - rgi_slave->speculation == rpl_group_info::SPECULATE_OPTIMISTIC; -} void wait_for_commit::reinit() diff --git a/sql/sql_class.h b/sql/sql_class.h index fb3604b899b..6d3f0965df0 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -4168,12 +4168,6 @@ public: (THD_TRANS::DID_WAIT | THD_TRANS::CREATED_TEMP_TABLE | THD_TRANS::DROPPED_TEMP_TABLE | THD_TRANS::DID_DDL)); } - - /* - Returns true when the thread handle belongs to a slave worker thread - running in the optimistic execution mode. - */ - bool is_optimistic_slave_worker(); }; From 170b43c1568ba6b9a772e8bdfbe90bd06cc2a345 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 20 Jun 2018 16:36:46 +0400 Subject: [PATCH 153/203] MDEV-16534 PPC64: Unexpected error with a negative value into auto-increment columns in HEAP, MyISAM, ARIA --- mysql-test/r/auto_increment.result | 29 +++++++++++++++++++ .../suite/heap/heap_auto_increment.result | 29 +++++++++++++++++++ .../suite/heap/heap_auto_increment.test | 25 ++++++++++++++++ mysql-test/suite/maria/maria.result | 29 +++++++++++++++++++ mysql-test/suite/maria/maria.test | 25 ++++++++++++++++ mysql-test/t/auto_increment.test | 25 ++++++++++++++++ storage/heap/hp_hash.c | 2 +- storage/maria/ma_key.c | 2 +- storage/myisam/mi_key.c | 2 +- 9 files changed, 165 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/auto_increment.result b/mysql-test/r/auto_increment.result index 12cbf294b69..4c04c00b79f 100644 --- a/mysql-test/r/auto_increment.result +++ b/mysql-test/r/auto_increment.result @@ -537,3 +537,32 @@ pk -5 1 drop table t1; +# +# Start of 5.5 tests +# +# +# MDEV-16534 PPC64: Unexpected error with a negative values into auto-increment columns in HEAP, MyISAM, ARIA +# +CREATE TABLE t1 ( +id TINYINT NOT NULL AUTO_INCREMENT, +name CHAR(30) NOT NULL, +PRIMARY KEY (id) +) ENGINE=MyISAM; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` tinyint(4) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +INSERT INTO t1 (name) VALUES ('dog'); +UPDATE t1 SET id=-1 WHERE id=1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +id name +-1 dog +2 cat +DROP TABLE t1; +# +# End of 5.5 tests +# diff --git a/mysql-test/suite/heap/heap_auto_increment.result b/mysql-test/suite/heap/heap_auto_increment.result index 5b04a77e9eb..ab0c852f253 100644 --- a/mysql-test/suite/heap/heap_auto_increment.result +++ b/mysql-test/suite/heap/heap_auto_increment.result @@ -39,3 +39,32 @@ _rowid _rowid skey sval 1 1 1 hello 2 2 2 hey drop table t1; +# +# Start of 5.5 tests +# +# +# MDEV-16534 PPC64: Unexpected error with a negative values into auto-increment columns in HEAP, MyISAM, ARIA +# +CREATE TABLE t1 ( +id TINYINT NOT NULL AUTO_INCREMENT, +name CHAR(30) NOT NULL, +PRIMARY KEY (id) +) ENGINE=MEMORY; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` tinyint(4) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MEMORY DEFAULT CHARSET=latin1 +INSERT INTO t1 (name) VALUES ('dog'); +UPDATE t1 SET id=-1 WHERE id=1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +id name +-1 dog +2 cat +DROP TABLE t1; +# +# End of 5.5 tests +# diff --git a/mysql-test/suite/heap/heap_auto_increment.test b/mysql-test/suite/heap/heap_auto_increment.test index 016bc946209..bd4a0eaa886 100644 --- a/mysql-test/suite/heap/heap_auto_increment.test +++ b/mysql-test/suite/heap/heap_auto_increment.test @@ -33,3 +33,28 @@ select _rowid,t1._rowid,skey,sval from t1; drop table t1; # End of 4.1 tests + +--echo # +--echo # Start of 5.5 tests +--echo # + +--echo # +--echo # MDEV-16534 PPC64: Unexpected error with a negative values into auto-increment columns in HEAP, MyISAM, ARIA +--echo # + +CREATE TABLE t1 ( + id TINYINT NOT NULL AUTO_INCREMENT, + name CHAR(30) NOT NULL, + PRIMARY KEY (id) +) ENGINE=MEMORY; +SHOW CREATE TABLE t1; +INSERT INTO t1 (name) VALUES ('dog'); +UPDATE t1 SET id=-1 WHERE id=1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +DROP TABLE t1; + +--echo # +--echo # End of 5.5 tests +--echo # + diff --git a/mysql-test/suite/maria/maria.result b/mysql-test/suite/maria/maria.result index 7f8badb78ae..2aca47538cb 100644 --- a/mysql-test/suite/maria/maria.result +++ b/mysql-test/suite/maria/maria.result @@ -2697,3 +2697,32 @@ ALTER TABLE t1 DISABLE KEYS; INSERT INTO t1 (b) VALUES (''); ALTER TABLE t1 ENABLE KEYS; DROP TABLE t1; +# +# Start of 5.5 tests +# +# +# MDEV-16534 PPC64: Unexpected error with a negative values into auto-increment columns in HEAP, MyISAM, ARIA +# +CREATE TABLE t1 ( +id TINYINT NOT NULL AUTO_INCREMENT, +name CHAR(30) NOT NULL, +PRIMARY KEY (id) +) ENGINE=ARIA; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` tinyint(4) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 +INSERT INTO t1 (name) VALUES ('dog'); +UPDATE t1 SET id=-1 WHERE id=1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +id name +-1 dog +2 cat +DROP TABLE t1; +# +# End of 5.5 tests +# diff --git a/mysql-test/suite/maria/maria.test b/mysql-test/suite/maria/maria.test index 39a9a06c4c2..ab8e72dc321 100644 --- a/mysql-test/suite/maria/maria.test +++ b/mysql-test/suite/maria/maria.test @@ -1990,3 +1990,28 @@ aria_page_checksum=$default_checksum, aria_log_file_size=$default_log_file_size; --enable_result_log --enable_query_log + + +--echo # +--echo # Start of 5.5 tests +--echo # + +--echo # +--echo # MDEV-16534 PPC64: Unexpected error with a negative values into auto-increment columns in HEAP, MyISAM, ARIA +--echo # + +CREATE TABLE t1 ( + id TINYINT NOT NULL AUTO_INCREMENT, + name CHAR(30) NOT NULL, + PRIMARY KEY (id) +) ENGINE=ARIA; +SHOW CREATE TABLE t1; +INSERT INTO t1 (name) VALUES ('dog'); +UPDATE t1 SET id=-1 WHERE id=1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +DROP TABLE t1; + +--echo # +--echo # End of 5.5 tests +--echo # diff --git a/mysql-test/t/auto_increment.test b/mysql-test/t/auto_increment.test index 7f0ab5dc169..2fea2cfd351 100644 --- a/mysql-test/t/auto_increment.test +++ b/mysql-test/t/auto_increment.test @@ -397,3 +397,28 @@ insert into t1 values(null); select last_insert_id(); select * from t1; drop table t1; + + +--echo # +--echo # Start of 5.5 tests +--echo # + +--echo # +--echo # MDEV-16534 PPC64: Unexpected error with a negative values into auto-increment columns in HEAP, MyISAM, ARIA +--echo # + +CREATE TABLE t1 ( + id TINYINT NOT NULL AUTO_INCREMENT, + name CHAR(30) NOT NULL, + PRIMARY KEY (id) +) ENGINE=MyISAM; +SHOW CREATE TABLE t1; +INSERT INTO t1 (name) VALUES ('dog'); +UPDATE t1 SET id=-1 WHERE id=1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +DROP TABLE t1; + +--echo # +--echo # End of 5.5 tests +--echo # diff --git a/storage/heap/hp_hash.c b/storage/heap/hp_hash.c index 2d4c0e9a031..ce59b79f990 100644 --- a/storage/heap/hp_hash.c +++ b/storage/heap/hp_hash.c @@ -1012,7 +1012,7 @@ void heap_update_auto_increment(HP_INFO *info, const uchar *record) switch (info->s->auto_key_type) { case HA_KEYTYPE_INT8: - s_value= (longlong) *(char*)key; + s_value= (longlong) *(const signed char*) key; break; case HA_KEYTYPE_BINARY: value=(ulonglong) *(uchar*) key; diff --git a/storage/maria/ma_key.c b/storage/maria/ma_key.c index 246e5fb6d94..42203e5d0f3 100644 --- a/storage/maria/ma_key.c +++ b/storage/maria/ma_key.c @@ -728,7 +728,7 @@ ulonglong ma_retrieve_auto_increment(const uchar *key, uint8 key_type) switch (key_type) { case HA_KEYTYPE_INT8: - s_value= (longlong) *(const char*)key; + s_value= (longlong) *(const signed char*) key; break; case HA_KEYTYPE_BINARY: value=(ulonglong) *key; diff --git a/storage/myisam/mi_key.c b/storage/myisam/mi_key.c index 9a2526ad2cf..18ecc9e8ba3 100644 --- a/storage/myisam/mi_key.c +++ b/storage/myisam/mi_key.c @@ -553,7 +553,7 @@ ulonglong retrieve_auto_increment(MI_INFO *info,const uchar *record) switch (keyseg->type) { case HA_KEYTYPE_INT8: - s_value= (longlong) *(char*)key; + s_value= (longlong) *(const signed char*) key; break; case HA_KEYTYPE_BINARY: value=(ulonglong) *(uchar*) key; From 621caad3ca007f84ae3d9216b8a23219b334347e Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Wed, 20 Jun 2018 17:14:04 +0400 Subject: [PATCH 154/203] MDEV-11917 enum/set command-line options aren't respecting max-* settings. --- .../suite/sys_vars/r/maximum_basic.result | 30 ++++ .../suite/sys_vars/t/maximum_basic-master.opt | 2 + .../suite/sys_vars/t/maximum_basic.test | 19 +++ sql/set_var.cc | 24 +++- sql/set_var.h | 6 + sql/sys_vars.ic | 129 +++++++++++------- sql/sys_vars_shared.h | 1 + 7 files changed, 155 insertions(+), 56 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/maximum_basic.result b/mysql-test/suite/sys_vars/r/maximum_basic.result index 20b6bbc962e..becd6da91f4 100644 --- a/mysql-test/suite/sys_vars/r/maximum_basic.result +++ b/mysql-test/suite/sys_vars/r/maximum_basic.result @@ -16,3 +16,33 @@ Warning 1292 Truncated incorrect max_join_size value: '40960' SELECT @@session.max_join_size; @@session.max_join_size 8192 +SET @@session.use_stat_tables= COMPLEMENTARY; +SELECT @@session.use_stat_tables; +@@session.use_stat_tables +COMPLEMENTARY +SET @@session.use_stat_tables= PREFERABLY; +Warnings: +Warning 1292 Truncated incorrect use_stat_tables value: 'PREFERABLY' +SELECT @@session.use_stat_tables; +@@session.use_stat_tables +COMPLEMENTARY +SET @@session.use_stat_tables= 2; +Warnings: +Warning 1292 Truncated incorrect use_stat_tables value: '2' +SELECT @@session.use_stat_tables; +@@session.use_stat_tables +COMPLEMENTARY +SET @@session.sql_mode= 'REAL_AS_FLOAT'; +SELECT @@session.sql_mode; +@@session.sql_mode +REAL_AS_FLOAT +SET @@session.sql_mode= 'REAL_AS_FLOAT,ANSI_QUOTES'; +SELECT @@session.sql_mode; +@@session.sql_mode +REAL_AS_FLOAT,ANSI_QUOTES +SET @@session.sql_mode= 'ANSI_QUOTES,IGNORE_SPACE'; +Warnings: +Warning 1292 Truncated incorrect sql_mode value: 'ANSI_QUOTES,IGNORE_SPACE' +SELECT @@session.sql_mode; +@@session.sql_mode +ANSI_QUOTES diff --git a/mysql-test/suite/sys_vars/t/maximum_basic-master.opt b/mysql-test/suite/sys_vars/t/maximum_basic-master.opt index b6e5666f4fb..16e365d491c 100644 --- a/mysql-test/suite/sys_vars/t/maximum_basic-master.opt +++ b/mysql-test/suite/sys_vars/t/maximum_basic-master.opt @@ -1,3 +1,5 @@ --maximum-auto-increment-increment=8192 --maximum-tmp-table-size=8192 --maximum-max-join-size=8192 +--maximum-use-stat-tables=COMPLEMENTARY +--maximum-sql-mode='REAL_AS_FLOAT,ANSI_QUOTES' diff --git a/mysql-test/suite/sys_vars/t/maximum_basic.test b/mysql-test/suite/sys_vars/t/maximum_basic.test index 9961f65883a..3153d62d562 100644 --- a/mysql-test/suite/sys_vars/t/maximum_basic.test +++ b/mysql-test/suite/sys_vars/t/maximum_basic.test @@ -18,3 +18,22 @@ SELECT @@session.tmp_table_size; SET @@session.max_join_size=40960; SELECT @@session.max_join_size; +# +# enum +# +SET @@session.use_stat_tables= COMPLEMENTARY; +SELECT @@session.use_stat_tables; +SET @@session.use_stat_tables= PREFERABLY; +SELECT @@session.use_stat_tables; +SET @@session.use_stat_tables= 2; +SELECT @@session.use_stat_tables; + +# +# set +# +SET @@session.sql_mode= 'REAL_AS_FLOAT'; +SELECT @@session.sql_mode; +SET @@session.sql_mode= 'REAL_AS_FLOAT,ANSI_QUOTES'; +SELECT @@session.sql_mode; +SET @@session.sql_mode= 'ANSI_QUOTES,IGNORE_SPACE'; +SELECT @@session.sql_mode; diff --git a/sql/set_var.cc b/sql/set_var.cc index e96e636e3d3..77036fd0f5a 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -452,6 +452,22 @@ void sys_var::do_deprecated_warning(THD *thd) @retval true on error, false otherwise (warning or ok) */ + + +bool throw_bounds_warning(THD *thd, const char *name,const char *v) +{ + if (thd->variables.sql_mode & MODE_STRICT_ALL_TABLES) + { + my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name, v); + return true; + } + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_TRUNCATED_WRONG_VALUE, + ER_THD(thd, ER_TRUNCATED_WRONG_VALUE), name, v); + return false; +} + + bool throw_bounds_warning(THD *thd, const char *name, bool fixed, bool is_unsigned, longlong v) { @@ -469,9 +485,7 @@ bool throw_bounds_warning(THD *thd, const char *name, my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name, buf); return true; } - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_TRUNCATED_WRONG_VALUE, - ER_THD(thd, ER_TRUNCATED_WRONG_VALUE), name, buf); + return throw_bounds_warning(thd, name, buf); } return false; } @@ -489,9 +503,7 @@ bool throw_bounds_warning(THD *thd, const char *name, bool fixed, double v) my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name, buf); return true; } - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_TRUNCATED_WRONG_VALUE, - ER_THD(thd, ER_TRUNCATED_WRONG_VALUE), name, buf); + return throw_bounds_warning(thd, name, buf); } return false; } diff --git a/sql/set_var.h b/sql/set_var.h index 7a48e319466..8dd2cb073b7 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -243,6 +243,12 @@ protected: uchar *global_var_ptr() { return ((uchar*)&global_system_variables) + offset; } + void *max_var_ptr() + { + return scope() == SESSION ? (((uchar*)&max_system_variables) + offset) : + 0; + } + friend class Session_sysvars_tracker; friend class Session_tracker; }; diff --git a/sql/sys_vars.ic b/sql/sys_vars.ic index bc913b1adbd..b92867f98e1 100644 --- a/sql/sys_vars.ic +++ b/sql/sys_vars.ic @@ -142,9 +142,10 @@ public: option.min_value= min_val; option.max_value= max_val; option.block_size= block_size; - option.u_max_value= (uchar**)max_var_ptr(); - if (max_var_ptr()) - *max_var_ptr()= max_val; + if ((option.u_max_value= (uchar**) max_var_ptr())) + { + *((T*) option.u_max_value)= max_val; + } global_var(T)= def_val; SYSVAR_ASSERT(size == sizeof(T)); @@ -176,8 +177,8 @@ public: var->save_result.ulonglong_value= getopt_ull_limit_value(uv, &option, &unused); - if (max_var_ptr() && (T)var->save_result.ulonglong_value > *max_var_ptr()) - var->save_result.ulonglong_value= *max_var_ptr(); + if (max_var_ptr() && (T)var->save_result.ulonglong_value > get_max_var()) + var->save_result.ulonglong_value= get_max_var(); fixed= fixed || var->save_result.ulonglong_value != uv; } @@ -193,8 +194,8 @@ public: var->save_result.longlong_value= getopt_ll_limit_value(v, &option, &unused); - if (max_var_ptr() && (T)var->save_result.longlong_value > *max_var_ptr()) - var->save_result.longlong_value= *max_var_ptr(); + if (max_var_ptr() && (T)var->save_result.longlong_value > get_max_var()) + var->save_result.longlong_value= get_max_var(); fixed= fixed || var->save_result.longlong_value != v; } @@ -216,11 +217,7 @@ public: void global_save_default(THD *thd, set_var *var) { var->save_result.ulonglong_value= option.def_value; } private: - T *max_var_ptr() - { - return scope() == SESSION ? (T*)(((uchar*)&max_system_variables) + offset) - : 0; - } + T get_max_var() { return *((T*) max_var_ptr()); } uchar *default_value_ptr(THD *thd) { return (uchar*) &option.def_value; } }; @@ -264,6 +261,9 @@ class Sys_var_typelib: public sys_var { protected: TYPELIB typelib; + virtual bool check_maximum(THD *thd, set_var *var, + const char *c_val, longlong i_val) + { return FALSE; } public: Sys_var_typelib(const char *name_arg, const char *comment, int flag_args, ptrdiff_t off, @@ -299,17 +299,14 @@ public: return true; else var->save_result.ulonglong_value--; - } - else - { - longlong tmp=var->value->val_int(); - if (tmp < 0 || tmp >= typelib.count) - return true; - else - var->save_result.ulonglong_value= tmp; + return check_maximum(thd, var, res->ptr(), 0); } - return false; + longlong tmp=var->value->val_int(); + if (tmp < 0 || tmp >= typelib.count) + return true; + var->save_result.ulonglong_value= tmp; + return check_maximum(thd, var, 0, tmp); } }; @@ -345,9 +342,25 @@ public: { option.var_type|= GET_ENUM; global_var(ulong)= def_val; + if ((option.u_max_value= (uchar**)max_var_ptr())) + { + *((ulong *) option.u_max_value)= ULONG_MAX; + } SYSVAR_ASSERT(def_val < typelib.count); SYSVAR_ASSERT(size == sizeof(ulong)); } + bool check_maximum(THD *thd, set_var *var, + const char *c_val, longlong i_val) + { + if (!max_var_ptr() || + var->save_result.ulonglong_value <= get_max_var()) + return FALSE; + var->save_result.ulonglong_value= get_max_var(); + + return c_val ? throw_bounds_warning(thd, name.str, c_val) : + throw_bounds_warning(thd, name.str, TRUE, + var->value->unsigned_flag, i_val); + } bool session_update(THD *thd, set_var *var) { session_var(thd, ulong)= static_cast(var->save_result.ulonglong_value); @@ -370,6 +383,8 @@ public: { return valptr(thd, global_var(ulong)); } uchar *default_value_ptr(THD *thd) { return valptr(thd, (ulong)option.def_value); } + + ulong get_max_var() { return *((ulong *) max_var_ptr()); } }; /** @@ -1335,11 +1350,27 @@ public: { option.var_type|= GET_SET; global_var(ulonglong)= def_val; + if ((option.u_max_value= (uchar**)max_var_ptr())) + { + *((ulonglong*) option.u_max_value)= ~0ULL; + } SYSVAR_ASSERT(typelib.count > 0); SYSVAR_ASSERT(typelib.count <= 64); SYSVAR_ASSERT(def_val <= my_set_bits(typelib.count)); SYSVAR_ASSERT(size == sizeof(ulonglong)); } + bool check_maximum(THD *thd, set_var *var, + const char *c_val, longlong i_val) + { + if (!max_var_ptr() || + (var->save_result.ulonglong_value & ~(get_max_var())) == 0) + return FALSE; + var->save_result.ulonglong_value&= get_max_var(); + + return c_val ? throw_bounds_warning(thd, name.str, c_val) : + throw_bounds_warning(thd, name.str, TRUE, + var->value->unsigned_flag, i_val); + } bool do_check(THD *thd, set_var *var) { char buff[STRING_BUFFER_USUAL_SIZE]; @@ -1347,41 +1378,37 @@ public: if (var->value->result_type() == STRING_RESULT) { + char *error; + uint error_len; + bool not_used; + if (!(res=var->value->val_str_ascii(&str))) return true; - else + + var->save_result.ulonglong_value= + find_set(&typelib, res->ptr(), res->length(), NULL, + &error, &error_len, ¬_used); + /* + note, we only issue an error if error_len > 0. + That is even while empty (zero-length) values are considered + errors by find_set(), these errors are ignored here + */ + if (error_len) { - char *error; - uint error_len; - bool not_used; - - var->save_result.ulonglong_value= - find_set(&typelib, res->ptr(), res->length(), NULL, - &error, &error_len, ¬_used); - /* - note, we only issue an error if error_len > 0. - That is even while empty (zero-length) values are considered - errors by find_set(), these errors are ignored here - */ - if (error_len) - { - ErrConvString err(error, error_len, res->charset()); - my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name.str, err.ptr()); - return true; - } - } - } - else - { - longlong tmp=var->value->val_int(); - if ((tmp < 0 && ! var->value->unsigned_flag) - || (ulonglong)tmp > my_set_bits(typelib.count)) + ErrConvString err(error, error_len, res->charset()); + my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name.str, err.ptr()); return true; - else - var->save_result.ulonglong_value= tmp; + } + return check_maximum(thd, var, res->ptr(), 0); } - return false; + longlong tmp=var->value->val_int(); + if ((tmp < 0 && ! var->value->unsigned_flag) + || (ulonglong)tmp > my_set_bits(typelib.count)) + return true; + + var->save_result.ulonglong_value= tmp; + return check_maximum(thd, var, 0, tmp); } bool session_update(THD *thd, set_var *var) { @@ -1405,6 +1432,8 @@ public: { return valptr(thd, global_var(ulonglong)); } uchar *default_value_ptr(THD *thd) { return valptr(thd, option.def_value); } + + ulonglong get_max_var() { return *((ulonglong*) max_var_ptr()); } }; /** diff --git a/sql/sys_vars_shared.h b/sql/sys_vars_shared.h index ff050f63064..dfc020a187c 100644 --- a/sql/sys_vars_shared.h +++ b/sql/sys_vars_shared.h @@ -28,6 +28,7 @@ #include #include "set_var.h" +extern bool throw_bounds_warning(THD *thd, const char *name,const char *v); extern bool throw_bounds_warning(THD *thd, const char *name, bool fixed, bool is_unsigned, longlong v); extern bool throw_bounds_warning(THD *thd, const char *name, bool fixed, From be9d923af2b1b9bfe4ce3e2cf06ead957cf6129c Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Wed, 20 Jun 2018 22:45:10 +0400 Subject: [PATCH 155/203] MDEV-11917 enum/set command-line options aren't respecting max-* settings. my_option strucures for command-line variables should be set properly. --- sql/sys_vars.ic | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sql/sys_vars.ic b/sql/sys_vars.ic index b92867f98e1..ffd08608e19 100644 --- a/sql/sys_vars.ic +++ b/sql/sys_vars.ic @@ -341,6 +341,8 @@ public: substitute) { option.var_type|= GET_ENUM; + option.min_value= 0; + option.max_value= ULONG_MAX; global_var(ulong)= def_val; if ((option.u_max_value= (uchar**)max_var_ptr())) { @@ -1349,6 +1351,8 @@ public: substitute) { option.var_type|= GET_SET; + option.min_value= 0; + option.max_value= ~0ULL; global_var(ulonglong)= def_val; if ((option.u_max_value= (uchar**)max_var_ptr())) { From 0a9d78f51d74be7708f2efd940311bf7b33108e9 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Jun 2018 23:27:23 +0200 Subject: [PATCH 156/203] Revert "MDEV-16075: Workaround to run MTR test suite for make test" This reverts commit d39629f01ebdd5b89186e6c8a4a8d3dd528bd26a. Because running mtr for many hours with no output whatsoever is not really what we should do. And in 5.5 `make test` just works anyway, nothing to fix here. --- CMakeLists.txt | 4 ---- cmake/ctest.cmake | 2 +- mysql-test/suite/unit/suite.pm | 2 +- unittest/sql/CMakeLists.txt | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 21ed1fcc24b..6ab17ebf64c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -366,10 +366,6 @@ INCLUDE(maintainer) IF(WITH_UNIT_TESTS) ENABLE_TESTING() - # This is the only instance where ADD_TEST should be used, - # to make sure that make test will run MTR, - # use MY_ADD_TEST macro to add other tests - ADD_TEST(NAME MTR COMMAND ./mysql-test-run WORKING_DIRECTORY "mysql-test") ADD_SUBDIRECTORY(unittest/mytap) ADD_SUBDIRECTORY(unittest/strings) ADD_SUBDIRECTORY(unittest/examples) diff --git a/cmake/ctest.cmake b/cmake/ctest.cmake index fde7e1632f6..08852a366f6 100644 --- a/cmake/ctest.cmake +++ b/cmake/ctest.cmake @@ -2,7 +2,7 @@ INCLUDE(${MYSQL_CMAKE_SCRIPT_DIR}/cmake_parse_arguments.cmake) MACRO(MY_ADD_TEST name) - ADD_TEST(NAME ${name} COMMAND ${name}-t CONFIGURATIONS default_ignore) + ADD_TEST(${name} ${name}-t) ENDMACRO() MACRO (MY_ADD_TESTS) diff --git a/mysql-test/suite/unit/suite.pm b/mysql-test/suite/unit/suite.pm index 43eaa970350..78d82ccb31d 100644 --- a/mysql-test/suite/unit/suite.pm +++ b/mysql-test/suite/unit/suite.pm @@ -31,7 +31,7 @@ sub start_test { return "Not run for embedded server" if $::opt_embedded_server; return "Not configured to run ctest" unless -f "../CTestTestfile.cmake"; my ($ctest_vs)= $opt_vs_config ? "--build-config $opt_vs_config" : ""; - my (@ctest_list)= `cd .. && ctest $opt_vs_config -E MTR -C default_ignore --show-only --verbose`; + my (@ctest_list)= `cd .. && ctest $opt_vs_config --show-only --verbose`; return "No ctest" if $?; my ($command, %tests); diff --git a/unittest/sql/CMakeLists.txt b/unittest/sql/CMakeLists.txt index 1efb0ac9cd1..f80f2a5ae70 100644 --- a/unittest/sql/CMakeLists.txt +++ b/unittest/sql/CMakeLists.txt @@ -26,4 +26,4 @@ ELSE() ENDIF() TARGET_LINK_LIBRARIES(explain_filename-t sql mytap) -MY_ADD_TEST(explain_filename explain_filename-t) +ADD_TEST(explain_filename explain_filename-t) From 2b8f2b3e882f7ea93c6743f2ad717f968bd245fe Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Jun 2018 23:30:49 +0200 Subject: [PATCH 157/203] Fix unit suite on Windows and in out-of-source builds --- mysql-test/suite/unit/suite.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/unit/suite.pm b/mysql-test/suite/unit/suite.pm index 78d82ccb31d..ce7ccefe926 100644 --- a/mysql-test/suite/unit/suite.pm +++ b/mysql-test/suite/unit/suite.pm @@ -28,10 +28,11 @@ sub start_test { } { + my $bin=$ENV{MTR_BINDIR} || '..'; return "Not run for embedded server" if $::opt_embedded_server; - return "Not configured to run ctest" unless -f "../CTestTestfile.cmake"; - my ($ctest_vs)= $opt_vs_config ? "--build-config $opt_vs_config" : ""; - my (@ctest_list)= `cd .. && ctest $opt_vs_config --show-only --verbose`; + return "Not configured to run ctest" unless -f "$bin/CTestTestfile.cmake"; + my ($ctest_vs)= $::opt_vs_config ? "-C ".substr($::opt_vs_config,1) : ""; + my (@ctest_list)= `cd $bin && ctest $ctest_vs --show-only --verbose`; return "No ctest" if $?; my ($command, %tests); From 1033fa4bccb67b42b9baae189fc95bc93f821395 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 12 Jun 2018 19:06:55 +0200 Subject: [PATCH 158/203] MDEV-13403 Mariadb (with TokuDB) excessive memory usage/leak In RPM/DEB packages - always ld-preload jemalloc, instead of linking ha_tokudb.so with it. Keep linking in bintars, because they don't install cnf files in the correct locations. --- debian/control | 1 + storage/tokudb/CMakeLists.txt | 2 +- storage/tokudb/tokudb.cnf.in | 1 - storage/tokudb/tokudb.conf.in | 1 - 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/debian/control b/debian/control index cc52a1c240b..20ccd4c163e 100644 --- a/debian/control +++ b/debian/control @@ -477,6 +477,7 @@ Description: OQGraph storage engine for MariaDB Package: mariadb-plugin-tokudb Architecture: any Depends: mariadb-server-10.2 (= ${binary:Version}), + libjemalloc1 (>= 3.0.0~), ${misc:Depends}, ${shlibs:Depends} Breaks: mariadb-server-10.0, diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 16f7ab4ce75..22c2cd0d490 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -54,7 +54,7 @@ ELSEIF(LIBJEMALLOC STREQUAL jemalloc) GET_FILENAME_COMPONENT(LIBJEMALLOC_PATH ${LIBJEMALLOC_SO} REALPATH CACHE) ENDIF() -IF(LIBJEMALLOC_PATH AND RPM MATCHES fedora28) # TODO check for jemalloc version +IF(LIBJEMALLOC_PATH AND (RPM OR DEB)) UNSET(LIBJEMALLOC) GET_DIRECTORY_PROPERTY(V DIRECTORY ${CMAKE_SOURCE_DIR} DEFINITION CPACK_RPM_tokudb-engine_PACKAGE_REQUIRES) SET(CPACK_RPM_tokudb-engine_PACKAGE_REQUIRES "${V} jemalloc" PARENT_SCOPE) diff --git a/storage/tokudb/tokudb.cnf.in b/storage/tokudb/tokudb.cnf.in index de9b5b711ee..ff7f0a5f5f6 100644 --- a/storage/tokudb/tokudb.cnf.in +++ b/storage/tokudb/tokudb.cnf.in @@ -5,5 +5,4 @@ plugin-load-add=ha_tokudb.so [mysqld_safe] -# it might be necessary to uncomment the following line if jemalloc >= 5.0.0 @cnf_malloc_lib@ diff --git a/storage/tokudb/tokudb.conf.in b/storage/tokudb/tokudb.conf.in index d22f6686d91..a5ff055f44c 100644 --- a/storage/tokudb/tokudb.conf.in +++ b/storage/tokudb/tokudb.conf.in @@ -1,3 +1,2 @@ [Service] -# it might be necessary to uncomment the following line if jemalloc >= 5.0.0 @systemd_env@ From 53db5edbcf7cc736e9102f3b5c15907278c42450 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 18 May 2018 15:10:52 +1000 Subject: [PATCH 159/203] MDEV-14578: mysql_install_db install unix_socket plugin when --auth-root-authentication-method=socket $ cmake -DPLUGIN_AUTH_SOCKET=STATIC ../mariadb-server-10.2/ $ scripts/mysql_install_db --datadir=/tmp/mysqldatadir-auth-static --auth-root-socket-user=dan --auth-root-authentication-method=socket --builddir=. --srcdir=../mariadb-server-10.2 Installing MariaDB/MySQL system tables in '/tmp/mysqldatadir-auth-static' ... OK $ client/mysql -S /tmp/mysql.sock -e 'show create user dan@localhost' +----------------------------------------------------------+ | CREATE USER for dan@localhost | +----------------------------------------------------------+ | CREATE USER 'dan'@'localhost' IDENTIFIED VIA unix_socket | +----------------------------------------------------------+ $ cmake -DPLUGIN_AUTH_SOCKET=NO ../mariadb-server-10.2/ $ scripts/mysql_install_db --datadir=/tmp/mysqldatadir-auth-none --auth-root-socket-user=dan --auth-root-authentication-method=socket --builddir=. --srcdir=../mariadb-server-10.2 Installing MariaDB/MySQL system tables in '/tmp/mysqldatadir-auth-none' ... ERROR: 1126 Can't open shared library '.../plugin/auth_socket/auth_socket.so' (errno: 2, cannot open shared object file: No such file or directory) 2018-05-01 11:38:56 0 [ERROR] Aborting $ cmake -DPLUGIN_AUTH_SOCKET=DYNAMIC ../mariadb-server-10.2/ $ scripts/mysql_install_db --datadir=/tmp/mysqldatadir-auth-dyn --auth-root-socket-user=dan --auth-root-authentication-method=socket --builddir=. --srcdir=../mariadb-server-10.2 Installing MariaDB/MySQL system tables in '/tmp/mysqldatadir-auth-dyn' ... OK $ ./sql/mysqld --datadir=/tmp/mysqldatadir-auth-dyn --lc-messages-dir=${PWD}/sql/share --plugin-dir=./plugin/auth_socket/ $ client/mysql -S /tmp/mysql.sock -e 'show create user dan@localhost' +----------------------------------------------------------+ | CREATE USER for dan@localhost | +----------------------------------------------------------+ | CREATE USER 'dan'@'localhost' IDENTIFIED VIA unix_socket | +----------------------------------------------------------+ $ sudo make install $ sudo chmod a+rwx /usr/local/mysql/data $ cd /usr/local/mysql/ $ scripts/mysql_install_db --auth-root-socket-user=dan --auth-root-authentication-method=socket Installing MariaDB/MySQL system tables in './data' ... OK .. $ bin/mysqld_safe $ client/mysql -S /tmp/mysql.sock -e 'show create user dan@localhost' +----------------------------------------------------------+ | CREATE USER for dan@localhost | +----------------------------------------------------------+ | CREATE USER 'dan'@'localhost' IDENTIFIED VIA unix_socket | +----------------------------------------------------------+ Merges #767 --- scripts/mysql_install_db.sh | 4 ++++ scripts/mysql_system_tables_data.sql | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index a84b8287e9e..9e3bc35423a 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -307,6 +307,7 @@ then langdir="$basedir/sql/share/english" srcpkgdatadir="$srcdir/scripts" buildpkgdatadir="$builddir/scripts" + plugindir="$builddir/plugin/auth_socket/" elif test -n "$basedir" then bindir="$basedir/bin" # only used in the help text @@ -335,6 +336,7 @@ then cannot_find_file fill_help_tables.sql @pkgdata_locations@ exit 1 fi + plugindir="$basedir/@INSTALL_PLUGINDIR@" else basedir="@prefix@" bindir="@bindir@" @@ -342,6 +344,7 @@ else mysqld="@libexecdir@/mysqld" srcpkgdatadir="@pkgdatadir@" buildpkgdatadir="@pkgdatadir@" + plugindir="@INSTALL_PLUGINDIR@" fi # Set up paths to SQL scripts required for bootstrap @@ -460,6 +463,7 @@ mysqld_install_cmd_line() { "$mysqld_bootstrap" $defaults $defaults_group_suffix "$mysqld_opt" --bootstrap $silent_startup\ "--basedir=$basedir" "--datadir=$ldata" --log-warnings=0 --enforce-storage-engine="" \ + "--plugin-dir=${plugindir}" \ $args --max_allowed_packet=8M \ --net_buffer_length=16K } diff --git a/scripts/mysql_system_tables_data.sql b/scripts/mysql_system_tables_data.sql index 9556e7ba160..e5efd48f918 100644 --- a/scripts/mysql_system_tables_data.sql +++ b/scripts/mysql_system_tables_data.sql @@ -46,8 +46,13 @@ INSERT INTO tmp_user_nopasswd VALUES ('localhost','root','','Y','Y','Y','Y','Y', REPLACE INTO tmp_user_nopasswd SELECT @current_hostname,'root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0 FROM dual WHERE @current_hostname != 'localhost'; REPLACE INTO tmp_user_nopasswd VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0); REPLACE INTO tmp_user_nopasswd VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0); --- More secure root account using unix sucket auth. +-- More secure root account using unix socket auth. INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root'),'','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'unix_socket','','N', 'N','', 0); +-- Need aria support to lookup information_schema.plugins (result is tmptable) +IF @auth_root_socket is not null THEN + IF exists (SELECT 1 FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'aria' AND support IN ('YES', 'DEFAULT', 'ENABLED')) THEN + IF not exists(select 1 from information_schema.plugins where plugin_name='unix_socket') THEN + INSTALL SONAME 'auth_socket'; END IF; END IF; END IF; -- Anonymous user with no privileges. INSERT INTO tmp_user_anonymous (host,user) VALUES ('localhost',''); INSERT INTO tmp_user_anonymous (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost'; From 1db1340c0c5302e16c88528d89f583f4f1b7bc90 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 18 Jun 2018 11:51:27 +0200 Subject: [PATCH 160/203] MDEV-14578: mysql_install_db install unix_socket plugin when --auth-root-authentication-method=socket post-merge fixes --- ...33_scripts__mysql_create_system_tables__no_test.dpatch | 8 ++++---- scripts/CMakeLists.txt | 2 ++ scripts/mysql_install_db.sh | 6 +++--- scripts/mysql_system_tables_data.sql | 6 ++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/debian/patches/33_scripts__mysql_create_system_tables__no_test.dpatch b/debian/patches/33_scripts__mysql_create_system_tables__no_test.dpatch index 183212ef678..a961cfce8a0 100755 --- a/debian/patches/33_scripts__mysql_create_system_tables__no_test.dpatch +++ b/debian/patches/33_scripts__mysql_create_system_tables__no_test.dpatch @@ -27,10 +27,10 @@ -- Fill "user" table with default users allowing root access -- from local machine if "user" table didn't exist before CREATE TEMPORARY TABLE tmp_user_nopasswd LIKE user; -@@ -48,9 +38,6 @@ REPLACE INTO tmp_user_nopasswd VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y' - REPLACE INTO tmp_user_nopasswd VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0); - -- More secure root account using unix sucket auth. - INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root'),'','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'unix_socket','','N', 'N','', 0); +@@ -51,9 +41,6 @@ INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root' + IF @auth_root_socket is not null THEN + IF not exists(select 1 from information_schema.plugins where plugin_name='unix_socket') THEN + INSTALL SONAME 'auth_socket'; END IF; END IF; --- Anonymous user with no privileges. -INSERT INTO tmp_user_anonymous (host,user) VALUES ('localhost',''); -INSERT INTO tmp_user_anonymous (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost'; diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index fc133f53b37..fb09e459f87 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -174,6 +174,7 @@ IF(INSTALL_LAYOUT MATCHES "STANDALONE") SET(scriptdir ${prefix}/${INSTALL_BINDIR}) SET(libexecdir ${prefix}/${INSTALL_SBINDIR}) SET(pkgdatadir ${prefix}/${INSTALL_MYSQLSHAREDIR}) + SET(pkgplugindir ${prefix}/${INSTALL_PLUGINDIR}) SET(localstatedir ${prefix}/data) ELSE() SET(prefix "${CMAKE_INSTALL_PREFIX}") @@ -182,6 +183,7 @@ ELSE() SET(scriptdir ${INSTALL_BINDIRABS}) SET(libexecdir ${INSTALL_SBINDIRABS}) SET(pkgdatadir ${INSTALL_MYSQLSHAREDIRABS}) + SET(pkgplugindir ${INSTALL_PLUGINDIRABS}) SET(localstatedir ${MYSQL_DATADIR}) ENDIF() diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index 9e3bc35423a..0a596e1e03b 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -307,7 +307,7 @@ then langdir="$basedir/sql/share/english" srcpkgdatadir="$srcdir/scripts" buildpkgdatadir="$builddir/scripts" - plugindir="$builddir/plugin/auth_socket/" + plugindir="$builddir/plugin/auth_socket" elif test -n "$basedir" then bindir="$basedir/bin" # only used in the help text @@ -336,7 +336,7 @@ then cannot_find_file fill_help_tables.sql @pkgdata_locations@ exit 1 fi - plugindir="$basedir/@INSTALL_PLUGINDIR@" + plugindir=`find_in_dirs --dir auth_socket.so $basedir/lib*/plugin $basedir/lib*/mysql/plugin` else basedir="@prefix@" bindir="@bindir@" @@ -344,7 +344,7 @@ else mysqld="@libexecdir@/mysqld" srcpkgdatadir="@pkgdatadir@" buildpkgdatadir="@pkgdatadir@" - plugindir="@INSTALL_PLUGINDIR@" + plugindir="@pkgplugindir@" fi # Set up paths to SQL scripts required for bootstrap diff --git a/scripts/mysql_system_tables_data.sql b/scripts/mysql_system_tables_data.sql index e5efd48f918..2835a7f8b77 100644 --- a/scripts/mysql_system_tables_data.sql +++ b/scripts/mysql_system_tables_data.sql @@ -48,11 +48,9 @@ REPLACE INTO tmp_user_nopasswd VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y' REPLACE INTO tmp_user_nopasswd VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0); -- More secure root account using unix socket auth. INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root'),'','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'unix_socket','','N', 'N','', 0); --- Need aria support to lookup information_schema.plugins (result is tmptable) IF @auth_root_socket is not null THEN - IF exists (SELECT 1 FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'aria' AND support IN ('YES', 'DEFAULT', 'ENABLED')) THEN - IF not exists(select 1 from information_schema.plugins where plugin_name='unix_socket') THEN - INSTALL SONAME 'auth_socket'; END IF; END IF; END IF; + IF not exists(select 1 from information_schema.plugins where plugin_name='unix_socket') THEN + INSTALL SONAME 'auth_socket'; END IF; END IF; -- Anonymous user with no privileges. INSERT INTO tmp_user_anonymous (host,user) VALUES ('localhost',''); INSERT INTO tmp_user_anonymous (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost'; From b4db59ba47f7670c5ea90962e36afb9dede0d238 Mon Sep 17 00:00:00 2001 From: Georg Richter Date: Mon, 18 Jun 2018 19:06:55 +0200 Subject: [PATCH 161/203] MDEV-15596 10.2 doesn't work with openssl 1.1.1 --- include/ssl_compat.h | 6 +++++- vio/viosslfactories.c | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/ssl_compat.h b/include/ssl_compat.h index 54e80af769d..2777ae94527 100644 --- a/include/ssl_compat.h +++ b/include/ssl_compat.h @@ -27,7 +27,7 @@ #define HAVE_OPENSSL11 1 #define SSL_LIBRARY OpenSSL_version(OPENSSL_VERSION) #define ERR_remove_state(X) ERR_clear_error() -#define EVP_CIPHER_CTX_SIZE 168 +#define EVP_CIPHER_CTX_SIZE 176 #define EVP_MD_CTX_SIZE 48 #undef EVP_MD_CTX_init #define EVP_MD_CTX_init(X) do { bzero((X), EVP_MD_CTX_SIZE); EVP_MD_CTX_reset(X); } while(0) @@ -77,6 +77,10 @@ #define X509_get0_notAfter(X) X509_get_notAfter(X) #endif +#ifndef TLS1_3_VERSION +#define SSL_CTX_set_ciphersuites(X,Y) 0 +#endif + #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ diff --git a/vio/viosslfactories.c b/vio/viosslfactories.c index 6358b976e16..fa02eb03caa 100644 --- a/vio/viosslfactories.c +++ b/vio/viosslfactories.c @@ -211,6 +211,7 @@ new_VioSSLFd(const char *key_file, const char *cert_file, none of the provided ciphers could be selected */ if (cipher && + SSL_CTX_set_ciphersuites(ssl_fd->ssl_context, cipher) == 0 && SSL_CTX_set_cipher_list(ssl_fd->ssl_context, cipher) == 0) { *error= SSL_INITERR_CIPHERS; From 5f0510225aa149377b8563f6e96b74d05d41f080 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 18 Jun 2018 21:00:25 +0200 Subject: [PATCH 162/203] MDEV-16238 root/localhost authn prioritizes authentication_string over Password Don't let SET PASSWORD to set the password, if auth_string is set. Now SET PASSWORD always sets the plugin/auth_string fields and clears the password field (on pre-plugin mysql.user table it works as before). --- mysql-test/r/grant2.result | 8 +-- ...plugin-9835.result => set_password.result} | 33 ++++++--- .../r/show_grants_with_plugin-7985.result | 6 +- .../rpl/include/rpl_mixed_check_user.inc | 4 +- mysql-test/suite/rpl/r/rpl_do_grant.result | 14 ++-- .../suite/rpl/r/rpl_innodb_mixed_dml.result | 68 +++++++++---------- mysql-test/suite/rpl/t/rpl_do_grant.test | 8 +-- mysql-test/t/grant2.test | 4 +- ...ord_plugin-9835.test => set_password.test} | 14 ++++ .../t/show_grants_with_plugin-7985.test | 2 +- sql/sql_acl.cc | 13 ++-- .../rpl/r/rpl_tokudb_mixed_dml.result | 68 +++++++++---------- 12 files changed, 138 insertions(+), 104 deletions(-) rename mysql-test/r/{set_password_plugin-9835.result => set_password.result} (75%) rename mysql-test/t/{set_password_plugin-9835.test => set_password.test} (86%) diff --git a/mysql-test/r/grant2.result b/mysql-test/r/grant2.result index d7e42f9b7aa..ffb41c1b5f8 100644 --- a/mysql-test/r/grant2.result +++ b/mysql-test/r/grant2.result @@ -372,8 +372,8 @@ mysqltest_1@127.0.0.1 set password = password('changed'); disconnect b12302; connection default; -select host, length(password) from mysql.user where user like 'mysqltest\_1'; -host length(password) +select host, length(authentication_string) from mysql.user where user like 'mysqltest\_1'; +host length(authentication_string) 127.0.0.1 41 revoke all on mysqltest_1.* from mysqltest_1@'127.0.0.1'; delete from mysql.user where user like 'mysqltest\_1'; @@ -387,8 +387,8 @@ mysqltest_1@127.0.0.0/255.0.0.0 set password = password('changed'); disconnect b12302_2; connection default; -select host, length(password) from mysql.user where user like 'mysqltest\_1'; -host length(password) +select host, length(authentication_string) from mysql.user where user like 'mysqltest\_1'; +host length(authentication_string) 127.0.0.0/255.0.0.0 41 revoke all on mysqltest_1.* from mysqltest_1@'127.0.0.0/255.0.0.0'; delete from mysql.user where user like 'mysqltest\_1'; diff --git a/mysql-test/r/set_password_plugin-9835.result b/mysql-test/r/set_password.result similarity index 75% rename from mysql-test/r/set_password_plugin-9835.result rename to mysql-test/r/set_password.result index 3cc723957d8..f0faeda1974 100644 --- a/mysql-test/r/set_password_plugin-9835.result +++ b/mysql-test/r/set_password.result @@ -11,10 +11,10 @@ select user, host, password, plugin, authentication_string from mysql.user where user host password plugin authentication_string natauth localhost *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 newpass localhost *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 -newpassnat localhost *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 +newpassnat localhost mysql_native_password *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 oldauth localhost 378b243e220ca493 oldpass localhost 378b243e220ca493 -oldpassold localhost 378b243e220ca493 +oldpassold localhost mysql_old_password 378b243e220ca493 connect con,localhost,natauth,test,; select current_user(); current_user() @@ -86,12 +86,12 @@ set password for oldpass@localhost = PASSWORD('test2'); set password for oldpassold@localhost = PASSWORD('test2'); select user, host, password, plugin, authentication_string from mysql.user where user != 'root'; user host password plugin authentication_string -natauth localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -newpass localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -newpassnat localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -oldauth localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -oldpass localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -oldpassold localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +natauth localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +newpass localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +newpassnat localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +oldauth localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +oldpass localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +oldpassold localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E connect con,localhost,natauth,test2,; select current_user(); current_user() @@ -158,3 +158,20 @@ connection default; drop user natauth@localhost, newpass@localhost, newpassnat@localhost; drop user oldauth@localhost, oldpass@localhost, oldpassold@localhost; set global secure_auth=default; +create user foo@localhost identified with mysql_native_password; +update mysql.user set authentication_string=password('foo'), plugin='mysql_native_password' where user='foo' and host='localhost'; +set password for 'foo'@'localhost' = password('bar'); +flush privileges; +connect foo, localhost, foo, bar; +select user(), current_user(); +user() current_user() +foo@localhost foo@localhost +show grants; +Grants for foo@localhost +GRANT USAGE ON *.* TO 'foo'@'localhost' IDENTIFIED BY PASSWORD '*E8D46CE25265E545D225A8A6F1BAF642FEBEE5CB' +disconnect foo; +connection default; +select user,host,password,plugin,authentication_string from mysql.user where user='foo'; +user host password plugin authentication_string +foo localhost mysql_native_password *E8D46CE25265E545D225A8A6F1BAF642FEBEE5CB +drop user foo@localhost; diff --git a/mysql-test/r/show_grants_with_plugin-7985.result b/mysql-test/r/show_grants_with_plugin-7985.result index 81880e5cc40..0f8e1e39969 100644 --- a/mysql-test/r/show_grants_with_plugin-7985.result +++ b/mysql-test/r/show_grants_with_plugin-7985.result @@ -71,7 +71,7 @@ connection default; set password for u1 = PASSWORD('SOMETHINGELSE'); select user, host, password, plugin, authentication_string from mysql.user where user = 'u1'; user host password plugin authentication_string -u1 % *054B7BBD2B9A553DA560520DCD3F76DA2D81B7C6 +u1 % mysql_native_password *054B7BBD2B9A553DA560520DCD3F76DA2D81B7C6 # # Here we should use the password field, as that primes over # the authentication_string field. @@ -112,7 +112,7 @@ connection default; # Now we remove the authentication plugin password, flush privileges and # try again. # -update mysql.user set authentication_string = '' where user='u1'; +update mysql.user set password=authentication_string, plugin='', authentication_string='' where user='u1'; select user, host, password, plugin, authentication_string from mysql.user where user = 'u1'; user host password plugin authentication_string u1 % *054B7BBD2B9A553DA560520DCD3F76DA2D81B7C6 @@ -172,7 +172,7 @@ connection default; set password for u1 = ''; select user, host, password, plugin, authentication_string from mysql.user where user = 'u1'; user host password plugin authentication_string -u1 % +u1 % mysql_native_password # # Test no password connect. # diff --git a/mysql-test/suite/rpl/include/rpl_mixed_check_user.inc b/mysql-test/suite/rpl/include/rpl_mixed_check_user.inc index 8bf5a4eea1e..548fdf789da 100644 --- a/mysql-test/suite/rpl/include/rpl_mixed_check_user.inc +++ b/mysql-test/suite/rpl/include/rpl_mixed_check_user.inc @@ -5,9 +5,9 @@ # Requirements: ######################################### -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; sync_slave_with_master; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; connection master; diff --git a/mysql-test/suite/rpl/r/rpl_do_grant.result b/mysql-test/suite/rpl/r/rpl_do_grant.result index aed8bba3bc0..9eca21b38e4 100644 --- a/mysql-test/suite/rpl/r/rpl_do_grant.result +++ b/mysql-test/suite/rpl/r/rpl_do_grant.result @@ -20,21 +20,21 @@ GRANT DROP ON `test`.* TO 'rpl_do_grant'@'localhost' connection master; set password for rpl_do_grant@localhost=password("does it work?"); connection slave; -select password<>_binary'' from mysql.user where user=_binary'rpl_do_grant'; -password<>_binary'' +select authentication_string<>_binary'' from mysql.user where user=_binary'rpl_do_grant'; +authentication_string<>_binary'' 1 connection master; -update mysql.user set password='' where user='rpl_do_grant'; +update mysql.user set authentication_string='' where user='rpl_do_grant'; flush privileges; -select password<>'' from mysql.user where user='rpl_do_grant'; -password<>'' +select authentication_string<>'' from mysql.user where user='rpl_do_grant'; +authentication_string<>'' 0 set sql_mode='ANSI_QUOTES'; set password for rpl_do_grant@localhost=password('does it work?'); set sql_mode=''; connection slave; -select password<>'' from mysql.user where user='rpl_do_grant'; -password<>'' +select authentication_string<>'' from mysql.user where user='rpl_do_grant'; +authentication_string<>'' 1 connection master; delete from mysql.user where user=_binary'rpl_do_grant'; diff --git a/mysql-test/suite/rpl/r/rpl_innodb_mixed_dml.result b/mysql-test/suite/rpl/r/rpl_innodb_mixed_dml.result index 1f6a9370090..24ce8899e2c 100644 --- a/mysql-test/suite/rpl/r/rpl_innodb_mixed_dml.result +++ b/mysql-test/suite/rpl/r/rpl_innodb_mixed_dml.result @@ -480,72 +480,72 @@ SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; ******************** CREATE USER ******************** CREATE USER 'user_test_rpl'@'localhost' IDENTIFIED BY PASSWORD '*1111111111111111111111111111111111111111'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 N connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 N connection master; ******************** GRANT ******************** GRANT SELECT ON *.* TO 'user_test_rpl'@'localhost'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 Y +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 Y connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 Y +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 Y connection master; ******************** REVOKE ******************** REVOKE SELECT ON *.* FROM 'user_test_rpl'@'localhost'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 N connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 N connection master; ******************** SET PASSWORD ******************** SET PASSWORD FOR 'user_test_rpl'@'localhost' = '*0000000000000000000000000000000000000000'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *0000000000000000000000000000000000000000 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl mysql_native_password *0000000000000000000000000000000000000000 N connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *0000000000000000000000000000000000000000 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl mysql_native_password *0000000000000000000000000000000000000000 N connection master; ******************** RENAME USER ******************** RENAME USER 'user_test_rpl'@'localhost' TO 'user_test_rpl_2'@'localhost'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl_2 *0000000000000000000000000000000000000000 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl_2 mysql_native_password *0000000000000000000000000000000000000000 N connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl_2 *0000000000000000000000000000000000000000 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl_2 mysql_native_password *0000000000000000000000000000000000000000 N connection master; ******************** DROP USER ******************** DROP USER 'user_test_rpl_2'@'localhost'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv connection master; INSERT INTO t1 VALUES(100, 'test'); diff --git a/mysql-test/suite/rpl/t/rpl_do_grant.test b/mysql-test/suite/rpl/t/rpl_do_grant.test index 83b9a84744f..0024c7039e4 100644 --- a/mysql-test/suite/rpl/t/rpl_do_grant.test +++ b/mysql-test/suite/rpl/t/rpl_do_grant.test @@ -27,20 +27,20 @@ show grants for rpl_do_grant@localhost; connection master; set password for rpl_do_grant@localhost=password("does it work?"); sync_slave_with_master; -select password<>_binary'' from mysql.user where user=_binary'rpl_do_grant'; +select authentication_string<>_binary'' from mysql.user where user=_binary'rpl_do_grant'; # # Bug#24158 SET PASSWORD in binary log fails under ANSI_QUOTES # connection master; -update mysql.user set password='' where user='rpl_do_grant'; +update mysql.user set authentication_string='' where user='rpl_do_grant'; flush privileges; -select password<>'' from mysql.user where user='rpl_do_grant'; +select authentication_string<>'' from mysql.user where user='rpl_do_grant'; set sql_mode='ANSI_QUOTES'; set password for rpl_do_grant@localhost=password('does it work?'); set sql_mode=''; sync_slave_with_master; -select password<>'' from mysql.user where user='rpl_do_grant'; +select authentication_string<>'' from mysql.user where user='rpl_do_grant'; # clear what we have done, to not influence other tests. diff --git a/mysql-test/t/grant2.test b/mysql-test/t/grant2.test index cee5df089df..1f7450df6c1 100644 --- a/mysql-test/t/grant2.test +++ b/mysql-test/t/grant2.test @@ -385,7 +385,7 @@ select current_user(); set password = password('changed'); disconnect b12302; connection default; -select host, length(password) from mysql.user where user like 'mysqltest\_1'; +select host, length(authentication_string) from mysql.user where user like 'mysqltest\_1'; revoke all on mysqltest_1.* from mysqltest_1@'127.0.0.1'; delete from mysql.user where user like 'mysqltest\_1'; flush privileges; @@ -396,7 +396,7 @@ select current_user(); set password = password('changed'); disconnect b12302_2; connection default; -select host, length(password) from mysql.user where user like 'mysqltest\_1'; +select host, length(authentication_string) from mysql.user where user like 'mysqltest\_1'; revoke all on mysqltest_1.* from mysqltest_1@'127.0.0.0/255.0.0.0'; delete from mysql.user where user like 'mysqltest\_1'; flush privileges; diff --git a/mysql-test/t/set_password_plugin-9835.test b/mysql-test/t/set_password.test similarity index 86% rename from mysql-test/t/set_password_plugin-9835.test rename to mysql-test/t/set_password.test index 6afccd74f9d..4d8010d3b25 100644 --- a/mysql-test/t/set_password_plugin-9835.test +++ b/mysql-test/t/set_password.test @@ -129,3 +129,17 @@ drop user natauth@localhost, newpass@localhost, newpassnat@localhost; drop user oldauth@localhost, oldpass@localhost, oldpassold@localhost; set global secure_auth=default; +# +# MDEV-16238 root/localhost authn prioritizes authentication_string over Password +# +create user foo@localhost identified with mysql_native_password; +update mysql.user set authentication_string=password('foo'), plugin='mysql_native_password' where user='foo' and host='localhost'; +set password for 'foo'@'localhost' = password('bar'); +flush privileges; +--connect foo, localhost, foo, bar +select user(), current_user(); +show grants; +--disconnect foo +--connection default +select user,host,password,plugin,authentication_string from mysql.user where user='foo'; +drop user foo@localhost; diff --git a/mysql-test/t/show_grants_with_plugin-7985.test b/mysql-test/t/show_grants_with_plugin-7985.test index 84f71c72667..85952870254 100644 --- a/mysql-test/t/show_grants_with_plugin-7985.test +++ b/mysql-test/t/show_grants_with_plugin-7985.test @@ -91,7 +91,7 @@ show grants; --echo # Now we remove the authentication plugin password, flush privileges and --echo # try again. --echo # -update mysql.user set authentication_string = '' where user='u1'; +update mysql.user set password=authentication_string, plugin='', authentication_string='' where user='u1'; select user, host, password, plugin, authentication_string from mysql.user where user = 'u1'; flush privileges; show grants for u1; diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 04191a0c8c8..d550aa1feb8 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -3919,13 +3919,16 @@ static bool update_user_table(THD *thd, const User_table& user_table, DBUG_RETURN(1); /* purecov: deadcode */ } store_record(table,record[1]); - /* If the password column is missing, we use the - authentication_string column. */ - if (user_table.password()) - user_table.password()->store(new_password, new_password_len, system_charset_info); - else + + if (user_table.plugin()) + { set_authentication_plugin_from_password(user_table, new_password, new_password_len); + new_password_len= 0; + } + + if (user_table.password()) + user_table.password()->store(new_password, new_password_len, system_charset_info); if ((error=table->file->ha_update_row(table->record[1],table->record[0])) && diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_mixed_dml.result b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_mixed_dml.result index 7f8b9dd5104..a5060d9e3bf 100644 --- a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_mixed_dml.result +++ b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_mixed_dml.result @@ -483,72 +483,72 @@ SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; ******************** CREATE USER ******************** CREATE USER 'user_test_rpl'@'localhost' IDENTIFIED BY PASSWORD '*1111111111111111111111111111111111111111'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 N connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 N connection master; ******************** GRANT ******************** GRANT SELECT ON *.* TO 'user_test_rpl'@'localhost'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 Y +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 Y connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 Y +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 Y connection master; ******************** REVOKE ******************** REVOKE SELECT ON *.* FROM 'user_test_rpl'@'localhost'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 N connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *1111111111111111111111111111111111111111 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl *1111111111111111111111111111111111111111 N connection master; ******************** SET PASSWORD ******************** SET PASSWORD FOR 'user_test_rpl'@'localhost' = '*0000000000000000000000000000000000000000'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *0000000000000000000000000000000000000000 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl mysql_native_password *0000000000000000000000000000000000000000 N connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl *0000000000000000000000000000000000000000 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl mysql_native_password *0000000000000000000000000000000000000000 N connection master; ******************** RENAME USER ******************** RENAME USER 'user_test_rpl'@'localhost' TO 'user_test_rpl_2'@'localhost'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl_2 *0000000000000000000000000000000000000000 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl_2 mysql_native_password *0000000000000000000000000000000000000000 N connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv -localhost user_test_rpl_2 *0000000000000000000000000000000000000000 N +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv +localhost user_test_rpl_2 mysql_native_password *0000000000000000000000000000000000000000 N connection master; ******************** DROP USER ******************** DROP USER 'user_test_rpl_2'@'localhost'; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv connection slave; USE test_rpl; -SELECT host, user, password, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; -host user password select_priv +SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; +host user password plugin authentication_string select_priv connection master; INSERT INTO t1 VALUES(100, 'test'); From af2dd582e64a6903bee766cd992666da98ca8d69 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 18 Jun 2018 21:28:27 +0200 Subject: [PATCH 163/203] empty password is a valid password, don't crash --- mysql-test/r/set_password.result | 4 ++++ mysql-test/t/set_password.test | 2 ++ sql/sql_acl.cc | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/set_password.result b/mysql-test/r/set_password.result index f0faeda1974..315d0bef9fb 100644 --- a/mysql-test/r/set_password.result +++ b/mysql-test/r/set_password.result @@ -174,4 +174,8 @@ connection default; select user,host,password,plugin,authentication_string from mysql.user where user='foo'; user host password plugin authentication_string foo localhost mysql_native_password *E8D46CE25265E545D225A8A6F1BAF642FEBEE5CB +set password for 'foo'@'localhost' = ''; +select user,host,password,plugin,authentication_string from mysql.user where user='foo'; +user host password plugin authentication_string +foo localhost mysql_native_password drop user foo@localhost; diff --git a/mysql-test/t/set_password.test b/mysql-test/t/set_password.test index 4d8010d3b25..fc1ecb5ef5c 100644 --- a/mysql-test/t/set_password.test +++ b/mysql-test/t/set_password.test @@ -142,4 +142,6 @@ show grants; --disconnect foo --connection default select user,host,password,plugin,authentication_string from mysql.user where user='foo'; +set password for 'foo'@'localhost' = ''; +select user,host,password,plugin,authentication_string from mysql.user where user='foo'; drop user foo@localhost; diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index d550aa1feb8..1809e22ffbf 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -3864,7 +3864,8 @@ void set_authentication_plugin_from_password(const User_table& user_table, const char* password, uint password_length) { - if (password_length == SCRAMBLED_PASSWORD_CHAR_LENGTH) + if (password_length == SCRAMBLED_PASSWORD_CHAR_LENGTH || + password_length == 0) { user_table.plugin()->store(native_password_plugin_name.str, native_password_plugin_name.length, From df704b5a1b581f4f9a02b9310f2b2c8ef36eb98f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 19 Jun 2018 18:28:39 +0200 Subject: [PATCH 164/203] don't use my_error(0) or my_printf_error(0) there's an assert that catches it --- plugin/auth_gssapi/gssapi_server.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugin/auth_gssapi/gssapi_server.cc b/plugin/auth_gssapi/gssapi_server.cc index 50c34ecc573..9197b3e06fc 100644 --- a/plugin/auth_gssapi/gssapi_server.cc +++ b/plugin/auth_gssapi/gssapi_server.cc @@ -44,21 +44,21 @@ static char* get_default_principal_name() if(krb5_init_context(&context)) { - my_printf_error(0, "GSSAPI plugin : krb5_init_context failed", + my_printf_error(1, "GSSAPI plugin : krb5_init_context failed", ME_ERROR_LOG | ME_WARNING); goto cleanup; } if (krb5_sname_to_principal(context, NULL, "mariadb", KRB5_NT_SRV_HST, &principal)) { - my_printf_error(0, "GSSAPI plugin : krb5_sname_to_principal failed", + my_printf_error(1, "GSSAPI plugin : krb5_sname_to_principal failed", ME_ERROR_LOG | ME_WARNING); goto cleanup; } if (krb5_unparse_name(context, principal, &unparsed_name)) { - my_printf_error(0, "GSSAPI plugin : krb5_unparse_name failed", + my_printf_error(1, "GSSAPI plugin : krb5_unparse_name failed", ME_ERROR_LOG | ME_WARNING); goto cleanup; } @@ -66,7 +66,7 @@ static char* get_default_principal_name() /* Check for entry in keytab */ if (krb5_kt_read_service_key(context, NULL, principal, 0, (krb5_enctype)0, &key)) { - my_printf_error(0, "GSSAPI plugin : default principal '%s' not found in keytab", + my_printf_error(1, "GSSAPI plugin : default principal '%s' not found in keytab", ME_ERROR_LOG | ME_WARNING, unparsed_name); goto cleanup; } @@ -104,7 +104,7 @@ int plugin_init() /* import service principal from plain text */ if(srv_principal_name && srv_principal_name[0]) { - my_printf_error(0, "GSSAPI plugin : using principal name '%s'", + my_printf_error(1, "GSSAPI plugin : using principal name '%s'", ME_ERROR_LOG | ME_NOTE, srv_principal_name); principal_name_buf.length= strlen(srv_principal_name); principal_name_buf.value= srv_principal_name; From 635c5e32815389bff928c3141e418b3cd2c5b887 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 19 Jun 2018 19:07:32 +0200 Subject: [PATCH 165/203] mysql_install_db: clarify the after-install message don't tell users to set the root password if they're using unix_socket --- scripts/mysql_install_db.sh | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index 0a596e1e03b..ec87acbc95d 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -525,19 +525,24 @@ then s_echo "To start mysqld at boot time you have to copy" s_echo "support-files/mysql.server to the right place for your system" - echo - echo "PLEASE REMEMBER TO SET A PASSWORD FOR THE MariaDB root USER !" - echo "To do so, start the server, then issue the following commands:" - echo - echo "'$bindir/mysqladmin' -u root password 'new-password'" - echo "'$bindir/mysqladmin' -u root -h $hostname password 'new-password'" - echo - echo "Alternatively you can run:" - echo "'$bindir/mysql_secure_installation'" - echo - echo "which will also give you the option of removing the test" - echo "databases and anonymous user created by default. This is" - echo "strongly recommended for production servers." + if test "$auth_root_authentication_method" = normal + then + echo + echo + echo "PLEASE REMEMBER TO SET A PASSWORD FOR THE MariaDB root USER !" + echo "To do so, start the server, then issue the following commands:" + echo + echo "'$bindir/mysqladmin' -u root password 'new-password'" + echo "'$bindir/mysqladmin' -u root -h $hostname password 'new-password'" + echo + echo "Alternatively you can run:" + echo "'$bindir/mysql_secure_installation'" + echo + echo "which will also give you the option of removing the test" + echo "databases and anonymous user created by default. This is" + echo "strongly recommended for production servers." + fi + echo echo "See the MariaDB Knowledgebase at http://mariadb.com/kb or the" echo "MySQL manual for more instructions." From fe3f9fa9183ea3d10397b6f7f4d422ae9bba00a4 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 21 Jun 2018 15:17:15 +0400 Subject: [PATCH 166/203] MDEV-12809 Bad column type created for TEXT(1431655798) CHARACTER SET utf8 --- mysql-test/r/type_blob.result | 16 ++++++++++++++++ mysql-test/t/type_blob.test | 18 ++++++++++++++++++ sql/field.cc | 2 +- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/type_blob.result b/mysql-test/r/type_blob.result index 569ba65df3f..3c99366168c 100644 --- a/mysql-test/r/type_blob.result +++ b/mysql-test/r/type_blob.result @@ -1063,3 +1063,19 @@ DROP TABLE t1; # # End of 5.5 tests # +# +# Start of 10.2 test +# +# +# MDEV-12809 Bad column type created for TEXT(1431655798) CHARACTER SET utf8 +# +CREATE TABLE t1 (a TEXT(1431655798) CHARACTER SET utf8); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` longtext CHARACTER SET utf8 DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +# +# End of 10.2 test +# diff --git a/mysql-test/t/type_blob.test b/mysql-test/t/type_blob.test index 8db6ac6da2a..2c74d4ea241 100644 --- a/mysql-test/t/type_blob.test +++ b/mysql-test/t/type_blob.test @@ -680,3 +680,21 @@ DROP TABLE t1; --echo # --echo # End of 5.5 tests --echo # + + +--echo # +--echo # Start of 10.2 test +--echo # + +--echo # +--echo # MDEV-12809 Bad column type created for TEXT(1431655798) CHARACTER SET utf8 +--echo # + +CREATE TABLE t1 (a TEXT(1431655798) CHARACTER SET utf8); +SHOW CREATE TABLE t1; +DROP TABLE t1; + + +--echo # +--echo # End of 10.2 test +--echo # diff --git a/sql/field.cc b/sql/field.cc index 56948acd8ba..f540c58757f 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -9777,7 +9777,7 @@ void Column_definition::create_length_to_internal_length(void) case MYSQL_TYPE_STRING: case MYSQL_TYPE_VARCHAR: length*= charset->mbmaxlen; - DBUG_ASSERT(length <= UINT_MAX32); + set_if_smaller(length, UINT_MAX32); key_length= (uint32)length; pack_length= calc_pack_length(sql_type, key_length); break; From 082eec1418bc1af5477f8a7d19c48d3a3b4c4ad5 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 21 Jun 2018 23:48:59 +0200 Subject: [PATCH 167/203] SET wsrep_on=1 - only check innodb_lock_schedule_algorithm if innodb is enabled --- sql/wsrep_var.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 0bca5d1177d..bd33847762d 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -47,6 +47,7 @@ int wsrep_init_vars() linking will succeed even if the server is built with a dynamically linked InnoDB. */ ulong innodb_lock_schedule_algorithm __attribute__((weak)); +struct handlerton* innodb_hton_ptr __attribute__((weak)); bool wsrep_on_update (sys_var *self, THD* thd, enum_var_type var_type) { @@ -65,7 +66,7 @@ bool wsrep_on_check(sys_var *self, THD* thd, set_var* var) if (check_has_super(self, thd, var)) return true; - if (new_wsrep_on && innodb_lock_schedule_algorithm != 0) { + if (new_wsrep_on && innodb_hton_ptr && innodb_lock_schedule_algorithm != 0) { my_message(ER_WRONG_ARGUMENTS, " WSREP (galera) can't be enabled " "if innodb_lock_schedule_algorithm=VATS. Please configure" " innodb_lock_schedule_algorithm=FCFS and restart.", MYF(0)); From ef64856b97be4d41b5a3581385057c67935bf77e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 21 Jun 2018 23:49:37 +0200 Subject: [PATCH 168/203] don't crash on innodb_undo_tablespaces=1 --- storage/innobase/srv/srv0start.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 98dca77c61d..d59d260940e 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -840,6 +840,10 @@ srv_undo_tablespaces_init(bool create_new_db) ut_a(srv_undo_tablespaces <= TRX_SYS_N_RSEGS); ut_a(!create_new_db || srv_operation == SRV_OPERATION_NORMAL); + if (srv_undo_tablespaces == 1) { /* 1 is not allowed, make it 0 */ + srv_undo_tablespaces = 0; + } + memset(undo_tablespace_ids, 0x0, sizeof(undo_tablespace_ids)); /* Create the undo spaces only if we are creating a new From 0d745343fc1e274a938bef9e91b18c508076ae62 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 22 Jun 2018 09:52:21 +0200 Subject: [PATCH 169/203] fix plugins.processlist make it not to fail when `show engine innodb status` output contains a double quote --- mysql-test/suite/plugins/t/processlist.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/plugins/t/processlist.test b/mysql-test/suite/plugins/t/processlist.test index 5aacef317b9..e8f03aacb10 100644 --- a/mysql-test/suite/plugins/t/processlist.test +++ b/mysql-test/suite/plugins/t/processlist.test @@ -8,7 +8,8 @@ start transaction; insert t1 values (1); let id=`select connection_id()`; connect con2,localhost,root; -let s=query_get_value(show engine innodb status,Status,1); +replace_regex /\"/-/; #" +let s=`show engine innodb status`; disable_query_log; eval select regexp_replace("$s", '(?s)^.*MySQL thread id $id,.*root([^\n]*)\n.*', '\\\\1') as `state from show engine innodb status`; eval select state as `state from show processlist` from information_schema.processlist where id = $id; From ecc4682672f604bf24edb79662ba0eafdfd16f0c Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 22 Jun 2018 15:24:09 +0100 Subject: [PATCH 170/203] MDEV-16519 : mariabackup should fail if MDL could not be acquired with lock-ddl-per-table There is a tiny chance for race condition during MDL acquisition. If table is renamed just prior to SELECT 1 FROM LIMIT 0 then this query would fail, yet mariabackup --backup does not handle it as fatal error and continues, only to fail later during file copy. The fix is to die on error, of MDL lock query fails. --- extra/mariabackup/backup_mysql.cc | 10 ++++++++-- .../suite/mariabackup/rename_during_mdl_lock.result | 3 +++ .../suite/mariabackup/rename_during_mdl_lock.test | 13 +++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/mariabackup/rename_during_mdl_lock.result create mode 100644 mysql-test/suite/mariabackup/rename_during_mdl_lock.test diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index d287dcf181d..60dfb66d2c8 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -186,6 +186,7 @@ xb_mysql_query(MYSQL *connection, const char *query, bool use_result, if (!use_result) { mysql_free_result(mysql_result); + mysql_result = NULL; } } @@ -1783,12 +1784,17 @@ mdl_lock_table(ulint space_id) MYSQL_RES *mysql_result = xb_mysql_query(mdl_con, oss.str().c_str(), true, true); while (MYSQL_ROW row = mysql_fetch_row(mysql_result)) { + + DBUG_EXECUTE_IF("rename_during_mdl_lock_table", + if (strcmp(row[0], "test/t1") == 0) + xb_mysql_query(mysql_connection, "RENAME TABLE test.t1 to test.t2", false, true + );); + std::string full_table_name = ut_get_name(0,row[0]); std::ostringstream lock_query; lock_query << "SELECT 1 FROM " << full_table_name << " LIMIT 0"; - msg_ts("Locking MDL for %s\n", full_table_name.c_str()); - xb_mysql_query(mdl_con, lock_query.str().c_str(), false, false); + xb_mysql_query(mdl_con, lock_query.str().c_str(), false, true); } pthread_mutex_unlock(&mdl_lock_con_mutex); diff --git a/mysql-test/suite/mariabackup/rename_during_mdl_lock.result b/mysql-test/suite/mariabackup/rename_during_mdl_lock.result new file mode 100644 index 00000000000..982851438f2 --- /dev/null +++ b/mysql-test/suite/mariabackup/rename_during_mdl_lock.result @@ -0,0 +1,3 @@ +CREATE TABLE t1(i int) ENGINE INNODB; +FOUND 1 /failed to execute query SELECT 1 FROM/ in backup.log +DROP TABLE t2; diff --git a/mysql-test/suite/mariabackup/rename_during_mdl_lock.test b/mysql-test/suite/mariabackup/rename_during_mdl_lock.test new file mode 100644 index 00000000000..0a41f1dfe74 --- /dev/null +++ b/mysql-test/suite/mariabackup/rename_during_mdl_lock.test @@ -0,0 +1,13 @@ +--source include/have_debug.inc +let $targetdir=$MYSQLTEST_VARDIR/backup; +mkdir $targetdir; +CREATE TABLE t1(i int) ENGINE INNODB; +--error 1 +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --lock-ddl-per-table --dbug=+d,rename_during_mdl_lock_table 2>$targetdir/backup.log; + +let SEARCH_FILE=$targetdir/backup.log; +let SEARCH_PATTERN=failed to execute query SELECT 1 FROM; +source include/search_pattern_in_file.inc; + +DROP TABLE t2; +rmdir $targetdir; From 95ef8de89149dcdd532f7ca51781c20c76c4dce3 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 22 Jun 2018 23:30:26 +0100 Subject: [PATCH 171/203] mariabackup - rename backup-rocksdb option to rocksdb-backup to avoid "using unique option prefix 'backup' is error-prone", there is already --backup option there. --- extra/mariabackup/xtrabackup.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 843a93cd251..390cdab52f2 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1247,7 +1247,7 @@ struct my_option xb_server_options[] = &xb_rocksdb_datadir, &xb_rocksdb_datadir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, - { "backup-rocksdb", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed." + { "rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed." "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data", &xb_backup_rocksdb, &xb_backup_rocksdb, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, From 364a20fe0b072fb1d2a9b54a8c4e47a5012f3e97 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Sat, 23 Jun 2018 19:36:26 -0700 Subject: [PATCH 172/203] MDEV-16507 SIGSEGV when use_stat_tables = preferably and optimizer_use_condition_selectivity = 4 It does not makes sense to try to read statistics for temporary tables because it's not collected. --- mysql-test/r/statistics.result | 20 ++++++++++++++++++++ mysql-test/t/statistics.test | 24 ++++++++++++++++++++++++ sql/sql_statistics.cc | 8 ++++---- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/statistics.result b/mysql-test/r/statistics.result index 74997c92d3e..23c8807897b 100644 --- a/mysql-test/r/statistics.result +++ b/mysql-test/r/statistics.result @@ -1659,3 +1659,23 @@ id set use_stat_tables=@save_use_stat_tables; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; +# +# MDEV-16507: statistics for temporary tables should not be used +# +SET +@save_optimizer_use_condition_selectivity= @@optimizer_use_condition_selectivity; +SET @@use_stat_tables = preferably ; +SET @@optimizer_use_condition_selectivity = 4; +CREATE TABLE t1 ( +TIMESTAMP TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +ON UPDATE CURRENT_TIMESTAMP +); +SET @had_t1_table= @@warning_count != 0; +CREATE TEMPORARY TABLE tmp_t1 LIKE t1; +INSERT INTO tmp_t1 VALUES (now()); +INSERT INTO t1 SELECT * FROM tmp_t1 WHERE @had_t1_table=0; +DROP TABLE t1; +SET +use_stat_tables=@save_use_stat_tables; +SET +optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; diff --git a/mysql-test/t/statistics.test b/mysql-test/t/statistics.test index 0ab42453125..5831e0b1d60 100644 --- a/mysql-test/t/statistics.test +++ b/mysql-test/t/statistics.test @@ -737,3 +737,27 @@ set use_stat_tables=@save_use_stat_tables; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; +--echo # +--echo # MDEV-16507: statistics for temporary tables should not be used +--echo # + +SET +@save_optimizer_use_condition_selectivity= @@optimizer_use_condition_selectivity; +SET @@use_stat_tables = preferably ; +SET @@optimizer_use_condition_selectivity = 4; + +CREATE TABLE t1 ( + TIMESTAMP TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP +); + +SET @had_t1_table= @@warning_count != 0; +CREATE TEMPORARY TABLE tmp_t1 LIKE t1; +INSERT INTO tmp_t1 VALUES (now()); +INSERT INTO t1 SELECT * FROM tmp_t1 WHERE @had_t1_table=0; +DROP TABLE t1; + +SET +use_stat_tables=@save_use_stat_tables; +SET +optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 471749ad346..1febc02b903 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -2952,7 +2952,7 @@ bool statistics_for_tables_is_needed(THD *thd, TABLE_LIST *tables) for (TABLE_LIST *tl= tables; tl; tl= tl->next_global) { - if (!tl->is_view_or_derived() && tl->table) + if (!tl->is_view_or_derived() && !is_temporary_table(tl) && tl->table) { TABLE_SHARE *table_share= tl->table->s; if (table_share && @@ -2964,7 +2964,7 @@ bool statistics_for_tables_is_needed(THD *thd, TABLE_LIST *tables) for (TABLE_LIST *tl= tables; tl; tl= tl->next_global) { - if (!tl->is_view_or_derived() && tl->table) + if (!tl->is_view_or_derived() && !is_temporary_table(tl) && tl->table) { TABLE_SHARE *table_share= tl->table->s; if (table_share && @@ -3093,7 +3093,7 @@ int read_statistics_for_tables_if_needed(THD *thd, TABLE_LIST *tables) for (TABLE_LIST *tl= tables; tl; tl= tl->next_global) { - if (!tl->is_view_or_derived() && tl->table) + if (!tl->is_view_or_derived() && !is_temporary_table(tl) && tl->table) { TABLE_SHARE *table_share= tl->table->s; if (table_share && @@ -3867,4 +3867,4 @@ bool is_stat_table(const char *db, const char *table) } } return false; -} \ No newline at end of file +} From e561a346c34da860206092f03742264246081005 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 23 Jun 2018 15:30:52 +0200 Subject: [PATCH 173/203] fix mtr warnings after 5f0510225aa --- mysql-test/r/grant.result | 1 + mysql-test/r/sp-security.result | 1 + mysql-test/suite/roles/grant_revoke_current.result | 1 + mysql-test/suite/roles/grant_revoke_current.test | 2 ++ mysql-test/suite/roles/set_default_role_ps-6960.result | 1 + mysql-test/suite/roles/set_default_role_ps-6960.test | 2 ++ mysql-test/t/grant.test | 3 +++ mysql-test/t/sp-security.test | 2 ++ 8 files changed, 13 insertions(+) diff --git a/mysql-test/r/grant.result b/mysql-test/r/grant.result index 118d1f2134b..81b41a083a3 100644 --- a/mysql-test/r/grant.result +++ b/mysql-test/r/grant.result @@ -1447,6 +1447,7 @@ CURRENT_USER() root@localhost SET PASSWORD FOR CURRENT_USER() = PASSWORD("admin"); SET PASSWORD FOR CURRENT_USER() = PASSWORD(""); +update mysql.user set plugin=''; # Bug#57952 diff --git a/mysql-test/r/sp-security.result b/mysql-test/r/sp-security.result index 882072ff7c7..7813ab6a192 100644 --- a/mysql-test/r/sp-security.result +++ b/mysql-test/r/sp-security.result @@ -755,6 +755,7 @@ GRANT EXECUTE ON PROCEDURE `test`.`sp1` TO 'root'@'localhost' GRANT PROXY ON ''@'%' TO 'root'@'localhost' WITH GRANT OPTION drop procedure sp1; set password=''; +update mysql.user set plugin=''; # # MDEV-13396 Unexpected "alter routine comand defined" during CREATE OR REPLACE PROCEDURE # diff --git a/mysql-test/suite/roles/grant_revoke_current.result b/mysql-test/suite/roles/grant_revoke_current.result index d9798463965..436bec92a8f 100644 --- a/mysql-test/suite/roles/grant_revoke_current.result +++ b/mysql-test/suite/roles/grant_revoke_current.result @@ -39,4 +39,5 @@ GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY PASSWORD '*34391 GRANT PROXY ON ''@'%' TO 'root'@'localhost' WITH GRANT OPTION GRANT USAGE ON *.* TO 'r1' set password=''; +update mysql.user set plugin=''; drop role r1; diff --git a/mysql-test/suite/roles/grant_revoke_current.test b/mysql-test/suite/roles/grant_revoke_current.test index 0ebe0170782..bffc04087b1 100644 --- a/mysql-test/suite/roles/grant_revoke_current.test +++ b/mysql-test/suite/roles/grant_revoke_current.test @@ -25,5 +25,7 @@ show grants; grant r1 to current_user() identified by 'barfoo'; show grants; set password=''; +#cleanup after MDEV-16238 +update mysql.user set plugin=''; drop role r1; diff --git a/mysql-test/suite/roles/set_default_role_ps-6960.result b/mysql-test/suite/roles/set_default_role_ps-6960.result index 60210d7f92c..c186e7bccb0 100644 --- a/mysql-test/suite/roles/set_default_role_ps-6960.result +++ b/mysql-test/suite/roles/set_default_role_ps-6960.result @@ -6,3 +6,4 @@ execute stmt; set password = ''; set default role NONE; drop role r1; +update mysql.user set plugin=''; diff --git a/mysql-test/suite/roles/set_default_role_ps-6960.test b/mysql-test/suite/roles/set_default_role_ps-6960.test index 8ac520e1776..8af95c9e8a0 100644 --- a/mysql-test/suite/roles/set_default_role_ps-6960.test +++ b/mysql-test/suite/roles/set_default_role_ps-6960.test @@ -13,3 +13,5 @@ execute stmt; set password = ''; set default role NONE; drop role r1; +#cleanup after MDEV-16238 +update mysql.user set plugin=''; diff --git a/mysql-test/t/grant.test b/mysql-test/t/grant.test index 72e427493da..cb4254fe8c6 100644 --- a/mysql-test/t/grant.test +++ b/mysql-test/t/grant.test @@ -1265,6 +1265,9 @@ SELECT CURRENT_USER(); SET PASSWORD FOR CURRENT_USER() = PASSWORD("admin"); SET PASSWORD FOR CURRENT_USER() = PASSWORD(""); +#cleanup after MDEV-16238 +update mysql.user set plugin=''; + # # Bug#57952: privilege change is not taken into account by EXECUTE. # diff --git a/mysql-test/t/sp-security.test b/mysql-test/t/sp-security.test index 059a5dd0fa8..73d0263dd69 100644 --- a/mysql-test/t/sp-security.test +++ b/mysql-test/t/sp-security.test @@ -1023,6 +1023,8 @@ grant execute on procedure sp1 to current_user() identified by 'barfoo'; show grants; drop procedure sp1; set password=''; +#cleanup after MDEV-16238 +update mysql.user set plugin=''; --echo # --echo # MDEV-13396 Unexpected "alter routine comand defined" during CREATE OR REPLACE PROCEDURE From b2190f859b652a0d4c5f2931b34d6e6e821bf715 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 22 Jun 2018 15:45:17 +0200 Subject: [PATCH 174/203] fix vcol.vcol_misc --embedded --- mysql-test/suite/vcol/t/vcol_misc.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/suite/vcol/t/vcol_misc.test b/mysql-test/suite/vcol/t/vcol_misc.test index 3efbaa0a376..b351e1eb4a6 100644 --- a/mysql-test/suite/vcol/t/vcol_misc.test +++ b/mysql-test/suite/vcol/t/vcol_misc.test @@ -330,8 +330,10 @@ SET sql_mode=DEFAULT; --copy_file std_data/frm/t1.frm $MYSQLD_DATADIR/test/t1.frm SHOW TABLES; +--replace_result $MYSQLD_DATADIR ./ --error ER_NOT_FORM_FILE SHOW CREATE TABLE t1; +--replace_result $MYSQLD_DATADIR ./ --error ER_NOT_FORM_FILE ALTER TABLE t1; --remove_file $MYSQLD_DATADIR/test/t1.frm From ffb96be9e7d9e9a76bb01a3cbbfa532b2135b86e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 22 Jun 2018 14:13:10 +0200 Subject: [PATCH 175/203] fix debian packaging for tokudb --- CMakeLists.txt | 2 +- cmake/install_layout.cmake | 1 + debian/mariadb-plugin-tokudb.install | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3875f01e525..48f573b772e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -281,7 +281,7 @@ ELSE() ENDIF() SET(DEFAULT_CHARSET_HOME "${DEFAULT_MYSQL_HOME}") SET(PLUGINDIR "${DEFAULT_MYSQL_HOME}/${INSTALL_PLUGINDIR}") -IF(INSTALL_SYSCONFDIR AND NOT DEFAULT_SYSCONFDIR) +IF(INSTALL_SYSCONFDIR AND NOT DEFAULT_SYSCONFDIR AND NOT DEB) SET(DEFAULT_SYSCONFDIR "${INSTALL_SYSCONFDIR}") ENDIF() diff --git a/cmake/install_layout.cmake b/cmake/install_layout.cmake index 5484691e94a..8bb638fd301 100644 --- a/cmake/install_layout.cmake +++ b/cmake/install_layout.cmake @@ -171,6 +171,7 @@ SET(INSTALL_SYSTEMD_TMPFILESDIR_RPM "/usr/lib/tmpfiles.d") SET(INSTALL_BINDIR_DEB "bin") SET(INSTALL_SBINDIR_DEB "sbin") SET(INSTALL_SCRIPTDIR_DEB "bin") +SET(INSTALL_SYSCONFDIR_DEB "/etc") SET(INSTALL_SYSCONF2DIR_DEB "/etc/mysql/conf.d") # SET(INSTALL_LIBDIR_DEB "lib") diff --git a/debian/mariadb-plugin-tokudb.install b/debian/mariadb-plugin-tokudb.install index 487c9df7120..48d16ffb2c0 100644 --- a/debian/mariadb-plugin-tokudb.install +++ b/debian/mariadb-plugin-tokudb.install @@ -1,4 +1,5 @@ etc/mysql/conf.d/tokudb.cnf etc/mysql/mariadb.conf.d +etc/systemd/system/mariadb.service.d/tokudb.conf usr/bin/tokuftdump usr/lib/mysql/plugin/ha_tokudb.so usr/share/doc/mariadb-server-10.2/README.md usr/share/doc/mariadb-plugin-tokudb/README.md From a19089ff4e69d596bbf5f4ca48a0d89452da0b8f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 22 Jun 2018 17:41:33 +0200 Subject: [PATCH 176/203] MDEV-16537 aws key management plugin on Ubuntu bionic has impossible dependencies remove manual libcurl3 dependency. it'll be automatically added (libcurl3 or libcurl4, whatever was actually used) --- debian/autobake-deb.sh | 2 +- debian/control | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index d4de8045788..b569df9118f 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -86,7 +86,7 @@ then fi if [[ $GCCVERSION -lt 40800 ]] then - sed '/Package: mariadb-plugin-aws-key-management-10.2/,+13d' -i debian/control + sed '/Package: mariadb-plugin-aws-key-management-10.2/,+12d' -i debian/control fi diff --git a/debian/control b/debian/control index 4b5ed47c606..e3d2dfcdb8f 100644 --- a/debian/control +++ b/debian/control @@ -590,7 +590,6 @@ Architecture: any Breaks: mariadb-aws-key-management-10.1 Replaces: mariadb-aws-key-management-10.1 Depends: mariadb-server-10.2, - libcurl3, ${misc:Depends}, ${shlibs:Depends} Description: Amazon Web Service Key Management Service Plugin for MariaDB From f25a5c39c1467df4677663607625c7444e65f837 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 23 Jun 2018 15:37:09 +0200 Subject: [PATCH 177/203] MDEV-14560 Extra engines enabled through additional config are not loaded on first installation Solution for Debian/Ubuntu: install a trigger to restart mysqld automatically whenever a package changes something in /etc/mysql or in /etc/systemd/system/mariadb.service.d --- debian/mariadb-server-10.2.postinst | 21 ++++++++++++++------- debian/mariadb-server-10.2.triggers | 2 ++ 2 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 debian/mariadb-server-10.2.triggers diff --git a/debian/mariadb-server-10.2.postinst b/debian/mariadb-server-10.2.postinst index 9131ca851a7..bf5cd289d76 100644 --- a/debian/mariadb-server-10.2.postinst +++ b/debian/mariadb-server-10.2.postinst @@ -50,14 +50,14 @@ EOF return $retval } -# This is necessary because mysql_install_db removes the pid file in /var/run -# and because changed configuration options should take effect immediately. -# In case the server wasn't running at all it should be ok if the stop -# script fails. I can't tell at this point because of the cleaned /var/run. -set +e; invoke stop; set -e - case "$1" in configure) + # This is needed because mysql_install_db removes the pid file in /var/run + # and because changed configuration options should take effect immediately. + # In case the server wasn't running at all it should be ok if the stop + # script fails. I can't tell at this point because of the cleaned /var/run. + set +e; invoke stop; set -e + mysql_statedir=/usr/share/mysql mysql_datadir=/var/lib/mysql mysql_logdir=/var/log/mysql @@ -91,7 +91,7 @@ case "$1" in mv "$targetdir" "$mysql_tmp" cat << EOF > "$mysql_tmp/README" -Ff you're reading this, it's most likely because you had replaced /var/lib/mysql +If you're reading this, it's most likely because you had replaced /var/lib/mysql with a symlink, then upgraded to a new version of mysql, and then dpkg removed your symlink (see #182747 and others). The mysql packages noticed that this happened, and as a workaround have restored it. However, because @@ -240,6 +240,13 @@ EOF abort-upgrade|abort-remove|abort-configure) ;; + triggered) + if [ -x "$(command -v systemctl)" ]; then + systemctl daemon-reload + fi + invoke restart + ;; + *) echo "postinst called with unknown argument '$1'" 1>&2 exit 1 diff --git a/debian/mariadb-server-10.2.triggers b/debian/mariadb-server-10.2.triggers new file mode 100644 index 00000000000..d1f5f5e14f1 --- /dev/null +++ b/debian/mariadb-server-10.2.triggers @@ -0,0 +1,2 @@ +interest-noawait /etc/mysql +interest-noawait /etc/systemd/system/mariadb.service.d From f57731102713b1b38e02cd15393cc15784b4d937 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 23 Jun 2018 23:44:11 +0200 Subject: [PATCH 178/203] fix mroonga post-install script it only worked if mroonga plugin wasn't installed before (normal case), but then it didn't need to delete anything. if, by some glitch, mroonga was already installed, it would delete mroonga from mysql.plugin, but INSTALL would fail (as mroonga was running), and the script aborted, leaving mroonga not in mysql.plugin at all. --- storage/mroonga/data/install.sql.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/mroonga/data/install.sql.in b/storage/mroonga/data/install.sql.in index d7d5f3c4ad6..0a2f308aef4 100644 --- a/storage/mroonga/data/install.sql.in +++ b/storage/mroonga/data/install.sql.in @@ -1,6 +1,6 @@ -DELETE IGNORE FROM mysql.plugin WHERE dl = 'ha_mroonga@MRN_PLUGIN_SUFFIX@'; - -INSTALL PLUGIN Mroonga SONAME 'ha_mroonga@MRN_PLUGIN_SUFFIX@'; +SET @inst=IF(EXISTS(SELECT * FROM mysql.plugin WHERE NAME='mroonga'),'DO 1', "INSTALL PLUGIN mroonga SONAME 'ha_mroonga'"); +PREPARE s FROM @inst; +EXECUTE s; DROP FUNCTION IF EXISTS last_insert_grn_id; CREATE FUNCTION last_insert_grn_id RETURNS INTEGER From 01bbb912bf688d1c1a4ef75967248ee2a9f8e12b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 24 Jun 2018 15:30:54 +0200 Subject: [PATCH 179/203] update C/C --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index 27b2f3d1f15..74fbbaefeb5 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit 27b2f3d1f1550dfaee0f63a331a406ab31c1b37e +Subproject commit 74fbbaefeb54a485786a7180e15c3995eeaeac0a From ee6ac4d313c5e8846c4f7b92a923154984528215 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 25 Jun 2018 09:08:07 +0400 Subject: [PATCH 180/203] MDEV-12574 MAX(old_decimal) produces a column of the old DECIMAL type --- mysql-test/r/type_decimal.result | 25 +++++++++++++++++++++++++ mysql-test/t/type_decimal.test | 21 +++++++++++++++++++++ sql/field.cc | 27 ++++++++++++++++++--------- sql/field.h | 13 +++++++++++++ 4 files changed, 77 insertions(+), 9 deletions(-) diff --git a/mysql-test/r/type_decimal.result b/mysql-test/r/type_decimal.result index d5232ff92a8..b858d1ebab1 100644 --- a/mysql-test/r/type_decimal.result +++ b/mysql-test/r/type_decimal.result @@ -1033,5 +1033,30 @@ c1 c2 0.123456 0.123456 SET sql_mode=DEFAULT; # +# MDEV-12574 MAX(old_decimal) produces a column of the old DECIMAL type +# +SHOW CREATE TABLE t1dec102; +Table Create Table +t1dec102 CREATE TABLE `t1dec102` ( + `a` decimal(10,2)/*old*/ DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +CREATE TABLE t1 AS SELECT a, MAX(a), COALESCE(a) FROM t1dec102; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` decimal(10,2) DEFAULT NULL, + `MAX(a)` decimal(10,2) DEFAULT NULL, + `COALESCE(a)` decimal(12,2) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +CREATE TABLE t1 AS SELECT a FROM t1dec102 UNION SELECT a FROM t1dec102; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` decimal(12,2) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +DROP TABLE t1dec102; +# # End of 10.2 tests # diff --git a/mysql-test/t/type_decimal.test b/mysql-test/t/type_decimal.test index e8b3ecbf0a9..c5b64846f65 100644 --- a/mysql-test/t/type_decimal.test +++ b/mysql-test/t/type_decimal.test @@ -627,6 +627,27 @@ SELECT CAST(TIME'00:00:00.123456' AS DECIMAL(10,6)) AS c2; SET sql_mode=DEFAULT; +--echo # +--echo # MDEV-12574 MAX(old_decimal) produces a column of the old DECIMAL type +--echo # + +let $MYSQLD_DATADIR= `select @@datadir`; +--copy_file std_data/old_decimal/t1dec102.frm $MYSQLD_DATADIR/test/t1dec102.frm +--copy_file std_data/old_decimal/t1dec102.MYD $MYSQLD_DATADIR/test/t1dec102.MYD +--copy_file std_data/old_decimal/t1dec102.MYI $MYSQLD_DATADIR/test/t1dec102.MYI + +SHOW CREATE TABLE t1dec102; + +CREATE TABLE t1 AS SELECT a, MAX(a), COALESCE(a) FROM t1dec102; +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 AS SELECT a FROM t1dec102 UNION SELECT a FROM t1dec102; +SHOW CREATE TABLE t1; +DROP TABLE t1; + +DROP TABLE t1dec102; + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/field.cc b/sql/field.cc index f540c58757f..e84658bb64d 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -2973,6 +2973,23 @@ void Field_decimal::sql_type(String &res) const } +Field *Field_decimal::make_new_field(MEM_ROOT *root, TABLE *new_table, + bool keep_type) +{ + if (keep_type) + return Field_real::make_new_field(root, new_table, keep_type); + + Field *field= new (root) Field_new_decimal(NULL, field_length, + maybe_null() ? (uchar*) "" : 0, 0, + NONE, field_name, + dec, flags & ZEROFILL_FLAG, + unsigned_flag); + if (field) + field->init_for_make_new_field(new_table, orig_table); + return field; +} + + /**************************************************************************** ** Field_new_decimal ****************************************************************************/ @@ -7439,15 +7456,7 @@ Field *Field_string::make_new_field(MEM_ROOT *root, TABLE *new_table, This is done to ensure that ALTER TABLE will convert old VARCHAR fields to now VARCHAR fields. */ - field->init(new_table); - /* - Normally orig_table is different from table only if field was - created via ::make_new_field. Here we alter the type of field, - so ::make_new_field is not applicable. But we still need to - preserve the original field metadata for the client-server - protocol. - */ - field->orig_table= orig_table; + field->init_for_make_new_field(new_table, orig_table); } return field; } diff --git a/sql/field.h b/sql/field.h index 820cc5f3a7a..22c276478b6 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1368,6 +1368,18 @@ public: orig_table= table= table_arg; set_table_name(&table_arg->alias); } + void init_for_make_new_field(TABLE *new_table_arg, TABLE *orig_table_arg) + { + init(new_table_arg); + /* + Normally orig_table is different from table only if field was + created via ::make_new_field. Here we alter the type of field, + so ::make_new_field is not applicable. But we still need to + preserve the original field metadata for the client-server + protocol. + */ + orig_table= orig_table_arg; + } /* maximum possible display length */ virtual uint32 max_display_length()= 0; @@ -1828,6 +1840,7 @@ public: unireg_check_arg, field_name_arg, dec_arg, zero_arg, unsigned_arg) {} + Field *make_new_field(MEM_ROOT *root, TABLE *new_table, bool keep_type); enum_field_types type() const { return MYSQL_TYPE_DECIMAL;} enum ha_base_keytype key_type() const { return zerofill ? HA_KEYTYPE_BINARY : HA_KEYTYPE_NUM; } From bb825194b87da63126960656e86a18692b381b12 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Mon, 25 Jun 2018 14:04:16 +0300 Subject: [PATCH 181/203] Updated list of unstable tests for 10.2.16 --- mysql-test/unstable-tests | 219 ++++++++++++++++++++++++-------------- 1 file changed, 142 insertions(+), 77 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 23d1924fff8..00a914cc266 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,102 +23,115 @@ # ############################################################################## -# Based on 10.2 4ab180ad5e9c67427b561035e4b8e193d429fe5a +# Based on 10.2 ee6ac4d313c5e8846c4f7b92a923154984528215 +main.alter_table : Modified in 10.2.16 main.alter_table_errors : Added in 10.2.15 main.analyze_stmt_slow_query_log : MDEV-12237 - Wrong result main.assign_key_cache : Added in 10.2.15 main.assign_key_cache_debug : Added in 10.2.15 main.auth_named_pipe : MDEV-14724 - System error 2 -main.case : Modified in 10.2.14 +main.auto_increment : Modified in 10.2.16 +main.check : Modified in 10.2.16 main.check_constraint : Modified in 10.2.15 main.connect : Modified in 10.2.15 main.connect_debug : Added in 10.2.15 main.connect2 : MDEV-13885 - Server crash -main.cte_nonrecursive : Modified in 10.2.14 -main.cte_recursive : Modified in 10.2.15 -main.ctype_latin1 : Modified in 10.2.14 +main.create_or_replace : Modified in 10.2.16 +main.cte_nonrecursive : Modified in 10.2.16 +main.cte_recursive : Modified in 10.2.16 main.ctype_ucs : Modified in 10.2.15 -main.ctype_utf8 : Modified in 10.2.14 main.ctype_utf8mb4 : Modified in 10.2.15 -main.derived_cond_pushdown : Modified in 10.2.15 -main.distinct : MDEV-14194 - Crash +main.derived_cond_pushdown : Modified in 10.2.16 +main.distinct : MDEV-14194 - Crash; modified in 10.2.16 main.drop_bad_db_type : MDEV-15676 - Wrong result main.events_2 : MDEV-13277 - Crash +main.events_bugs : MDEV-12892 - Crash main.events_slowlog : MDEV-12821 - Wrong result -main.fast_prefix_index_fetch_innodb : Modified in 10.2.14 -main.func_date_add : Modified in 10.2.14 -main.func_json : Modified in 10.2.14 +main.explain_slowquerylog : Modified in 10.2.16 +main.func_json : Modified in 10.2.16 main.func_str : Modified in 10.2.15 -main.func_time : Modified in 10.2.14 -main.having : Modified in 10.2.14 +main.grant : Modified in 10.2.16 +main.grant2 : Modified in 10.2.16 +main.grant_not_windows : Added in 10.2.16 +main.having : Modified in 10.2.16 main.index_merge_innodb : MDEV-7142 - Plan mismatch main.innodb_mysql_lock : MDEV-7861 - Wrong result -main.join_outer : Modified in 10.2.14 +main.insert_select : Modified in 10.2.16 main.kill-2 : MDEV-13257 - Wrong result main.kill_processlist-6619 : MDEV-10793 - Wrong result +main.limit : Modified in 10.2.16 main.lock : Modified in 10.2.15 main.log_slow : MDEV-13263 - Wrong result main.mdev375 : Modified in 10.2.15 main.mdev-504 : MDEV-15171 - warning main.myisam : Modified in 10.2.15 main.myisam_recover : Modified in 10.2.15 +main.mysql : Modified in 10.2.16 main.mysql_client_test_nonblock : CONC-208 - Error on Power; MDEV-15096 - exec failed +main.mysql_cp932 : Modified in 10.2.16 main.mysql_upgrade_noengine : MDEV-14355 - Wrong result main.mysql_upgrade_ssl : MDEV-13492 - Unknown SSL error -main.mysqldump : MDEV-14800 - Stack smashing detected +main.mysqldump : MDEV-14800 - Stack smashing detected; modified in 10.2.16 main.mysqld_option_err : MDEV-12747 - Timeout main.mysqlhotcopy_myisam : MDEV-10995 - Hang on debug +main.mysqlslap : Modified in 10.2.16 main.mysqltest : MDEV-13887 - Wrong result +main.olap : Modified in 10.2.16 main.openssl_1 : MDEV-13492 - Unknown SSL error main.order_by_optimizer_innodb : MDEV-10683 - Wrong result main.parser : Modified in 10.2.15 main.partition_debug_sync : MDEV-15669 - Deadlock found when trying to get lock main.partition_list : Modified in 10.2.15 -main.ps : MDEV-11017 - Wrong result; modified in 10.2.15 -main.ps_qc_innodb : Added in 10.2.14 -main.query_cache : MDEV-16180 - Wrong result; modified in 10.2.14 +main.partition_open_files_limit : Modified in 10.2.16 +main.ps : MDEV-11017 - Wrong result; modified in 10.2.16 +main.query_cache : MDEV-16180 - Wrong result main.query_cache_debug : MDEV-15281 - Query cache is disabled main.range_vs_index_merge_innodb : MDEV-15283 - Server has gone away -main.read_only_innodb : Modified in 10.2.15 +main.read_only_innodb : Modified in 10.2.16 +main.rename : Modified in 10.2.16 main.select : MDEV-15430 - Wrong result with clang-4 main.select_jcl6 : MDEV-15430 - Wrong result with clang-4 main.select_pkeycache : MDEV-15430 - Wrong result with clang-4 +main.selectivity : Modified in 10.2.16 +main.set_password : Added in 10.2.16 main.set_statement : MDEV-13183 - Wrong result -main.shutdown : Modified in 10.2.14 +main.show_grants_with_plugin-7985 : Modified in 10.2.16 main.shm : MDEV-12727 - Mismatch, ERROR 2013 main.show_explain : MDEV-10674 - Wrong result code -main.sp : MDEV-7866 - Mismatch; modified in 10.2.15 +main.sp : MDEV-7866 - Mismatch; modified in 10.2.16 main.sp-destruct : Modified in 10.2.15 main.sp-innodb : Modified in 10.2.15 +main.sp-security : Modified in 10.2.16 main.ssl_ca : MDEV-10895 - SSL connection error on Power main.ssl_cert_verify : MDEV-13735 - Server crash main.ssl_connect : MDEV-13492 - Unknown SSL error main.ssl_timeout : MDEV-11244 - Crash main.stat_tables_par : MDEV-13266 - Wrong result +main.stat_tables_par_innodb : MDEV-14155 - Wrong rounding main.statistics : Modified in 10.2.15 +main.statistics_close : Modified in 10.2.16 main.status : MDEV-13255 - Wrong result main.subselect4 : Modified in 10.2.15 main.subselect-crash_15755 : Added in 10.2.15 main.subselect_innodb : MDEV-10614 - Wrong result -main.subselect_mat : Modified in 10.2.14 main.subselect_sj : Modified in 10.2.15 +main.subselect_sj2_mat : Modified in 10.2.16 main.symlink-myisam-11902 : MDEV-15098 - Error 40 from storage engine main.tc_heuristic_recover : MDEV-14189 - Wrong result -main.type_blob : MDEV-15195 - Wrong result +main.trigger : Modified in 10.2.16 +main.type_bit : Modified in 10.2.16 +main.type_blob : MDEV-15195 - Wrong result; modified in 10.2.16 main.type_datetime_hires : MDEV-15430 - Wrong result with clang-4 -main.type_decimal : Modified in 10.2.14 +main.type_decimal : Modified in 10.2.16 main.type_float : MDEV-15430 - Wrong result with clang-4 -main.type_temporal_innodb : Modified in 10.2.14 -main.type_time : Modified in 10.2.14 +main.type_int : Modified in 10.2.16 main.type_time_hires : MDEV-15430 - Wrong result with clang-4 main.type_timestamp_hires : MDEV-15430 - Wrong result with clang-4 main.userstat : MDEV-12904 - SSL errors main.variables : Modified in 10.2.15 main.view : Modified in 10.2.15 -main.warnings : Modified in 10.2.14 main.win : Modified in 10.2.15 -main.xa : Modified in 10.2.14 #---------------------------------------------------------------- @@ -129,14 +142,14 @@ archive.mysqlhotcopy_archive : MDEV-10995 - Hang on debug binlog.binlog_commit_wait : MDEV-10150 - Mismatch binlog.binlog_flush_binlogs_delete_domain : MDEV-14431 - Wrong exit code -binlog.binlog_stm_datetime_ranges_mdev15289 : Added in 10.2.14 binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- binlog_encryption.binlog_xa_recover : MDEV-12908 - Extra checkpoint -binlog_encryption.encrypted_master : MDEV-14201 - Extra warnings +binlog_encryption.encrypted_master : MDEV-14201 - Extra warnings; modified in 10.2.16 binlog_encryption.encrypted_master_switch_to_unencrypted : MDEV-14190 - Can't init tc log +binlog_encryption.encrypted_slave : Modified in 10.2.16 binlog_encryption.encryption_combo : MDEV-14199 - Table is marked as crashed binlog_encryption.rpl_binlog_errors : MDEV-12742 - Crash binlog_encryption.rpl_parallel : MDEV-10653 - Timeout in include @@ -150,43 +163,35 @@ binlog_encryption.rpl_typeconv : Include file modified in 10.2.15 #---------------------------------------------------------------- -connect.jdbc_postgresql : Modified in 10.2.14 -connect.json_udf : Modified in 10.2.14 connect.pivot : MDEV-14803 - Failed to discover table -connect.tbl_thread : Modified in 10.2.14 -connect.vcol : MDEV-12374 - Fails on Windows; modified in 10.2.14 - -#---------------------------------------------------------------- - -disks.disks : Added in 10.2.14 +connect.vcol : MDEV-12374 - Fails on Windows #---------------------------------------------------------------- encryption.create_or_replace : MDEV-12694 - Timeout; MDEV-16115 - Trying to access tablespace encryption.debug_key_management : MDEV-13841 - Timeout -encryption.encrypt_and_grep : MDEV-13765 - Wrong result; modified in 10.2.14 +encryption.encrypt_and_grep : MDEV-13765 - Wrong result encryption.innochecksum : MDEV-13644 - Assertion failure -encryption.innodb-bad-key-change : Modified in 10.2.14 +encryption.innodb-checksum-algorithm : MDEV-12898 - Deadlock of threads encryption.innodb-compressed-blob : MDEV-14728 - Unable to get certificate +encryption.innodb-discard-import : Modified in 10.2.16 encryption.innodb_encrypt_log : MDEV-13725 - Wrong result -encryption.innodb_encryption : MDEV-15675 - Timeout; modified in 10.2.14 +encryption.innodb_encryption : MDEV-15675 - Timeout encryption.innodb-encryption-alter : MDEV-13566 - Lock wait timeout encryption.innodb_encryption_discard_import : MDEV-16116 - Wrong result; modified in 10.2.15 encryption.innodb_encryption_filekeys : MDEV-15673 - Timeout; modified in 10.2.15 -encryption.innodb_encryption-page-compression : MDEV-12630 - crash or assertion failure; modified in 10.2.14 +encryption.innodb_encryption-page-compression : MDEV-12630 - crash or assertion failure encryption.innodb_encryption_row_compressed : MDEV-16113 - Crash encryption.innodb-first-page-read : MDEV-14356 - Timeout in wait condition encryption.innodb_lotoftables : Modified in 10.2.15 +encryption.innodb-read-only : MDEV-16563 - Crash on startup encryption.innodb-redo-badkey : MDEV-13893 - Page cannot be decrypted; include file modified in 10.2.15 -encryption.innodb-redo-nokeys : Include file modified in 10.2.15 -encryption.innodb-remove-encryption : Added in 10.2.15 -encryption.innodb_scrub_background : Modified in 10.2.14 -encryption.innodb-spatial-index : MDEV-13746 - Wrong result; modified in 10.2.14 +encryption.innodb-redo-nokeys : Modified in 10.2.16 +encryption.innodb-remove-encryption : MDEV-16493 - Timeout in wait condition; added in 10.2.15 +encryption.innodb-spatial-index : MDEV-13746 - Wrong result #---------------------------------------------------------------- -engines/iuds.update_time : Modified in 10.2.14 - engines/rr_trx.* : MDEV-10998 - Not maintained #---------------------------------------------------------------- @@ -209,22 +214,31 @@ galera_3nodes.* : Suite is not stable yet #---------------------------------------------------------------- +gcol.innodb_virtual_debug : Modified in 10.2.16 gcol.innodb_virtual_fk : Modified in 10.2.15 gcol.innodb_virtual_index : Modified in 10.2.15 #---------------------------------------------------------------- +handler.ps : Added in 10.2.16 + +#---------------------------------------------------------------- + +heap.heap_auto_increment : Modified in 10.2.16 + +#---------------------------------------------------------------- + innodb.101_compatibility : MDEV-13891 - Wrong result innodb.alter_copy : MDEV-16181 - Assertion failure +innodb.alter_foreign_crash : Added in 10.2.16 +innodb.alter_kill : Added in 10.2.16 innodb.alter_missing_tablespace : Modified in 10.2.15 innodb.alter_partitioned_debug : Added in 10.2.15 innodb.alter_partitioned_xa : Added in 10.2.15 +innodb.alter_rename_files : Added in 10.2.16 +innodb.analyze_table : Added in 10.2.16 innodb.autoinc_persist : MDEV-15282 - Assertion failure -innodb.default_row_format_alter : Added in 10.2.14 -innodb.default_row_format_compatibility : Added in 10.2.14 -innodb.default_row_format_create : Added in 10.2.14 innodb.doublewrite : MDEV-12905 - Server crash; include file modified in 10.2.15 -innodb.file_format_defaults : Added in 10.2.14 innodb.foreign_key : Modified in 10.2.15 innodb.group_commit_crash : MDEV-14191 - InnoDB registration failed innodb.group_commit_crash_no_optimize_thread : MDEV-13830 - Assertion failure @@ -232,13 +246,16 @@ innodb.innodb-64k-crash : MDEV-13872 - Failure and crash on startup innodb.innodb-alter : Modified in 10.2.15 innodb.innodb-alter-nullable : Modified in 10.2.15 innodb.innodb-alter-tempfile : MDEV-15285 - Table already exists +innodb.innodb-autoinc : Modified in 10.2.16 +innodb.innodb-blob : Modified in 10.2.16 innodb.innodb_bug14147491 : MDEV-11808 - Index is corrupt -innodb.innodb_bug27216817 : Added in 10.2.15 innodb.innodb_bug30423 : MDEV-7311 - Wrong result innodb.innodb_bug48024 : MDEV-14352 - Assertion failure +innodb.innodb_bug54044 : Modified in 10.2.16 innodb.innodb_bug59641 : MDEV-13830 - Assertion failure innodb.innodb_bulk_create_index_replication : MDEV-15273 - Slave failed to start innodb.innodb_defrag_stats_many_tables : MDEV-14198 - Table is full +innodb.innodb_defragment_small : Modified in 10.2.16 innodb.innodb-get-fk : MDEV-13276 - Server crash innodb.innodb-index : Modified in 10.2.15 innodb.innodb-index-online : MDEV-14809 - Cannot save statistics @@ -246,9 +263,13 @@ innodb.innodb_information_schema : MDEV-8851 - Wrong result innodb.innodb-isolation : Modified in 10.2.15 innodb.innodb_max_recordsize_32k : MDEV-14801 - Operation failed innodb.innodb_max_recordsize_64k : MDEV-15203 - Wrong result -innodb.innodb-page_compression_default : MDEV-13644 - Assertion failure +innodb.innodb-mdev7046 : Modified in 10.2.16 +innodb.innodb-online-alter-gis : Modified in 10.2.16 +innodb.innodb-page_compression_default : MDEV-13644 - Assertion failure; modified in 10.2.16 innodb.innodb-page_compression_lzma : MDEV-14353 - Wrong result +innodb.innodb-page_compression_snappy : Modified in 10.2.16 innodb.innodb_prefix_index_restart_server : Modified in 10.2.15 +innodb.innodb_query_cache : Added in 10.2.16 innodb.innodb_stats_persistent_debug : MDEV-14801 - Operation failed innodb.innodb-table-online : MDEV-13894 - Wrong result innodb.innodb_sys_semaphore_waits : MDEV-10331 - Semaphore wait @@ -263,12 +284,11 @@ innodb.log_file_name_debug : Include file modified in 10.2.15 innodb.log_file_size : MDEV-15668 - Not found pattern innodb.mdev-15707 : Added in 10.2.15 innodb.monitor : MDEV-16179 - Wrong result -innodb.purge_secondary : MDEV-15681 - Wrong result; modified in 10.2.14 +innodb.purge_secondary : MDEV-15681 - Wrong result innodb.purge_thread_shutdown : MDEV-13792 - Wrong result innodb.read_only_recovery : MDEV-13886 - Server crash -innodb.read_only_recover_committed : Added in 10.2.14 innodb.recovery_shutdown : MDEV-15671 - Checksum mismatch in datafile -innodb.restart : Added in 10.2.14 +innodb.rename_table : Modified in 10.2.16 innodb.row_format_redundant : MDEV-15192 - Trying to access missing tablespace innodb.stored_fk : Added in 10.2.15 innodb.table_definition_cache_debug : MDEV-14206 - Extra warning @@ -303,28 +323,39 @@ innodb_zip.wl6501_scale_1 : MDEV-13254 - Timeout, MDEV-14104 - Error #---------------------------------------------------------------- -maria.dynamic : Added in 10.2.14 +maria.alter : Modified in 10.2.16 maria.insert_select : MDEV-12757 - Timeout -maria.maria : MDEV-14430 - Extra warning +maria.insert_select-7314 : MDEV-16492 - Timeout +maria.lock : Modified in 10.2.16 +maria.maria : MDEV-14430 - Extra warning; modified in 10.2.16 +maria.maria-autoinc : Added in 10.2.16 #---------------------------------------------------------------- mariabackup.absolute_ibdata_paths : Added in 10.2.15 mariabackup.apply-log-only : MDEV-14192 - Assertion failure -mariabackup.apply-log-only-incr : MDEV-14192 - Assertion failure -mariabackup.backup_ssl : Added in 10.2.15 +mariabackup.apply-log-only-incr : MDEV-14192 - Assertion failure; modified in 10.2.16 +mariabackup.backup_ssl : MDEV-14192 - Assertion failure; added in 10.2.15 mariabackup.data_directory : MDEV-15270 - Error on exec -mariabackup.extra_lsndir : Added in 10.2.14 +mariabackup.full_backup : MDEV-16571 - Wrong result +mariabackup.huge_lsn : MDEV-15662 - Sequence number is in the future mariabackup.incremental_backup : MDEV-14192 - Assertion failure -mariabackup.incremental_encrypted : MDEV-14188 - Wrong result, MDEV-15667 - timeout -mariabackup.mdev-14447 : MDEV-15201 - Timeout +mariabackup.incremental_encrypted : MDEV-14188 - Wrong result, MDEV-15667 - timeout, MDEV-14192 - Assertion failure +mariabackup.lock_ddl_per_table : Modified in 10.2.16 +mariabackup.log_checksum_mismatch : MDEV-16568 - Sequence number is in the future +mariabackup.mdev-14447 : MDEV-15201 - Timeout; modified in 10.2.16 mariabackup.partial_exclude : MDEV-15270 - Error on exec -mariabackup.undo_space_id : Added in 10.2.14 +mariabackup.partition_datadir : Modified in 10.2.16 +mariabackup.rename_during_mdl_lock : Added in 10.2.16 mariabackup.unsupported_redo : Modified in 10.2.15 mariabackup.xbstream : MDEV-14192 - Crash mariabackup.xb_aws_key_management : MDEV-15680 - Error: xtrabackup_copy_logfile() failed mariabackup.xb_page_compress : MDEV-14810 - status: 1, errno: 11 mariabackup.xb_compressed_encrypted : MDEV-14812 - Segmentation fault +mariabackup.xb_file_key_management : MDEV-16569 - Table doesn't exist +mariabackup.xb_rocksdb : Added in 10.2.16 +mariabackup.xb_rocksdb_datadir : Added in 10.2.16 +mariabackup.xb_rocksdb_datadir_debug : Added in 10.2.16 #---------------------------------------------------------------- @@ -342,6 +373,7 @@ multi_source.simple : MDEV-4633 - Wrong result parts.partition_alter_innodb : Include file modified in 10.2.15 parts.partition_alter_maria : Include file modified in 10.2.15 parts.partition_alter_myisam : Include file modified in 10.2.15 +parts.partition_auto_increment_archive : MDEV-16491 - Marked as crashed and should be repaired parts.partition_auto_increment_maria : MDEV-14430 - Extra warning parts.partition_debug_innodb : MDEV-10891 - Can't create UNIX socket; MDEV-15095 - Table doesn't exist parts.show_create : Added in 10.2.15 @@ -362,6 +394,7 @@ perfschema.hostcache_ipv6_addrinfo_again_allow : MDEV-12752 - Crash perfschema.hostcache_ipv6_addrinfo_bad_allow : MDEV-13260 - Crash perfschema.hostcache_ipv6_max_con : Modified in 10.2.15 perfschema.hostcache_ipv6_ssl : MDEV-10696 - Crash +perfschema.partition : Added in 10.2.16 perfschema.setup_actors : MDEV-10679 - Crash perfschema.socket_connect : MDEV-15677 - Wrong result perfschema.socket_summary_by_event_name_func : MDEV-10622 - Wrong result @@ -375,48 +408,77 @@ perfschema_stress.* : MDEV-10996 - Not maintained #---------------------------------------------------------------- plugins.feedback_plugin_send : MDEV-7932, MDEV-11118 - Connection problems and such +plugins.processlist : MDEV-16574 - Wrong result; modified in 10.2.16 plugins.server_audit : Modified in 10.2.15 plugins.thread_pool_server_audit : MDEV-14295 - Wrong result #---------------------------------------------------------------- -rocksdb.* : Tests are unstable -rocksdb_sys_vars.* : Tests are unstable +rocksdb.2pc_group_commit : MDEV-14455 - Wrong result +rocksdb.allow_no_primary_key : Modified in 10.2.16 +rocksdb.analyze_table : Modified in 10.2.16 +rocksdb.autoinc_debug : MDEV-16203 - Wrong result +rocksdb.autoinc_vars_thread : MDEV-16573 - Debug sync timed out +rocksdb.bloomfilter2 : MDEV-16564 - Wrong result +rocksdb.bulk_load_errors : MDEV-16575 - Wrong result +rocksdb.check_ignore_unknown_options : Modified in 10.2.16 +rocksdb.deadlock : MDEV-16033 - Timeout +rocksdb.drop_index_inplace : MDEV-14162 - Crash on shutdown +rocksdb.drop_table : MDEV-14308 - Timeout +rocksdb.issue255 : MDEV-16577 - Wrong plan; modified in 10.2.16 +rocksdb.mariadb_port_fixes : MDEV-16387 - Wrong plan; modified in 10.2.16 +rocksdb.read_only_tx : MDEV-16565 - Server crash +rocksdb.rocksdb_parts : MDEV-13843 - Wrong result +rocksdb.truncate_table3 : MDEV-14506 - Lost connection to server +rocksdb.ttl_primary_read_filtering : MDEV-16560 - Wrong result +rocksdb.ttl_secondary_read_filtering : MDEV-16560 - Wrong result +rocksdb.unique_check : MDEV-16576 - Wrong errno + +rocksdb_sys_vars.rocksdb_remove_mariabackup_checkpoint_basic : Added in 10.2.16 #---------------------------------------------------------------- +roles.acl_load_mutex-5170 : Modified in 10.2.16 +roles.grant_revoke_current : Modified in 10.2.16 +roles.set_default_role_ps-6960 : Modified in 10.2.16 + +#---------------------------------------------------------------- + +rpl.rename : Added in 10.2.16 rpl.rpl_binlog_errors : MDEV-12742 - Crash rpl.rpl_binlog_index : MDEV-9501 - Failed registering on master rpl.rpl_colSize : MDEV-16112 - Server crash rpl.rpl_ctype_latin1 : MDEV-14813 - Wrong result on Mac +rpl.rpl_do_grant : Modified in 10.2.16 rpl.rpl_domain_id_filter_io_crash : MDEV-12729 - Timeout in include file, MDEV-13677 - Server crash rpl.rpl_domain_id_filter_restart : MDEV-10684 - Wrong result +rpl.rpl_extra_col_master_innodb : MDEV-16570 - Extra warning rpl.rpl_extra_col_master_myisam : MDEV-14203 - Extra warning rpl.rpl_gtid_crash : MDEV-9501 - Failed registering on master, MDEV-13643 - Lost connection rpl.rpl_gtid_delete_domain : MDEV-14463 - Timeout rpl.rpl_gtid_errorhandling : MDEV-13261 - Crash rpl.rpl_gtid_reconnect : MDEV-14497 - Crash -rpl.rpl_gtid_stop_start : MDEV-11621 - Table marked as crashed rpl.rpl_insert_id : MDEV-15197 - Wrong result +rpl.rpl_insert_id_pk : MDEV-16567 - Assertion failure rpl.rpl_mariadb_slave_capability : MDEV-11018 - Extra lines in binlog rpl.rpl_mdev6020 : MDEV-15272 - Server crash rpl.rpl_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed +rpl.rpl_non_direct_row_mixing_engines : MDEV-16561 - Timeout in master_pos_wait rpl.rpl_non_direct_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed -rpl.rpl_non_direct_row_mixing_engines : MDEV-14491 - Long semaphore wait rpl.rpl_non_direct_stm_mixing_engines : MDEV-14489 - Failed sync_slave_with_master rpl.rpl_parallel : MDEV-10653 - Timeouts rpl.rpl_parallel_conflicts : MDEV-15272 - Server crash rpl.rpl_parallel_mdev6589 : MDEV-12979 - Assertion failure -rpl.rpl_parallel_optimistic : MDEV-15278 - Failed to sync with master +rpl.rpl_parallel_optimistic : MDEV-15278 - Failed to sync with master; modified in 10.2.16 rpl.rpl_parallel_optimistic_nobinlog : MDEV-15278 - Failed to sync with master -rpl.rpl_parallel_retry : MDEV-11119 - Crash; modified in 10.2.14 +rpl.rpl_parallel_retry : MDEV-11119 - Crash rpl.rpl_parallel_temptable : MDEV-10356 - Crash rpl.rpl_row_basic_2myisam : MDEV-13875 - command "diff_files" failed rpl.rpl_row_drop_create_temp_table : MDEV-14487 - Wrong result rpl.rpl_row_img_eng_min : MDEV-13875 - diff_files failed rpl.rpl_row_img_eng_noblob : MDEV-13875 - command "diff_files" failed rpl.rpl_row_index_choice : MDEV-15196 - Slave crash -rpl.rpl_row_mixing_engines : MDEV-14491 - Long semaphore wait +rpl.rpl_row_mixing_engines : MDEV-16515 - Assertion failure rpl.rpl_semi_sync : MDEV-11220 - Wrong result rpl.rpl_semi_sync_after_sync : MDEV-14366 - Wrong result rpl.rpl_semi_sync_after_sync_row : MDEV-14366 - Wrong result @@ -449,7 +511,6 @@ rpl/extra/rpl_tests.* : MDEV-10994 - Not maintained spider.basic_sql : MDEV-11186 - Internal check fails spider/bg.direct_aggregate : MDEV-7098 - Packets out of order -spider/bg.ha_part : MDEV-7914 - Crash (only fixed in 10.3) spider/bg.spider3_fixes : MDEV-12639 - Syntax error spider/handler.* : MDEV-10990 - Not maintained @@ -467,13 +528,13 @@ storage_engine.* : Not always timely maintained sys_vars.innodb_buffer_pool_dump_at_shutdown_basic : MDEV-14280 - Unexpected error sys_vars.max_prepared_stmt_count_basic : Modified in 10.2.15 +sys_vars.maximum_basic : Modified in 10.2.16 sys_vars.rpl_init_slave_func : MDEV-10149 - Test assertion sys_vars.slow_query_log_func : MDEV-14273 - Wrong result sys_vars.sysvars_innodb : Opt file modified in 10.2.15 sys_vars.sysvars_server_embedded : Opt file added in 10.2.15 sys_vars.sysvars_server_notembedded : Opt file added in 10.2.15 sys_vars.thread_cache_size_func : MDEV-11775 - Wrong result -sys_vars.wsrep_sst_receive_address_basic : Modified in 10.2.14 #---------------------------------------------------------------- @@ -481,6 +542,7 @@ tokudb.change_column_all_1000_10 : MDEV-12640 - Lost connection tokudb.change_column_bin : MDEV-12640 - Lost connection tokudb.change_column_char : MDEV-12822 - Lost connection tokudb.dir_per_db : MDEV-11537 - Wrong result +tokudb.hotindex-del-0 : MDEV-16559 - Timeout tokudb.hotindex-insert-0 : MDEV-15271 - Timeout tokudb.hotindex-insert-2 : MDEV-15271 - Timeout tokudb.hotindex-insert-bigchar : MDEV-12640 - Crash @@ -500,8 +562,6 @@ tokudb_backup.* : MDEV-11001 - Missing include file tokudb_sys_vars.* : MDEV-11001 - Missing include file tokudb_rpl.* : MDEV-11001 - Missing include file -tokudb_mariadb.mdev6657 : Modified in 10.2.14 - tokudb_parts.partition_alter4_tokudb : MDEV-12640 - Lost connection #---------------------------------------------------------------- @@ -514,7 +574,12 @@ unit.my_atomic : MDEV-15670 - Signal 11 thrown #---------------------------------------------------------------- +vcol.binlog : Added in 10.2.16 +vcol.index : Added in 10.2.16 vcol.partition : Modified in 10.2.15 +vcol.update : Modified in 10.2.16 +vcol.update_binlog : Added in 10.2.16 +vcol.vcol_misc : Modified in 10.2.16 #---------------------------------------------------------------- @@ -522,7 +587,7 @@ wsrep.binlog_format : MDEV-11532 - Could not execute check-testca wsrep.foreign_key : MDEV-14725 - WSREP has not yet prepared node wsrep.mdev_6832 : MDEV-14195 - Check testcase failed wsrep.pool_of_threads : MDEV-12234 - GLIBCXX_3.4.20 not found -wsrep.variables : MDEV-14311 - Wrong result +wsrep.variables : MDEV-14311 - Wrong result; modified in 10.2.16 wsrep_info.plugin : MDEV-13569 - No nodes coming from prim view From 28e1f1453f12b8b748c31a86a22b336f62002654 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Tue, 19 Jun 2018 18:14:47 +0300 Subject: [PATCH 182/203] MDEV-15242 Poor RBR update performance with partitioned tables Observed and described partitioned engine execution time difference between master and slave was caused by excessive invocation of base_engine::rnd_init which was done also for partitions uninvolved into Rows-event operation. The bug's slave slowdown therefore scales with the number of partitions. Fixed with applying an upstream patch. References: ---------- https://bugs.mysql.com/bug.php?id=73648 Bug#25687813 REPLICATION REGRESSION WITH RBR AND PARTITIONED TABLES --- sql/ha_partition.cc | 3 ++- sql/handler.h | 10 +++++++++- sql/log_event.cc | 4 ---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 6a6627f9276..0488ebfb60f 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -5090,7 +5090,8 @@ int ha_partition::rnd_pos_by_record(uchar *record) if (unlikely(get_part_for_delete(record, m_rec0, m_part_info, &m_last_part))) DBUG_RETURN(1); - DBUG_RETURN(handler::rnd_pos_by_record(record)); + int err= m_file[m_last_part]->rnd_pos_by_record(record); + DBUG_RETURN(err); } diff --git a/sql/handler.h b/sql/handler.h index 74d50536ec4..52396b84c0d 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -3042,9 +3042,17 @@ private: */ virtual int rnd_pos_by_record(uchar *record) { + int error; DBUG_ASSERT(table_flags() & HA_PRIMARY_KEY_REQUIRED_FOR_POSITION); + + error = ha_rnd_init(false); + if (error != 0) + return error; + position(record); - return rnd_pos(record, ref); + error = ha_rnd_pos(record, ref); + ha_rnd_end(); + return error; } virtual int read_first_row(uchar *buf, uint primary_key); public: diff --git a/sql/log_event.cc b/sql/log_event.cc index 7989db9c687..3638269cbf5 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -12135,10 +12135,6 @@ int Rows_log_event::find_row(rpl_group_info *rgi) int error; DBUG_PRINT("info",("locating record using primary key (position)")); - if (!table->file->inited && - (error= table->file->ha_rnd_init_with_error(0))) - DBUG_RETURN(error); - error= table->file->ha_rnd_pos_by_record(table->record[0]); if (error) { From a8e1eef89940892f7ae87a512c2ace60372d450a Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Fri, 9 Mar 2018 14:26:10 +0100 Subject: [PATCH 183/203] Reset connection support in mysqltest (port from mysql) --- client/mysqltest.cc | 28 ++++++++++++++++++++++++++++ mysql-test/r/reset_connection.result | 7 +++++++ mysql-test/t/reset_connection.test | 23 +++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 mysql-test/r/reset_connection.result create mode 100644 mysql-test/t/reset_connection.test diff --git a/client/mysqltest.cc b/client/mysqltest.cc index aadb46da0c1..ebfa9ec0bb8 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -384,6 +384,7 @@ enum enum_commands { Q_RESULT_FORMAT_VERSION, Q_MOVE_FILE, Q_REMOVE_FILES_WILDCARD, Q_SEND_EVAL, Q_ENABLE_PREPARE_WARNINGS, Q_DISABLE_PREPARE_WARNINGS, + Q_RESET_CONNECTION, Q_UNKNOWN, /* Unknown command. */ Q_COMMENT, /* Comments, ignored. */ Q_COMMENT_WITH_COMMAND, @@ -491,6 +492,7 @@ const char *command_names[]= "send_eval", "enable_prepare_warnings", "disable_prepare_warnings", + "reset_connection", 0 }; @@ -6503,6 +6505,29 @@ void do_delimiter(struct st_command* command) } +/* + do_reset_connection + + DESCRIPTION + Reset the current session. +*/ + +static void do_reset_connection() +{ + MYSQL *mysql = cur_con->mysql; + + DBUG_ENTER("do_reset_connection"); + if (mysql_reset_connection(mysql)) + die("reset connection failed: %s", mysql_error(mysql)); + if (cur_con->stmt) + { + mysql_stmt_close(cur_con->stmt); + cur_con->stmt= NULL; + } + DBUG_VOID_RETURN; +} + + my_bool match_delimiter(int c, const char *delim, uint length) { uint i; @@ -9543,6 +9568,9 @@ int main(int argc, char **argv) case Q_PING: handle_command_error(command, mysql_ping(cur_con->mysql), -1); break; + case Q_RESET_CONNECTION: + do_reset_connection(); + break; case Q_SEND_SHUTDOWN: handle_command_error(command, mysql_shutdown(cur_con->mysql, diff --git a/mysql-test/r/reset_connection.result b/mysql-test/r/reset_connection.result new file mode 100644 index 00000000000..925195f704e --- /dev/null +++ b/mysql-test/r/reset_connection.result @@ -0,0 +1,7 @@ +FLUSH STATUS; +SHOW local STATUS LIKE 'com_select'; +Variable_name Value +Com_select 10 +SHOW local STATUS LIKE 'com_select'; +Variable_name Value +Com_select 0 diff --git a/mysql-test/t/reset_connection.test b/mysql-test/t/reset_connection.test new file mode 100644 index 00000000000..39465677483 --- /dev/null +++ b/mysql-test/t/reset_connection.test @@ -0,0 +1,23 @@ +FLUSH STATUS; + +--disable_result_log +--disable_query_log + +let $i = 10; +begin; +while ($i) +{ + dec $i; + SELECT 1; +} +commit; + +--enable_query_log +--enable_result_log + +SHOW local STATUS LIKE 'com_select'; + +--reset_connection + +SHOW local STATUS LIKE 'com_select'; + From 73de63e898bd67f708c6df8d154a76b7ddd0d5d7 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Fri, 9 Mar 2018 14:27:13 +0100 Subject: [PATCH 184/203] Session tracking info support in mysqltest (port from mysql) --- client/mysqltest.cc | 90 ++++++++++++++++++++- mysql-test/r/mysqltest_tracking_info.result | 47 +++++++++++ mysql-test/t/mysqltest_tracking_info.test | 25 ++++++ mysql-test/t/reset_connection.test | 2 + 4 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 mysql-test/r/mysqltest_tracking_info.result create mode 100644 mysql-test/t/mysqltest_tracking_info.test diff --git a/client/mysqltest.cc b/client/mysqltest.cc index ebfa9ec0bb8..513ef39d2a5 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -125,7 +125,8 @@ static my_bool view_protocol= 0, view_protocol_enabled= 0; static my_bool cursor_protocol= 0, cursor_protocol_enabled= 0; static my_bool parsing_disabled= 0; static my_bool display_result_vertically= FALSE, display_result_lower= FALSE, - display_metadata= FALSE, display_result_sorted= FALSE; + display_metadata= FALSE, display_result_sorted= FALSE, + display_session_track_info= FALSE; static my_bool disable_query_log= 0, disable_result_log= 0; static my_bool disable_connect_log= 0; static my_bool disable_warnings= 0, disable_column_names= 0; @@ -153,6 +154,7 @@ static struct property prop_list[] = { { &abort_on_error, 0, 1, 0, "$ENABLED_ABORT_ON_ERROR" }, { &disable_connect_log, 0, 1, 1, "$ENABLED_CONNECT_LOG" }, { &disable_info, 0, 1, 1, "$ENABLED_INFO" }, + { &display_session_track_info, 0, 1, 1, "$ENABLED_STATE_CHANGE_INFO" }, { &display_metadata, 0, 0, 0, "$ENABLED_METADATA" }, { &ps_protocol_enabled, 0, 0, 0, "$ENABLED_PS_PROTOCOL" }, { &disable_query_log, 0, 0, 1, "$ENABLED_QUERY_LOG" }, @@ -166,6 +168,7 @@ enum enum_prop { P_ABORT= 0, P_CONNECT, P_INFO, + P_SESSION_TRACK, P_META, P_PS, P_QUERY, @@ -362,6 +365,7 @@ enum enum_commands { Q_WAIT_FOR_SLAVE_TO_STOP, Q_ENABLE_WARNINGS, Q_DISABLE_WARNINGS, Q_ENABLE_INFO, Q_DISABLE_INFO, + Q_ENABLE_SESSION_TRACK_INFO, Q_DISABLE_SESSION_TRACK_INFO, Q_ENABLE_METADATA, Q_DISABLE_METADATA, Q_ENABLE_COLUMN_NAMES, Q_DISABLE_COLUMN_NAMES, Q_EXEC, Q_DELIMITER, @@ -436,6 +440,8 @@ const char *command_names[]= "disable_warnings", "enable_info", "disable_info", + "enable_session_track_info", + "disable_session_track_info", "enable_metadata", "disable_metadata", "enable_column_names", @@ -6514,6 +6520,7 @@ void do_delimiter(struct st_command* command) static void do_reset_connection() { +#ifndef EMBEDDED_LIBRARY MYSQL *mysql = cur_con->mysql; DBUG_ENTER("do_reset_connection"); @@ -6525,6 +6532,10 @@ static void do_reset_connection() cur_con->stmt= NULL; } DBUG_VOID_RETURN; +#else + die("reset connection failed: unsupported by embedded server client library"); + return; +#endif } @@ -7782,6 +7793,70 @@ void append_info(DYNAMIC_STRING *ds, ulonglong affected_rows, } +/** + @brief Append state change information (received through Ok packet) to the output. + + @param [in,out] ds Dynamic string to hold the content to be printed. + @param [in] mysql Connection handle. +*/ + +static void append_session_track_info(DYNAMIC_STRING *ds, MYSQL *mysql) +{ +#ifndef EMBEDDED_LIBRARY + for (unsigned int type= SESSION_TRACK_BEGIN; type <= SESSION_TRACK_END; type++) + { + const char *data; + size_t data_length; + + if (!mysql_session_track_get_first(mysql, + (enum_session_state_type) type, + &data, &data_length)) + { + dynstr_append(ds, "-- "); + switch (type) + { + case SESSION_TRACK_SYSTEM_VARIABLES: + dynstr_append(ds, "Tracker : SESSION_TRACK_SYSTEM_VARIABLES\n"); + break; + case SESSION_TRACK_SCHEMA: + dynstr_append(ds, "Tracker : SESSION_TRACK_SCHEMA\n"); + break; + case SESSION_TRACK_STATE_CHANGE: + dynstr_append(ds, "Tracker : SESSION_TRACK_STATE_CHANGE\n"); + break; + case SESSION_TRACK_GTIDS: + dynstr_append(ds, "Tracker : SESSION_TRACK_GTIDS\n"); + break; + case SESSION_TRACK_TRANSACTION_CHARACTERISTICS: + dynstr_append(ds, "Tracker : SESSION_TRACK_TRANSACTION_CHARACTERISTICS\n"); + break; + case SESSION_TRACK_TRANSACTION_TYPE: + dynstr_append(ds, "Tracker : SESSION_TRACK_TRANSACTION_TYPE\n"); + break; + default: + DBUG_ASSERT(0); + dynstr_append(ds, "\n"); + } + + + dynstr_append(ds, "-- "); + dynstr_append_mem(ds, data, data_length); + } + else + continue; + while (!mysql_session_track_get_next(mysql, + (enum_session_state_type) type, + &data, &data_length)) + { + dynstr_append(ds, "\n-- "); + dynstr_append_mem(ds, data, data_length); + } + dynstr_append(ds, "\n\n"); + } +#endif /* EMBEDDED_LIBRARY */ +} + + /* Display the table headings with the names tab separated */ @@ -7962,6 +8037,9 @@ void run_query_normal(struct st_connection *cn, struct st_command *command, if (!disable_info) append_info(ds, mysql_affected_rows(mysql), mysql_info(mysql)); + if (display_session_track_info) + append_session_track_info(ds, mysql); + /* Add all warnings to the result. We can't do this if we are in the middle of processing results from multi-statement, because @@ -8377,6 +8455,10 @@ void run_query_stmt(struct st_connection *cn, struct st_command *command, if (!disable_info) append_info(ds, mysql_stmt_affected_rows(stmt), mysql_info(mysql)); + if (display_session_track_info) + append_session_track_info(ds, mysql); + + if (!disable_warnings) { /* Get the warnings from execute */ @@ -9358,6 +9440,12 @@ int main(int argc, char **argv) case Q_DISABLE_INFO: set_property(command, P_INFO, 1); break; + case Q_ENABLE_SESSION_TRACK_INFO: + set_property(command, P_SESSION_TRACK, 1); + break; + case Q_DISABLE_SESSION_TRACK_INFO: + set_property(command, P_SESSION_TRACK, 0); + break; case Q_ENABLE_METADATA: set_property(command, P_META, 1); break; diff --git a/mysql-test/r/mysqltest_tracking_info.result b/mysql-test/r/mysqltest_tracking_info.result new file mode 100644 index 00000000000..b5885a37cde --- /dev/null +++ b/mysql-test/r/mysqltest_tracking_info.result @@ -0,0 +1,47 @@ +SELECT @@session.character_set_client, @@session.character_set_results, @@session.character_set_connection; +@@session.character_set_client @@session.character_set_results @@session.character_set_connection +latin1 latin1 latin1 +SET @@session.session_track_system_variables='character_set_client,character_set_results,character_set_connection'; +# tracking info on +SET NAMES 'utf8'; +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- character_set_client +-- utf8 +-- character_set_connection +-- utf8 +-- character_set_results +-- utf8 + +SET NAMES 'big5'; +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- character_set_client +-- big5 +-- character_set_connection +-- big5 +-- character_set_results +-- big5 + +# tracking info on once +SET NAMES 'utf8'; +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- character_set_client +-- utf8 +-- character_set_connection +-- utf8 +-- character_set_results +-- utf8 + +SET NAMES 'big5'; +# tracking info on +SET NAMES 'utf8'; +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- character_set_client +-- utf8 +-- character_set_connection +-- utf8 +-- character_set_results +-- utf8 + +# tracking info off once +SET NAMES 'big5'; +SET @@session.session_track_system_variables= default; diff --git a/mysql-test/t/mysqltest_tracking_info.test b/mysql-test/t/mysqltest_tracking_info.test new file mode 100644 index 00000000000..c32e20703a5 --- /dev/null +++ b/mysql-test/t/mysqltest_tracking_info.test @@ -0,0 +1,25 @@ + +--source include/no_protocol.inc +--source include/not_embedded.inc + +SELECT @@session.character_set_client, @@session.character_set_results, @@session.character_set_connection; +SET @@session.session_track_system_variables='character_set_client,character_set_results,character_set_connection'; + +--echo # tracking info on +--enable_session_track_info +SET NAMES 'utf8'; +SET NAMES 'big5'; +--disable_session_track_info +--echo # tracking info on once +--enable_session_track_info ONCE +SET NAMES 'utf8'; +SET NAMES 'big5'; +--echo # tracking info on +--enable_session_track_info +SET NAMES 'utf8'; +--echo # tracking info off once +--disable_session_track_info ONCE +SET NAMES 'big5'; +--disable_session_track_info + +SET @@session.session_track_system_variables= default; diff --git a/mysql-test/t/reset_connection.test b/mysql-test/t/reset_connection.test index 39465677483..49f41c32fc3 100644 --- a/mysql-test/t/reset_connection.test +++ b/mysql-test/t/reset_connection.test @@ -1,3 +1,5 @@ +--source include/not_embedded.inc + FLUSH STATUS; --disable_result_log From 517d7182013b212db4680a22da2c91314c41f51b Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Fri, 9 Mar 2018 14:39:40 +0100 Subject: [PATCH 185/203] MDEV-15477: SESSION_SYSVARS_TRACKER does not track last_gtid register changes of last_gtid --- mysql-test/r/mysqltest_tracking_info.result | 24 +++---------- mysql-test/r/session_tracker_last_gtid.result | 34 +++++++++++++++++++ mysql-test/t/mysqltest_tracking_info.test | 4 +-- mysql-test/t/session_tracker_last_gtid.test | 19 +++++++++++ sql/log.cc | 5 ++- sql/set_var.h | 2 +- sql/sql_class.cc | 17 +++++++++- sql/sql_class.h | 8 ++++- sql/sys_vars.cc | 7 ++-- 9 files changed, 92 insertions(+), 28 deletions(-) create mode 100644 mysql-test/r/session_tracker_last_gtid.result create mode 100644 mysql-test/t/session_tracker_last_gtid.test diff --git a/mysql-test/r/mysqltest_tracking_info.result b/mysql-test/r/mysqltest_tracking_info.result index b5885a37cde..df966ae1d39 100644 --- a/mysql-test/r/mysqltest_tracking_info.result +++ b/mysql-test/r/mysqltest_tracking_info.result @@ -1,46 +1,30 @@ -SELECT @@session.character_set_client, @@session.character_set_results, @@session.character_set_connection; -@@session.character_set_client @@session.character_set_results @@session.character_set_connection -latin1 latin1 latin1 -SET @@session.session_track_system_variables='character_set_client,character_set_results,character_set_connection'; +SELECT @@session.character_set_connection; +@@session.character_set_connection +latin1 +SET @@session.session_track_system_variables='character_set_connection'; # tracking info on SET NAMES 'utf8'; -- Tracker : SESSION_TRACK_SYSTEM_VARIABLES --- character_set_client --- utf8 -- character_set_connection -- utf8 --- character_set_results --- utf8 SET NAMES 'big5'; -- Tracker : SESSION_TRACK_SYSTEM_VARIABLES --- character_set_client --- big5 -- character_set_connection -- big5 --- character_set_results --- big5 # tracking info on once SET NAMES 'utf8'; -- Tracker : SESSION_TRACK_SYSTEM_VARIABLES --- character_set_client --- utf8 -- character_set_connection -- utf8 --- character_set_results --- utf8 SET NAMES 'big5'; # tracking info on SET NAMES 'utf8'; -- Tracker : SESSION_TRACK_SYSTEM_VARIABLES --- character_set_client --- utf8 -- character_set_connection -- utf8 --- character_set_results --- utf8 # tracking info off once SET NAMES 'big5'; diff --git a/mysql-test/r/session_tracker_last_gtid.result b/mysql-test/r/session_tracker_last_gtid.result new file mode 100644 index 00000000000..795d0aaa2a2 --- /dev/null +++ b/mysql-test/r/session_tracker_last_gtid.result @@ -0,0 +1,34 @@ +# +# MDEV-15477: SESSION_SYSVARS_TRACKER does not track last_gtid +# +SET gtid_seq_no=1000; +SET @@session.session_track_system_variables='last_gtid'; +create table t1 (a int) engine=innodb; +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- last_gtid +-- 0-1-1000 + +select @@last_gtid; +@@last_gtid +0-1-1000 +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- last_gtid +-- 0-1-1000 + +insert into t1 values (1); +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- last_gtid +-- 0-1-1001 + +select @@last_gtid; +@@last_gtid +0-1-1001 +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- last_gtid +-- 0-1-1001 + +drop table t1; +-- Tracker : SESSION_TRACK_SYSTEM_VARIABLES +-- last_gtid +-- 0-1-1002 + diff --git a/mysql-test/t/mysqltest_tracking_info.test b/mysql-test/t/mysqltest_tracking_info.test index c32e20703a5..d31f68b3cbd 100644 --- a/mysql-test/t/mysqltest_tracking_info.test +++ b/mysql-test/t/mysqltest_tracking_info.test @@ -2,8 +2,8 @@ --source include/no_protocol.inc --source include/not_embedded.inc -SELECT @@session.character_set_client, @@session.character_set_results, @@session.character_set_connection; -SET @@session.session_track_system_variables='character_set_client,character_set_results,character_set_connection'; +SELECT @@session.character_set_connection; +SET @@session.session_track_system_variables='character_set_connection'; --echo # tracking info on --enable_session_track_info diff --git a/mysql-test/t/session_tracker_last_gtid.test b/mysql-test/t/session_tracker_last_gtid.test new file mode 100644 index 00000000000..24cf2104a0c --- /dev/null +++ b/mysql-test/t/session_tracker_last_gtid.test @@ -0,0 +1,19 @@ +--source include/not_embedded.inc +--source include/have_innodb.inc +--source include/have_binlog_format_statement.inc + +--enable_session_track_info + +--echo # +--echo # MDEV-15477: SESSION_SYSVARS_TRACKER does not track last_gtid +--echo # + +SET gtid_seq_no=1000; +SET @@session.session_track_system_variables='last_gtid'; +create table t1 (a int) engine=innodb; +select @@last_gtid; +insert into t1 values (1); +select @@last_gtid; +drop table t1; + +--disable_session_track_info diff --git a/sql/log.cc b/sql/log.cc index 923810aa945..f74e73f47ba 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -46,6 +46,8 @@ #include #include // For test_if_number +#include // for Sys_last_gtid_ptr + #ifdef _WIN32 #include "message.h" #endif @@ -5945,7 +5947,8 @@ MYSQL_BIN_LOG::write_gtid_event(THD *thd, bool standalone, } if (err) DBUG_RETURN(true); - thd->last_commit_gtid= gtid; + + thd->set_last_commit_gtid(gtid); Gtid_log_event gtid_event(thd, seq_no, domain_id, standalone, LOG_EVENT_SUPPRESS_USE_F, is_transactional, diff --git a/sql/set_var.h b/sql/set_var.h index 5a7f8c2d8a3..c218c40fb38 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -430,7 +430,7 @@ sql_mode_t expand_sql_mode(sql_mode_t sql_mode); bool sql_mode_string_representation(THD *thd, sql_mode_t sql_mode, LEX_STRING *ls); int default_regex_flags_pcre(const THD *thd); -extern sys_var *Sys_autocommit_ptr; +extern sys_var *Sys_autocommit_ptr, *Sys_last_gtid_ptr; CHARSET_INFO *get_old_charset_by_name(const char *old_name); diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 8590a0fe179..a1010fb4c35 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1190,7 +1190,7 @@ void THD::init(void) bzero((char *) &org_status_var, sizeof(org_status_var)); status_in_global= 0; start_bytes_received= 0; - last_commit_gtid.seq_no= 0; + m_last_commit_gtid.seq_no= 0; last_stmt= NULL; /* Reset status of last insert id */ arg_of_last_insert_id_function= FALSE; @@ -6972,6 +6972,21 @@ THD::signal_wakeup_ready() mysql_cond_signal(&COND_wakeup_ready); } +void THD::set_last_commit_gtid(rpl_gtid >id) +{ +#ifndef EMBEDDED_LIBRARY + bool changed_gtid= (m_last_commit_gtid.seq_no != gtid.seq_no); +#endif + m_last_commit_gtid= gtid; +#ifndef EMBEDDED_LIBRARY + if (changed_gtid && + session_tracker.get_tracker(SESSION_SYSVARS_TRACKER)->is_enabled()) + { + session_tracker.get_tracker(SESSION_SYSVARS_TRACKER)-> + mark_as_changed(this, (LEX_CSTRING*)Sys_last_gtid_ptr); + } +#endif +} void wait_for_commit::reinit() diff --git a/sql/sql_class.h b/sql/sql_class.h index 72cbd03cd1b..d03852ab9a2 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -4255,7 +4255,13 @@ public: The GTID assigned to the last commit. If no GTID was assigned to any commit so far, this is indicated by last_commit_gtid.seq_no == 0. */ - rpl_gtid last_commit_gtid; +private: + rpl_gtid m_last_commit_gtid; + +public: + rpl_gtid get_last_commit_gtid() { return m_last_commit_gtid; } + void set_last_commit_gtid(rpl_gtid >id); + LF_PINS *tdc_hash_pins; LF_PINS *xid_hash_pins; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index de546175da8..5c893f64a9d 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1828,6 +1828,8 @@ static Sys_var_last_gtid Sys_last_gtid( "or the empty string if none.", READ_ONLY sys_var::ONLY_SESSION, NO_CMD_LINE); +export sys_var *Sys_last_gtid_ptr= &Sys_last_gtid; // for check changing + uchar * Sys_var_last_gtid::session_value_ptr(THD *thd, const LEX_STRING *base) @@ -1838,8 +1840,9 @@ Sys_var_last_gtid::session_value_ptr(THD *thd, const LEX_STRING *base) bool first= true; str.length(0); - if ((thd->last_commit_gtid.seq_no > 0 && - rpl_slave_state_tostring_helper(&str, &thd->last_commit_gtid, &first)) || + rpl_gtid gtid= thd->get_last_commit_gtid(); + if ((gtid.seq_no > 0 && + rpl_slave_state_tostring_helper(&str, >id, &first)) || !(p= thd->strmake(str.ptr(), str.length()))) { my_error(ER_OUT_OF_RESOURCES, MYF(0)); From 31e52b163206c9dc9f4c3542d07e0b64b33dade5 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Mon, 12 Mar 2018 14:46:00 +0100 Subject: [PATCH 186/203] Optimize charset tracking a bit. --- sql/set_var.cc | 24 ++++++------------------ sql/set_var.h | 4 +++- sql/sys_vars.cc | 6 ++++++ 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/sql/set_var.cc b/sql/set_var.cc index 77036fd0f5a..a17e7b0aa58 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -1016,24 +1016,12 @@ int set_var_collation_client::update(THD *thd) #ifndef EMBEDDED_LIBRARY if (thd->session_tracker.get_tracker(SESSION_SYSVARS_TRACKER)->is_enabled()) { - sys_var *svar; - mysql_mutex_lock(&LOCK_plugin); - if ((svar= find_sys_var_ex(thd, "character_set_client", - sizeof("character_set_client") - 1, - false, true))) - thd->session_tracker.get_tracker(SESSION_SYSVARS_TRACKER)-> - mark_as_changed(thd, (LEX_CSTRING*)svar); - if ((svar= find_sys_var_ex(thd, "character_set_results", - sizeof("character_set_results") - 1, - false, true))) - thd->session_tracker.get_tracker(SESSION_SYSVARS_TRACKER)-> - mark_as_changed(thd, (LEX_CSTRING*)svar); - if ((svar= find_sys_var_ex(thd, "character_set_connection", - sizeof("character_set_connection") - 1, - false, true))) - thd->session_tracker.get_tracker(SESSION_SYSVARS_TRACKER)-> - mark_as_changed(thd, (LEX_CSTRING*)svar); - mysql_mutex_unlock(&LOCK_plugin); + thd->session_tracker.get_tracker(SESSION_SYSVARS_TRACKER)-> + mark_as_changed(thd, (LEX_CSTRING*)Sys_character_set_client_ptr); + thd->session_tracker.get_tracker(SESSION_SYSVARS_TRACKER)-> + mark_as_changed(thd, (LEX_CSTRING*)Sys_character_set_results_ptr); + thd->session_tracker.get_tracker(SESSION_SYSVARS_TRACKER)-> + mark_as_changed(thd, (LEX_CSTRING*)Sys_character_set_connection_ptr); } thd->session_tracker.mark_as_changed(thd, SESSION_STATE_CHANGE_TRACKER, NULL); #endif //EMBEDDED_LIBRARY diff --git a/sql/set_var.h b/sql/set_var.h index c218c40fb38..d92b244cd1a 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -430,7 +430,9 @@ sql_mode_t expand_sql_mode(sql_mode_t sql_mode); bool sql_mode_string_representation(THD *thd, sql_mode_t sql_mode, LEX_STRING *ls); int default_regex_flags_pcre(const THD *thd); -extern sys_var *Sys_autocommit_ptr, *Sys_last_gtid_ptr; +extern sys_var *Sys_autocommit_ptr, *Sys_last_gtid_ptr, + *Sys_character_set_client_ptr, *Sys_character_set_connection_ptr, + *Sys_character_set_results_ptr; CHARSET_INFO *get_old_charset_by_name(const char *old_name); diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 5c893f64a9d..e6aec472b9e 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -673,6 +673,8 @@ static Sys_var_struct Sys_character_set_client( offsetof(CHARSET_INFO, csname), DEFAULT(&default_charset_info), NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(check_cs_client), ON_UPDATE(fix_thd_charset)); +// for check changing +export sys_var *Sys_character_set_client_ptr= &Sys_character_set_client; static Sys_var_struct Sys_character_set_connection( "character_set_connection", "The character set used for " @@ -682,6 +684,8 @@ static Sys_var_struct Sys_character_set_connection( offsetof(CHARSET_INFO, csname), DEFAULT(&default_charset_info), NO_MUTEX_GUARD, IN_BINLOG, ON_CHECK(check_charset_not_null), ON_UPDATE(fix_thd_charset)); +// for check changing +export sys_var *Sys_character_set_connection_ptr= &Sys_character_set_connection; static Sys_var_struct Sys_character_set_results( "character_set_results", "The character set used for returning " @@ -689,6 +693,8 @@ static Sys_var_struct Sys_character_set_results( SESSION_VAR(character_set_results), NO_CMD_LINE, offsetof(CHARSET_INFO, csname), DEFAULT(&default_charset_info), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_charset)); +// for check changing +export sys_var *Sys_character_set_results_ptr= &Sys_character_set_results; static Sys_var_struct Sys_character_set_filesystem( "character_set_filesystem", "The filesystem character set", From 7d0d934ca642e485b2c008727dc20c83e26cce10 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Mon, 25 Jun 2018 19:19:10 -0700 Subject: [PATCH 187/203] MDEV-16473 WITH statement throws 'no database selected' error Before this patch if no default database was set the server threw an error for any table name reference that was not fully qualified by database name. In particular it happened for table names referenced CTE tables. This was incorrect. The error message was thrown at the parser stage when the names referencing different tables were not resolved yet. Now if no default database is set and a with clause is used in the processed statement any table reference is just supplied with a dummy database name "*none*" at the parser stage. Later after a call of check_dependencies_in_with_clauses() when the names for CTE tables can be resolved error messages are thrown only for those names that refer to non-CTE tables. This is done in open_and_process_table(). --- mysql-test/r/cte_nonrecursive.result | 34 ++++++++++++++++++++++++++++ mysql-test/t/cte_nonrecursive.test | 29 +++++++++++++++++++++++- sql/sql_base.cc | 8 +++++++ sql/sql_class.h | 25 ++++++++++++++++---- sql/sql_lex.h | 2 ++ sql/sql_parse.cc | 1 + 6 files changed, 94 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/cte_nonrecursive.result b/mysql-test/r/cte_nonrecursive.result index 1d079c3bfa6..f6b80156ee0 100644 --- a/mysql-test/r/cte_nonrecursive.result +++ b/mysql-test/r/cte_nonrecursive.result @@ -1478,3 +1478,37 @@ select 2 as f; f 2 drop table t1; +# +# MDEV-16473: query with CTE when no database is set +# +create database db_mdev_16473; +use db_mdev_16473; +drop database db_mdev_16473; +# Now no default database is set +select database(); +database() +NULL +with cte as (select 1 as a) select * from cte; +a +1 +create database db_mdev_16473; +create table db_mdev_16473.t1 (a int); +insert into db_mdev_16473.t1 values (2), (7), (3), (1); +with cte as (select * from db_mdev_16473.t1) select * from cte; +a +2 +7 +3 +1 +with cte as (select * from db_mdev_16473.t1) +select * from cte, t1 as t where cte.a=t.a; +ERROR 3D000: No database selected +with cte as (select * from db_mdev_16473.t1) +select * from cte, db_mdev_16473.t1 as t where cte.a=t.a; +a a +2 2 +7 7 +3 3 +1 1 +drop database db_mdev_16473; +use test; diff --git a/mysql-test/t/cte_nonrecursive.test b/mysql-test/t/cte_nonrecursive.test index 98a77940c99..11c864bcac1 100644 --- a/mysql-test/t/cte_nonrecursive.test +++ b/mysql-test/t/cte_nonrecursive.test @@ -1029,4 +1029,31 @@ with cte as select 2 as f; drop table t1; - \ No newline at end of file + +--echo # +--echo # MDEV-16473: query with CTE when no database is set +--echo # + +create database db_mdev_16473; +use db_mdev_16473; +drop database db_mdev_16473; + +--echo # Now no default database is set +select database(); + +with cte as (select 1 as a) select * from cte; + +create database db_mdev_16473; +create table db_mdev_16473.t1 (a int); +insert into db_mdev_16473.t1 values (2), (7), (3), (1); +with cte as (select * from db_mdev_16473.t1) select * from cte; + +--error ER_NO_DB_ERROR +with cte as (select * from db_mdev_16473.t1) +select * from cte, t1 as t where cte.a=t.a; +with cte as (select * from db_mdev_16473.t1) +select * from cte, db_mdev_16473.t1 as t where cte.a=t.a; + +drop database db_mdev_16473; + +use test; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 83ab24fc27d..062106d2fae 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -3330,6 +3330,14 @@ open_and_process_table(THD *thd, LEX *lex, TABLE_LIST *tables, /* Not a placeholder: must be a base/temporary table or a view. Let us open it. */ + + if (tables->db == no_db) + { + my_message(ER_NO_DB_ERROR, ER(ER_NO_DB_ERROR), MYF(0)); + error= TRUE; + goto end; + } + if (tables->table) { /* diff --git a/sql/sql_class.h b/sql/sql_class.h index d03852ab9a2..ad2d0abf746 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3946,11 +3946,28 @@ public: { if (db == NULL) { - my_message(ER_NO_DB_ERROR, ER(ER_NO_DB_ERROR), MYF(0)); - return TRUE; + /* + No default database is set. In this case if it's guaranteed that + no CTE can be used in the statement then we can throw an error right + now at the parser stage. Otherwise the decision about throwing such + a message must be postponed until a post-parser stage when we are able + to resolve all CTE names as we don't need this message to be thrown + for any CTE references. + */ + if (!lex->with_clauses_list) + { + my_message(ER_NO_DB_ERROR, ER(ER_NO_DB_ERROR), MYF(0)); + return TRUE; + } + /* This will allow to throw an error later for non-CTE references */ + *p_db= (char *) no_db; + *p_db_length= strlen(no_db); + } + else + { + *p_db= strmake(db, db_length); + *p_db_length= db_length; } - *p_db= strmake(db, db_length); - *p_db_length= db_length; return FALSE; } thd_scheduler event_scheduler; diff --git a/sql/sql_lex.h b/sql/sql_lex.h index a1f6b202ae6..ae010a88e46 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -156,6 +156,8 @@ struct LEX_TYPE extern const LEX_STRING null_lex_str; extern const LEX_STRING empty_lex_str; +extern const char *no_db; + enum enum_sp_suid_behaviour { SP_IS_DEFAULT_SUID= 0, diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index db7c3b5a7dc..3631c8edd13 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -139,6 +139,7 @@ static bool execute_show_status(THD *, TABLE_LIST *); static bool check_rename_table(THD *, TABLE_LIST *, TABLE_LIST *); const char *any_db="*any*"; // Special symbol for check_access +const char *no_db="*none*"; // Used when no default db is set const LEX_STRING command_name[257]={ { C_STRING_WITH_LEN("Sleep") }, //0 From ff8b3c8df89e973f3b91fa82a5fec65ef0e01c53 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sat, 23 Jun 2018 13:49:36 +0300 Subject: [PATCH 188/203] MDEV-15953 Alter InnoDB Partitioned Table Moves Files (which were originally not in the datadir) to the datadir ha_innobase::prepare_inplace_alter_table: preserve DATA DICTIONARY for table --- .../r/alter_data_directory_innodb.result | 65 +++++++++++++++++++ .../parts/t/alter_data_directory_innodb.test | 46 +++++++++++++ storage/innobase/handler/handler0alter.cc | 6 ++ storage/xtradb/handler/handler0alter.cc | 6 ++ 4 files changed, 123 insertions(+) create mode 100644 mysql-test/suite/parts/r/alter_data_directory_innodb.result create mode 100644 mysql-test/suite/parts/t/alter_data_directory_innodb.test diff --git a/mysql-test/suite/parts/r/alter_data_directory_innodb.result b/mysql-test/suite/parts/r/alter_data_directory_innodb.result new file mode 100644 index 00000000000..ee0a7b80ebb --- /dev/null +++ b/mysql-test/suite/parts/r/alter_data_directory_innodb.result @@ -0,0 +1,65 @@ +# +# MDEV-15953 Alter InnoDB Partitioned Table Moves Files (which were originally not in the datadir) to the datadir +# +CREATE TABLE t ( +a INT NOT NULL +) ENGINE=INNODB +PARTITION BY HASH (a) ( +PARTITION p1 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here/' ENGINE = INNODB, +PARTITION p2 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here/' ENGINE = INNODB +); +INSERT INTO t VALUES (1); +SHOW CREATE TABLE t; +Table Create Table +t CREATE TABLE `t` ( + `a` int(11) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY HASH (a) +(PARTITION p1 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB, + PARTITION p2 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB) */ +ALTER TABLE t ADD PRIMARY KEY pk(a), ALGORITHM=INPLACE; +SHOW CREATE TABLE t; +Table Create Table +t CREATE TABLE `t` ( + `a` int(11) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY HASH (a) +(PARTITION p1 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB, + PARTITION p2 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB) */ +ALTER TABLE t DROP PRIMARY KEY, ALGORITHM=COPY; +SHOW CREATE TABLE t; +Table Create Table +t CREATE TABLE `t` ( + `a` int(11) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY HASH (a) +(PARTITION p1 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB, + PARTITION p2 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB) */ +SET @TMP = @@GLOBAL.INNODB_FILE_PER_TABLE; +SET GLOBAL INNODB_FILE_PER_TABLE=OFF; +ALTER TABLE t ADD PRIMARY KEY pk(a), ALGORITHM=INPLACE; +SHOW CREATE TABLE t; +Table Create Table +t CREATE TABLE `t` ( + `a` int(11) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY HASH (a) +(PARTITION p1 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB, + PARTITION p2 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB) */ +SET GLOBAL INNODB_FILE_PER_TABLE=@TMP; +ALTER TABLE t REORGANIZE PARTITION p1,p2 INTO ( +PARTITION p1 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_somewhere_else/' ENGINE = INNODB, +PARTITION p2 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_somewhere_else/' ENGINE = INNODB +); +SHOW CREATE TABLE t; +Table Create Table +t CREATE TABLE `t` ( + `a` int(11) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY HASH (a) +(PARTITION p1 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB, + PARTITION p2 DATA DIRECTORY = 'MYSQLTEST_VARDIR/tmp/partitions_here' ENGINE = InnoDB) */ +DROP TABLE t; diff --git a/mysql-test/suite/parts/t/alter_data_directory_innodb.test b/mysql-test/suite/parts/t/alter_data_directory_innodb.test new file mode 100644 index 00000000000..ac15e9bec6c --- /dev/null +++ b/mysql-test/suite/parts/t/alter_data_directory_innodb.test @@ -0,0 +1,46 @@ +--source include/have_innodb.inc +--source include/have_partition.inc + +--echo # +--echo # MDEV-15953 Alter InnoDB Partitioned Table Moves Files (which were originally not in the datadir) to the datadir +--echo # + +mkdir $MYSQLTEST_VARDIR/tmp/partitions_here; + +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +eval CREATE TABLE t ( + a INT NOT NULL +) ENGINE=INNODB +PARTITION BY HASH (a) ( + PARTITION p1 DATA DIRECTORY = '$MYSQLTEST_VARDIR/tmp/partitions_here/' ENGINE = INNODB, + PARTITION p2 DATA DIRECTORY = '$MYSQLTEST_VARDIR/tmp/partitions_here/' ENGINE = INNODB +); +INSERT INTO t VALUES (1); + +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +SHOW CREATE TABLE t; +ALTER TABLE t ADD PRIMARY KEY pk(a), ALGORITHM=INPLACE; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +SHOW CREATE TABLE t; +ALTER TABLE t DROP PRIMARY KEY, ALGORITHM=COPY; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +SHOW CREATE TABLE t; +SET @TMP = @@GLOBAL.INNODB_FILE_PER_TABLE; +SET GLOBAL INNODB_FILE_PER_TABLE=OFF; +ALTER TABLE t ADD PRIMARY KEY pk(a), ALGORITHM=INPLACE; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +SHOW CREATE TABLE t; +SET GLOBAL INNODB_FILE_PER_TABLE=@TMP; + +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +eval ALTER TABLE t REORGANIZE PARTITION p1,p2 INTO ( + PARTITION p1 DATA DIRECTORY = '$MYSQLTEST_VARDIR/tmp/partitions_somewhere_else/' ENGINE = INNODB, + PARTITION p2 DATA DIRECTORY = '$MYSQLTEST_VARDIR/tmp/partitions_somewhere_else/' ENGINE = INNODB +); +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +SHOW CREATE TABLE t; + +DROP TABLE t; + +rmdir $MYSQLTEST_VARDIR/tmp/partitions_here/test; +rmdir $MYSQLTEST_VARDIR/tmp/partitions_here; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 86935a4aa34..3de46ed4f80 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -3572,6 +3572,12 @@ check_if_ok_to_rename: goto err_exit_no_heap; } + /* Preserve this flag, because it currenlty can't be changed during + ALTER TABLE*/ + if (flags2 & DICT_TF2_USE_TABLESPACE) { + flags |= prebuilt->table->flags & 1U << DICT_TF_POS_DATA_DIR; + } + max_col_len = DICT_MAX_FIELD_LEN_BY_FORMAT_FLAG(flags); /* Check each index's column length to make sure they do not diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index ff524e6a9be..755c07848e1 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -3586,6 +3586,12 @@ check_if_ok_to_rename: goto err_exit_no_heap; } + /* Preserve this flag, because it currenlty can't be changed during + ALTER TABLE*/ + if (flags2 & DICT_TF2_USE_TABLESPACE) { + flags |= prebuilt->table->flags & 1U << DICT_TF_POS_DATA_DIR; + } + max_col_len = DICT_MAX_FIELD_LEN_BY_FORMAT_FLAG(flags); /* Check each index's column length to make sure they do not From c4eb4bcef648eb2ebdc6edc06905f39f95ef7f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 26 Jun 2018 11:34:51 +0300 Subject: [PATCH 189/203] MDEV-16515 InnoDB: Failing assertion: ++retries < 10000 in file dict0dict.cc buf_LRU_drop_page_hash_for_tablespace(): Return whether any adaptive hash index entries existed. If yes, the caller should keep retrying to drop the adaptive hash index. row_import_for_mysql(), row_truncate_table_for_mysql(), row_drop_table_for_mysql(): Ensure that the adaptive hash index was entirely dropped for the table. --- storage/innobase/buf/buf0lru.cc | 11 +++++--- storage/innobase/dict/dict0dict.cc | 5 ++-- storage/innobase/include/buf0lru.h | 8 ++++-- storage/innobase/row/row0import.cc | 28 +++++++++++-------- storage/innobase/row/row0mysql.cc | 44 +++++++++++++++++++----------- storage/xtradb/buf/buf0lru.cc | 11 +++++--- storage/xtradb/dict/dict0dict.cc | 4 +-- storage/xtradb/include/buf0lru.h | 8 ++++-- storage/xtradb/row/row0import.cc | 28 +++++++++++-------- storage/xtradb/row/row0mysql.cc | 44 +++++++++++++++++++----------- 10 files changed, 118 insertions(+), 73 deletions(-) diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 7039ecdf4a6..1f2b6f40529 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -356,9 +356,10 @@ next_page: ut_free(page_arr); } -/** Drop the adaptive hash index for a tablespace. -@param[in,out] table table */ -UNIV_INTERN void buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table) +/** Try to drop the adaptive hash index for a tablespace. +@param[in,out] table table +@return whether anything was dropped */ +UNIV_INTERN bool buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table) { for (dict_index_t* index = dict_table_get_first_index(table); index != NULL; @@ -369,13 +370,15 @@ UNIV_INTERN void buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table) } } - return; + return false; drop_ahi: ulint id = table->space; for (ulint i = 0; i < srv_buf_pool_instances; i++) { buf_LRU_drop_page_hash_for_tablespace(buf_pool_from_array(i), id); } + + return true; } /******************************************************************//** diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 9609ef96343..b48d51c15a7 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -2719,12 +2719,11 @@ dict_index_remove_from_cache_low( zero. See also: dict_table_can_be_evicted() */ do { - if (!btr_search_info_get_ref_count(info)) { + if (!btr_search_info_get_ref_count(info) + || !buf_LRU_drop_page_hash_for_tablespace(table)) { break; } - buf_LRU_drop_page_hash_for_tablespace(table); - ut_a(++retries < 10000); } while (srv_shutdown_state == SRV_SHUTDOWN_NONE || !lru_evict); diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h index 623883433c2..b04086287ce 100644 --- a/storage/innobase/include/buf0lru.h +++ b/storage/innobase/include/buf0lru.h @@ -53,9 +53,11 @@ These are low-level functions /** Minimum LRU list length for which the LRU_old pointer is defined */ #define BUF_LRU_OLD_MIN_LEN 512 /* 8 megabytes of 16k pages */ -/** Drop the adaptive hash index for a tablespace. -@param[in,out] table table */ -UNIV_INTERN void buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table); +/** Try to drop the adaptive hash index for a tablespace. +@param[in,out] table table +@return whether anything was dropped */ +UNIV_INTERN bool buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table) + MY_ATTRIBUTE((warn_unused_result,nonnull)); /** Empty the flush list for all pages belonging to a tablespace. @param[in] id tablespace identifier diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index c808a991f25..68acf656034 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -3983,6 +3983,23 @@ row_import_for_mysql( DBUG_EXECUTE_IF("ib_import_reset_space_and_lsn_failure", err = DB_TOO_MANY_CONCURRENT_TRXS;); + /* On DISCARD TABLESPACE, we did not drop any adaptive hash + index entries. If we replaced the discarded tablespace with a + smaller one here, there could still be some adaptive hash + index entries that point to cached garbage pages in the buffer + pool, because PageConverter::operator() only evicted those + pages that were replaced by the imported pages. We must + discard all remaining adaptive hash index entries, because the + adaptive hash index must be a subset of the table contents; + false positives are not tolerated. */ + while (buf_LRU_drop_page_hash_for_tablespace(table)) { + if (trx_is_interrupted(trx) + || srv_shutdown_state != SRV_SHUTDOWN_NONE) { + err = DB_INTERRUPTED; + break; + } + } + if (err != DB_SUCCESS) { char table_name[MAX_FULL_NAME_LEN + 1]; @@ -4000,17 +4017,6 @@ row_import_for_mysql( return(row_import_cleanup(prebuilt, trx, err)); } - /* On DISCARD TABLESPACE, we did not drop any adaptive hash - index entries. If we replaced the discarded tablespace with a - smaller one here, there could still be some adaptive hash - index entries that point to cached garbage pages in the buffer - pool, because PageConverter::operator() only evicted those - pages that were replaced by the imported pages. We must - discard all remaining adaptive hash index entries, because the - adaptive hash index must be a subset of the table contents; - false positives are not tolerated. */ - buf_LRU_drop_page_hash_for_tablespace(table); - row_mysql_lock_data_dictionary(trx); /* If the table is stored in a remote tablespace, we need to diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index be24ae885a2..6a971a973f5 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -3516,7 +3516,13 @@ row_truncate_table_for_mysql( fil_space_release(space); } - buf_LRU_drop_page_hash_for_tablespace(table); + while (buf_LRU_drop_page_hash_for_tablespace(table)) { + if (trx_is_interrupted(trx) + || srv_shutdown_state != SRV_SHUTDOWN_NONE) { + err = DB_INTERRUPTED; + goto funct_exit; + } + } if (flags != ULINT_UNDEFINED && fil_discard_tablespace(space_id) == DB_SUCCESS) { @@ -4172,6 +4178,27 @@ row_drop_table_for_mysql( ut_a(!lock_table_has_locks(table)); + if (table->space != TRX_SYS_SPACE) { + /* On DISCARD TABLESPACE, we would not drop the + adaptive hash index entries. If the tablespace is + missing here, delete-marking the record in SYS_INDEXES + would not free any pages in the buffer pool. Thus, + dict_index_remove_from_cache() would hang due to + adaptive hash index entries existing in the buffer + pool. To prevent this hang, and also to guarantee + that btr_search_drop_page_hash_when_freed() will avoid + calling btr_search_drop_page_hash_index() while we + hold the InnoDB dictionary lock, we will drop any + adaptive hash index entries upfront. */ + while (buf_LRU_drop_page_hash_for_tablespace(table)) { + if (trx_is_interrupted(trx) + || srv_shutdown_state != SRV_SHUTDOWN_NONE) { + err = DB_INTERRUPTED; + goto funct_exit; + } + } + } + switch (trx_get_dict_operation(trx)) { case TRX_DICT_OP_NONE: trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); @@ -4211,21 +4238,6 @@ row_drop_table_for_mysql( rw_lock_x_unlock(dict_index_get_lock(index)); } - if (table->space != TRX_SYS_SPACE) { - /* On DISCARD TABLESPACE, we would not drop the - adaptive hash index entries. If the tablespace is - missing here, delete-marking the record in SYS_INDEXES - would not free any pages in the buffer pool. Thus, - dict_index_remove_from_cache() would hang due to - adaptive hash index entries existing in the buffer - pool. To prevent this hang, and also to guarantee - that btr_search_drop_page_hash_when_freed() will avoid - calling btr_search_drop_page_hash_index() while we - hold the InnoDB dictionary lock, we will drop any - adaptive hash index entries upfront. */ - buf_LRU_drop_page_hash_for_tablespace(table); - } - /* We use the private SQL parser of Innobase to generate the query graphs needed in deleting the dictionary data from system tables in Innobase. Deleting a row from SYS_INDEXES table also diff --git a/storage/xtradb/buf/buf0lru.cc b/storage/xtradb/buf/buf0lru.cc index 2c4a4049de6..ec65bfbcce4 100644 --- a/storage/xtradb/buf/buf0lru.cc +++ b/storage/xtradb/buf/buf0lru.cc @@ -354,9 +354,10 @@ next_page: ut_free(page_arr); } -/** Drop the adaptive hash index for a tablespace. -@param[in,out] table table */ -UNIV_INTERN void buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table) +/** Try to drop the adaptive hash index for a tablespace. +@param[in,out] table table +@return whether anything was dropped */ +UNIV_INTERN bool buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table) { for (dict_index_t* index = dict_table_get_first_index(table); index != NULL; @@ -367,13 +368,15 @@ UNIV_INTERN void buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table) } } - return; + return false; drop_ahi: ulint id = table->space; for (ulint i = 0; i < srv_buf_pool_instances; i++) { buf_LRU_drop_page_hash_for_tablespace(buf_pool_from_array(i), id); } + + return true; } /******************************************************************//** diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 7ade6d79adf..23082e53ec7 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -2729,11 +2729,11 @@ dict_index_remove_from_cache_low( zero. See also: dict_table_can_be_evicted() */ do { - if (!btr_search_info_get_ref_count(info, index)) { + if (!btr_search_info_get_ref_count(info, index) + || !buf_LRU_drop_page_hash_for_tablespace(table)) { break; } - buf_LRU_drop_page_hash_for_tablespace(table); ut_a(++retries < 10000); } while (srv_shutdown_state == SRV_SHUTDOWN_NONE || !lru_evict); diff --git a/storage/xtradb/include/buf0lru.h b/storage/xtradb/include/buf0lru.h index f0ba1bb227d..ee84d168e28 100644 --- a/storage/xtradb/include/buf0lru.h +++ b/storage/xtradb/include/buf0lru.h @@ -55,9 +55,11 @@ These are low-level functions /** Minimum LRU list length for which the LRU_old pointer is defined */ #define BUF_LRU_OLD_MIN_LEN 512 /* 8 megabytes of 16k pages */ -/** Drop the adaptive hash index for a tablespace. -@param[in,out] table table */ -UNIV_INTERN void buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table); +/** Try to drop the adaptive hash index for a tablespace. +@param[in,out] table table +@return whether anything was dropped */ +UNIV_INTERN bool buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table) + MY_ATTRIBUTE((warn_unused_result,nonnull)); /** Empty the flush list for all pages belonging to a tablespace. @param[in] id tablespace identifier diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index 4af40953567..731412c22ec 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -3982,6 +3982,23 @@ row_import_for_mysql( DBUG_EXECUTE_IF("ib_import_reset_space_and_lsn_failure", err = DB_TOO_MANY_CONCURRENT_TRXS;); + /* On DISCARD TABLESPACE, we did not drop any adaptive hash + index entries. If we replaced the discarded tablespace with a + smaller one here, there could still be some adaptive hash + index entries that point to cached garbage pages in the buffer + pool, because PageConverter::operator() only evicted those + pages that were replaced by the imported pages. We must + discard all remaining adaptive hash index entries, because the + adaptive hash index must be a subset of the table contents; + false positives are not tolerated. */ + while (buf_LRU_drop_page_hash_for_tablespace(table)) { + if (trx_is_interrupted(trx) + || srv_shutdown_state != SRV_SHUTDOWN_NONE) { + err = DB_INTERRUPTED; + break; + } + } + if (err != DB_SUCCESS) { char table_name[MAX_FULL_NAME_LEN + 1]; @@ -3999,17 +4016,6 @@ row_import_for_mysql( return(row_import_cleanup(prebuilt, trx, err)); } - /* On DISCARD TABLESPACE, we did not drop any adaptive hash - index entries. If we replaced the discarded tablespace with a - smaller one here, there could still be some adaptive hash - index entries that point to cached garbage pages in the buffer - pool, because PageConverter::operator() only evicted those - pages that were replaced by the imported pages. We must - discard all remaining adaptive hash index entries, because the - adaptive hash index must be a subset of the table contents; - false positives are not tolerated. */ - buf_LRU_drop_page_hash_for_tablespace(table); - row_mysql_lock_data_dictionary(trx); /* If the table is stored in a remote tablespace, we need to diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 2b6f38ba2af..1347cbdedf8 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -3540,7 +3540,13 @@ row_truncate_table_for_mysql( fil_space_release(space); } - buf_LRU_drop_page_hash_for_tablespace(table); + while (buf_LRU_drop_page_hash_for_tablespace(table)) { + if (trx_is_interrupted(trx) + || srv_shutdown_state != SRV_SHUTDOWN_NONE) { + err = DB_INTERRUPTED; + goto funct_exit; + } + } if (flags != ULINT_UNDEFINED && fil_discard_tablespace(space_id) == DB_SUCCESS) { @@ -4202,6 +4208,27 @@ row_drop_table_for_mysql( ut_a(!lock_table_has_locks(table)); + if (table->space != TRX_SYS_SPACE) { + /* On DISCARD TABLESPACE, we would not drop the + adaptive hash index entries. If the tablespace is + missing here, delete-marking the record in SYS_INDEXES + would not free any pages in the buffer pool. Thus, + dict_index_remove_from_cache() would hang due to + adaptive hash index entries existing in the buffer + pool. To prevent this hang, and also to guarantee + that btr_search_drop_page_hash_when_freed() will avoid + calling btr_search_drop_page_hash_index() while we + hold the InnoDB dictionary lock, we will drop any + adaptive hash index entries upfront. */ + while (buf_LRU_drop_page_hash_for_tablespace(table)) { + if (trx_is_interrupted(trx) + || srv_shutdown_state != SRV_SHUTDOWN_NONE) { + err = DB_INTERRUPTED; + goto funct_exit; + } + } + } + switch (trx_get_dict_operation(trx)) { case TRX_DICT_OP_NONE: trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); @@ -4241,21 +4268,6 @@ row_drop_table_for_mysql( rw_lock_x_unlock(dict_index_get_lock(index)); } - if (table->space != TRX_SYS_SPACE) { - /* On DISCARD TABLESPACE, we would not drop the - adaptive hash index entries. If the tablespace is - missing here, delete-marking the record in SYS_INDEXES - would not free any pages in the buffer pool. Thus, - dict_index_remove_from_cache() would hang due to - adaptive hash index entries existing in the buffer - pool. To prevent this hang, and also to guarantee - that btr_search_drop_page_hash_when_freed() will avoid - calling btr_search_drop_page_hash_index() while we - hold the InnoDB dictionary lock, we will drop any - adaptive hash index entries upfront. */ - buf_LRU_drop_page_hash_for_tablespace(table); - } - /* We use the private SQL parser of Innobase to generate the query graphs needed in deleting the dictionary data from system tables in Innobase. Deleting a row from SYS_INDEXES table also From 0e937f30f6cdadd2bc1607efa4a07f19c88e1b68 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 26 Jun 2018 11:04:57 -0400 Subject: [PATCH 190/203] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 390743d6cd3..e066e232278 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=2 -MYSQL_VERSION_PATCH=16 +MYSQL_VERSION_PATCH=17 From 6d377a523c78b1e442d69ae6be548bccddd54416 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Tue, 26 Jun 2018 10:49:23 -0700 Subject: [PATCH 191/203] Correction for the patch to fix mdev-16473. --- sql/sql_base.cc | 2 +- sql/sql_class.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 062106d2fae..093f7cf3427 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -3331,7 +3331,7 @@ open_and_process_table(THD *thd, LEX *lex, TABLE_LIST *tables, Not a placeholder: must be a base/temporary table or a view. Let us open it. */ - if (tables->db == no_db) + if (tables->db[0] == no_db[0]) { my_message(ER_NO_DB_ERROR, ER(ER_NO_DB_ERROR), MYF(0)); error= TRUE; diff --git a/sql/sql_class.h b/sql/sql_class.h index ad2d0abf746..e79fde055c2 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3960,8 +3960,8 @@ public: return TRUE; } /* This will allow to throw an error later for non-CTE references */ - *p_db= (char *) no_db; *p_db_length= strlen(no_db); + *p_db= strmake(no_db, *p_db_length); } else { From be5698265a4195586142d1a34fdd1cce9d95d8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 27 Jun 2018 12:37:21 +0300 Subject: [PATCH 192/203] MDEV-15607: mysqld crashed few after node is being joined with sst This is a typical systemd response where it tries to shutdown the joiner (due to "timeout") before the joiner manages to complete SST. wsrep_sst_wait wsrep_SE_init_wait While waiting the operation to finish use mysql_cond_timedwait instead of mysql_cond_wait and if operation is not finished extend systemd timeout (if needed). --- sql/wsrep_sst.cc | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index 4df969496bc..60683bf740c 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -30,6 +30,10 @@ #include #include +#if MYSQL_VERSION_ID < 100200 +# include +#endif + static char wsrep_defaults_file[FN_REFLEN * 2 + 10 + 30 + sizeof(WSREP_SST_OPT_CONF) + sizeof(WSREP_SST_OPT_CONF_SUFFIX) + @@ -186,6 +190,9 @@ bool wsrep_before_SE() static bool sst_complete = false; static bool sst_needed = false; +#define WSREP_EXTEND_TIMEOUT_INTERVAL 30 +#define WSREP_TIMEDWAIT_SECONDS 10 + void wsrep_sst_grab () { WSREP_INFO("wsrep_sst_grab()"); @@ -197,11 +204,25 @@ void wsrep_sst_grab () // Wait for end of SST bool wsrep_sst_wait () { - if (mysql_mutex_lock (&LOCK_wsrep_sst)) abort(); + struct timespec wtime = {WSREP_TIMEDWAIT_SECONDS, 0}; + uint32 total_wtime = 0; + + if (mysql_mutex_lock (&LOCK_wsrep_sst)) + abort(); + + WSREP_INFO("Waiting for SST to complete."); + while (!sst_complete) { - WSREP_INFO("Waiting for SST to complete."); - mysql_cond_wait (&COND_wsrep_sst, &LOCK_wsrep_sst); + mysql_cond_timedwait (&COND_wsrep_sst, &LOCK_wsrep_sst, &wtime); + + if (!sst_complete) + { + total_wtime += wtime.tv_sec; + WSREP_DEBUG("Waiting for SST to complete. waited %u secs.", total_wtime); + service_manager_extend_timeout(WSREP_EXTEND_TIMEOUT_INTERVAL, + "WSREP state transfer ongoing, current seqno: %ld", local_seqno); + } } if (local_seqno >= 0) @@ -1298,10 +1319,22 @@ void wsrep_SE_init_grab() void wsrep_SE_init_wait() { + struct timespec wtime = {WSREP_TIMEDWAIT_SECONDS, 0}; + uint32 total_wtime=0; + while (SE_initialized == false) { - mysql_cond_wait (&COND_wsrep_sst_init, &LOCK_wsrep_sst_init); + mysql_cond_timedwait (&COND_wsrep_sst_init, &LOCK_wsrep_sst_init, &wtime); + + if (!SE_initialized) + { + total_wtime += wtime.tv_sec; + WSREP_DEBUG("Waiting for SST to complete. waited %u secs.", total_wtime); + service_manager_extend_timeout(WSREP_EXTEND_TIMEOUT_INTERVAL, + "WSREP SE initialization ongoing."); + } } + mysql_mutex_unlock (&LOCK_wsrep_sst_init); } From bf4244d1a0d921defbd92deb1e54b59ee1ce35b4 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Wed, 27 Jun 2018 17:01:09 +0400 Subject: [PATCH 193/203] MDEV-8540 - Crash on server shutdown since 10.0.16 Only close stdin if it was open initinally. Otherwise we may close file descriptor which is reused for different puprose (specifically for binlog index file in case of this bug). --- sql/mysqld.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index fb49c05def5..d760986a303 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5149,6 +5149,9 @@ int win_main(int argc, char **argv) int mysqld_main(int argc, char **argv) #endif { + /* We can't close stdin just now, because it may be booststrap mode. */ + bool please_close_stdin= fcntl(STDIN_FILENO, F_GETFD) >= 0; + /* Perform basic thread library and malloc initialization, to be able to read defaults files and parse options. @@ -5492,7 +5495,9 @@ int mysqld_main(int argc, char **argv) (char*) "" : mysqld_unix_port), mysqld_port, MYSQL_COMPILATION_COMMENT); - fclose(stdin); + /* Only close stdin if it was open initinally. */ + if (please_close_stdin) + fclose(stdin); #if defined(_WIN32) && !defined(EMBEDDED_LIBRARY) Service.SetRunning(); #endif From 377cd520643eba3dccc21336af582b170137d407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 27 Jun 2018 11:38:09 +0300 Subject: [PATCH 194/203] Pretty-print table names in some error messages --- storage/innobase/btr/btr0defragment.cc | 4 ++-- storage/innobase/dict/dict0dict.cc | 2 +- storage/innobase/handler/handler0alter.cc | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 7a8899ee22f..036e2ea776c 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -782,7 +782,7 @@ DECLARE_THREAD(btr_defragment_thread)(void*) err = dict_stats_save_defrag_stats(index); if (err != DB_SUCCESS) { ib::error() << "Saving defragmentation stats for table " - << index->table->name.m_name + << index->table->name << " index " << index->name() << " failed with error " << err; } else { @@ -790,7 +790,7 @@ DECLARE_THREAD(btr_defragment_thread)(void*) if (err != DB_SUCCESS) { ib::error() << "Saving defragmentation summary for table " - << index->table->name.m_name + << index->table->name << " index " << index->name() << " failed with error " << err; } diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index b1b1f396617..df5c678ffb5 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -574,7 +574,7 @@ dict_table_close_and_drop( if (err != DB_SUCCESS) { ib::error() << "At " << __FILE__ << ":" << __LINE__ << " row_merge_drop_table returned error: " << err - << " table: " << table->name.m_name; + << " table: " << table->name; } } diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index dd4362a9fe9..e38dc1951ca 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -9086,12 +9086,11 @@ foreign_fail: error = row_merge_drop_table(trx, ctx->old_table); if (error != DB_SUCCESS) { - ib::error() << "Inplace alter table " << ctx->old_table->name.m_name + ib::error() << "Inplace alter table " << ctx->old_table->name << " dropping copy of the old table failed error " << error << ". tmp_name " << (ctx->tmp_name ? ctx->tmp_name : "N/A") - << " new_table " << (ctx->new_table ? ctx->new_table->name.m_name - : "N/A"); + << " new_table " << ctx->new_table->name; } trx_commit_for_mysql(trx); From 090febbb2dedad764a4d9fdbc9216921dd27225f Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 28 Jun 2018 00:33:44 -0700 Subject: [PATCH 195/203] This is another attempt to fix mdev-16473. The previous correction of the patch for mdev-16473 did not work correctly for the databases whose names started with '*'. Added a test case with a database named "*". --- mysql-test/r/cte_nonrecursive.result | 11 +++++++++++ mysql-test/t/cte_nonrecursive.test | 7 +++++++ sql/sql_base.cc | 2 +- sql/sql_class.h | 5 ++++- sql/sql_parse.cc | 3 ++- sql/table.h | 3 ++- 6 files changed, 27 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/cte_nonrecursive.result b/mysql-test/r/cte_nonrecursive.result index f6b80156ee0..32104c53af5 100644 --- a/mysql-test/r/cte_nonrecursive.result +++ b/mysql-test/r/cte_nonrecursive.result @@ -1511,4 +1511,15 @@ a a 3 3 1 1 drop database db_mdev_16473; +create database `*` ; +create table `*`.t1 (a int); +insert into `*`.t1 values (2), (7), (3), (1); +use `*`; +select * from t1; +a +2 +7 +3 +1 +drop database `*`; use test; diff --git a/mysql-test/t/cte_nonrecursive.test b/mysql-test/t/cte_nonrecursive.test index 11c864bcac1..22bb292d8d9 100644 --- a/mysql-test/t/cte_nonrecursive.test +++ b/mysql-test/t/cte_nonrecursive.test @@ -1056,4 +1056,11 @@ select * from cte, db_mdev_16473.t1 as t where cte.a=t.a; drop database db_mdev_16473; +create database `*` ; +create table `*`.t1 (a int); +insert into `*`.t1 values (2), (7), (3), (1); +use `*`; +select * from t1; +drop database `*`; + use test; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 093f7cf3427..9905400109e 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -3331,7 +3331,7 @@ open_and_process_table(THD *thd, LEX *lex, TABLE_LIST *tables, Not a placeholder: must be a base/temporary table or a view. Let us open it. */ - if (tables->db[0] == no_db[0]) + if (tables->no_default_db && !tables->is_fqtn) { my_message(ER_NO_DB_ERROR, ER(ER_NO_DB_ERROR), MYF(0)); error= TRUE; diff --git a/sql/sql_class.h b/sql/sql_class.h index e79fde055c2..88d3af04376 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3959,7 +3959,10 @@ public: my_message(ER_NO_DB_ERROR, ER(ER_NO_DB_ERROR), MYF(0)); return TRUE; } - /* This will allow to throw an error later for non-CTE references */ + /* + It does not matter what database name is set in this case + because it will never be used after parser stage + */ *p_db_length= strlen(no_db); *p_db= strmake(no_db, *p_db_length); } diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 3631c8edd13..6bbc33d000a 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -139,7 +139,7 @@ static bool execute_show_status(THD *, TABLE_LIST *); static bool check_rename_table(THD *, TABLE_LIST *, TABLE_LIST *); const char *any_db="*any*"; // Special symbol for check_access -const char *no_db="*none*"; // Used when no default db is set +const char *no_db="*"; // Used when no default db is set const LEX_STRING command_name[257]={ { C_STRING_WITH_LEN("Sleep") }, //0 @@ -8188,6 +8188,7 @@ TABLE_LIST *st_select_lex::add_table_to_list(THD *thd, DBUG_RETURN(0); else ptr->is_fqtn= FALSE; + ptr->no_default_db= !thd->db && !(lex->sphead && lex->sphead->m_name.str); ptr->alias= alias_str; ptr->is_alias= alias ? TRUE : FALSE; diff --git a/sql/table.h b/sql/table.h index c0cca1026ea..f5d504e39d8 100644 --- a/sql/table.h +++ b/sql/table.h @@ -2095,7 +2095,8 @@ struct TABLE_LIST qualified name (.). */ bool is_fqtn; - + /** TRUE if no default database is defined for the table name */ + bool no_default_db; /* TRUE <=> derived table should be filled right after optimization. */ bool fill_me; /* TRUE <=> view/DT is merged. */ From 52a25d7b674464013b7749a366b0879929985a1e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 28 Jun 2018 12:36:32 +0200 Subject: [PATCH 196/203] MDEV-16473 WITH statement throws 'no database selected' error Different fix, just use NULL, not no_db, --- mysql-test/r/cte_nonrecursive.result | 11 ----------- mysql-test/t/cte_nonrecursive.test | 7 ------- sql/sql_acl.cc | 2 +- sql/sql_base.cc | 14 ++++++-------- sql/sql_class.h | 9 +++------ sql/sql_lex.h | 2 -- sql/sql_parse.cc | 10 ++-------- sql/table.h | 3 +-- 8 files changed, 13 insertions(+), 45 deletions(-) diff --git a/mysql-test/r/cte_nonrecursive.result b/mysql-test/r/cte_nonrecursive.result index 32104c53af5..f6b80156ee0 100644 --- a/mysql-test/r/cte_nonrecursive.result +++ b/mysql-test/r/cte_nonrecursive.result @@ -1511,15 +1511,4 @@ a a 3 3 1 1 drop database db_mdev_16473; -create database `*` ; -create table `*`.t1 (a int); -insert into `*`.t1 values (2), (7), (3), (1); -use `*`; -select * from t1; -a -2 -7 -3 -1 -drop database `*`; use test; diff --git a/mysql-test/t/cte_nonrecursive.test b/mysql-test/t/cte_nonrecursive.test index 22bb292d8d9..11c864bcac1 100644 --- a/mysql-test/t/cte_nonrecursive.test +++ b/mysql-test/t/cte_nonrecursive.test @@ -1056,11 +1056,4 @@ select * from cte, db_mdev_16473.t1 as t where cte.a=t.a; drop database db_mdev_16473; -create database `*` ; -create table `*`.t1 (a int); -insert into `*`.t1 values (2), (7), (3), (1); -use `*`; -select * from t1; -drop database `*`; - use test; diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 1809e22ffbf..792fe700473 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -7581,7 +7581,7 @@ bool check_grant(THD *thd, ulong want_access, TABLE_LIST *tables, tl->correspondent_table ? tl->correspondent_table : tl; sctx= t_ref->security_ctx ? t_ref->security_ctx : thd->security_ctx; - if (tl->with || + if (tl->with || !tl->db || (tl->select_lex && (tl->with= tl->select_lex->find_table_def_in_with_clauses(tl)))) continue; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 9905400109e..32e42daf7c4 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -1531,6 +1531,12 @@ bool open_table(THD *thd, TABLE_LIST *table_list, Open_table_context *ot_ctx) DBUG_RETURN(true); } + if (!table_list->db) + { + my_error(ER_NO_DB_ERROR, MYF(0)); + DBUG_RETURN(true); + } + key_length= get_table_def_key(table_list, &key); /* @@ -3330,14 +3336,6 @@ open_and_process_table(THD *thd, LEX *lex, TABLE_LIST *tables, /* Not a placeholder: must be a base/temporary table or a view. Let us open it. */ - - if (tables->no_default_db && !tables->is_fqtn) - { - my_message(ER_NO_DB_ERROR, ER(ER_NO_DB_ERROR), MYF(0)); - error= TRUE; - goto end; - } - if (tables->table) { /* diff --git a/sql/sql_class.h b/sql/sql_class.h index 88d3af04376..b377d74445f 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3959,12 +3959,9 @@ public: my_message(ER_NO_DB_ERROR, ER(ER_NO_DB_ERROR), MYF(0)); return TRUE; } - /* - It does not matter what database name is set in this case - because it will never be used after parser stage - */ - *p_db_length= strlen(no_db); - *p_db= strmake(no_db, *p_db_length); + /* This will allow to throw an error later for non-CTE references */ + *p_db= NULL; + *p_db_length= 0; } else { diff --git a/sql/sql_lex.h b/sql/sql_lex.h index ae010a88e46..a1f6b202ae6 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -156,8 +156,6 @@ struct LEX_TYPE extern const LEX_STRING null_lex_str; extern const LEX_STRING empty_lex_str; -extern const char *no_db; - enum enum_sp_suid_behaviour { SP_IS_DEFAULT_SUID= 0, diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 6bbc33d000a..df0ee2bd680 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -139,7 +139,6 @@ static bool execute_show_status(THD *, TABLE_LIST *); static bool check_rename_table(THD *, TABLE_LIST *, TABLE_LIST *); const char *any_db="*any*"; // Special symbol for check_access -const char *no_db="*"; // Used when no default db is set const LEX_STRING command_name[257]={ { C_STRING_WITH_LEN("Sleep") }, //0 @@ -6685,11 +6684,7 @@ check_access(THD *thd, ulong want_access, const char *db, ulong *save_priv, THD_STAGE_INFO(thd, stage_checking_permissions); if ((!db || !db[0]) && !thd->db && !dont_check_global_grants) { - DBUG_PRINT("error",("No database")); - if (!no_errors) - my_message(ER_NO_DB_ERROR, ER_THD(thd, ER_NO_DB_ERROR), - MYF(0)); /* purecov: tested */ - DBUG_RETURN(TRUE); /* purecov: tested */ + DBUG_RETURN(FALSE); // CTE reference or an error later } if ((db != NULL) && (db != any_db)) @@ -8188,7 +8183,6 @@ TABLE_LIST *st_select_lex::add_table_to_list(THD *thd, DBUG_RETURN(0); else ptr->is_fqtn= FALSE; - ptr->no_default_db= !thd->db && !(lex->sphead && lex->sphead->m_name.str); ptr->alias= alias_str; ptr->is_alias= alias ? TRUE : FALSE; @@ -8301,7 +8295,7 @@ TABLE_LIST *st_select_lex::add_table_to_list(THD *thd, lex->add_to_query_tables(ptr); // Pure table aliases do not need to be locked: - if (!MY_TEST(table_options & TL_OPTION_ALIAS)) + if (ptr->db && !(table_options & TL_OPTION_ALIAS)) { ptr->mdl_request.init(MDL_key::TABLE, ptr->db, ptr->table_name, mdl_type, MDL_TRANSACTION); diff --git a/sql/table.h b/sql/table.h index f5d504e39d8..c0cca1026ea 100644 --- a/sql/table.h +++ b/sql/table.h @@ -2095,8 +2095,7 @@ struct TABLE_LIST qualified name (.). */ bool is_fqtn; - /** TRUE if no default database is defined for the table name */ - bool no_default_db; + /* TRUE <=> derived table should be filled right after optimization. */ bool fill_me; /* TRUE <=> view/DT is merged. */ From 00ccff48af17d82c815909aaee170c59c9439675 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Sun, 18 Mar 2018 21:01:41 +0200 Subject: [PATCH 197/203] MDEV-14014 Multi-Slave Replication Fail: bogus data in log event MDEV-7257 made a dump thread to read from binlog concurrently with writers as long as the read bytes are below a water-mark (MYSQL_BIN_LOG::binlog_end_pos). However it appeared to be possible a dump thread reader reach out for bytes past the water mark through a feature of IO_CACHE that fills in the internal buffer and while doing so it could read what the reader is not supposed to see (the bytes above MYSQL_BIN_LOG::binlog_end_pos). The issue is fixed with constraining the IO_CACHE buffer fill to respect the watermark. An added unit test proves reading from file is bound to an external parameter passed to {IO_CACHE::end_of_file} cache member. --- sql/sql_repl.cc | 1 + unittest/sql/mf_iocache-t.cc | 37 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 4ec70bf31da..35cdc98e0f4 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2538,6 +2538,7 @@ static int send_events(binlog_send_info *info, IO_CACHE* log, LOG_INFO* linfo, linfo->pos= my_b_tell(log); info->last_pos= my_b_tell(log); + log->end_of_file= end_pos; while (linfo->pos < end_pos) { if (should_stop(info)) diff --git a/unittest/sql/mf_iocache-t.cc b/unittest/sql/mf_iocache-t.cc index 9f2e4b1b303..34717f04e33 100644 --- a/unittest/sql/mf_iocache-t.cc +++ b/unittest/sql/mf_iocache-t.cc @@ -253,10 +253,43 @@ void mdev10259() } +void mdev14014() +{ + int res; + uchar buf_o[200]; + uchar buf_i[200]; + memset(buf_i, 0, sizeof( buf_i)); + memset(buf_o, FILL, sizeof(buf_o)); + + diag("MDEV-14014 Dump thread reads past last 'officially' written byte"); + + init_io_cache_encryption(); + + res= open_cached_file(&info, 0, 0, CACHE_SIZE, 0); + ok(res == 0, "open_cached_file" INFO_TAIL); + + res= my_b_write(&info, buf_o, sizeof(buf_o)); + ok(res == 0, "buffer is written" INFO_TAIL); + + res= my_b_flush_io_cache(&info, 1); + ok(res == 0, "flush" INFO_TAIL); + + res= reinit_io_cache(&info, READ_CACHE, 0, 0, 0); + ok(res == 0, "reinit READ_CACHE" INFO_TAIL); + + info.end_of_file= 100; + res= my_b_read(&info, buf_i, sizeof(buf_i)); + ok(res == 1 && buf_i[100] == 0 && buf_i[200-1] == 0, + "short read leaves buf_i[100..200-1] == 0"); + + close_cached_file(&info); +} + + int main(int argc __attribute__((unused)),char *argv[]) { MY_INIT(argv[0]); - plan(46); + plan(51); /* temp files with and without encryption */ encrypt_tmp_files= 1; @@ -272,6 +305,8 @@ int main(int argc __attribute__((unused)),char *argv[]) mdev10259(); encrypt_tmp_files= 0; + mdev14014(); + my_end(0); return exit_status(); } From 78a0646fe4ab7d58d900192ef7f4df905ed7442e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 28 Jun 2018 09:05:01 +0200 Subject: [PATCH 198/203] make plugins.processlist more robust --- mysql-test/suite/plugins/r/processlist.result | 4 +--- mysql-test/suite/plugins/t/processlist.test | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/plugins/r/processlist.result b/mysql-test/suite/plugins/r/processlist.result index 22515c8174b..c27534d9d78 100644 --- a/mysql-test/suite/plugins/r/processlist.result +++ b/mysql-test/suite/plugins/r/processlist.result @@ -2,10 +2,8 @@ create table t1 (a int) engine=innodb; start transaction; insert t1 values (1); connect con2,localhost,root; -state from show engine innodb status +state from show engine innodb status, must be empty -state from show processlist - disconnect con2; connection default; drop table t1; diff --git a/mysql-test/suite/plugins/t/processlist.test b/mysql-test/suite/plugins/t/processlist.test index e8f03aacb10..39b715b867b 100644 --- a/mysql-test/suite/plugins/t/processlist.test +++ b/mysql-test/suite/plugins/t/processlist.test @@ -8,11 +8,12 @@ start transaction; insert t1 values (1); let id=`select connection_id()`; connect con2,localhost,root; +let $wait_condition=select state='' from information_schema.processlist where id = $id; +--source include/wait_condition.inc replace_regex /\"/-/; #" let s=`show engine innodb status`; disable_query_log; -eval select regexp_replace("$s", '(?s)^.*MySQL thread id $id,.*root([^\n]*)\n.*', '\\\\1') as `state from show engine innodb status`; -eval select state as `state from show processlist` from information_schema.processlist where id = $id; +eval select regexp_replace("$s", '(?s)^.*MySQL thread id $id,.*root([^\n]*)\n.*', '\\\\1') as `state from show engine innodb status, must be empty`; enable_query_log; disconnect con2; connection default; From 3d4beee1a98cebd6eb566e38569e19599c2b9a98 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 28 Jun 2018 11:59:25 +0200 Subject: [PATCH 199/203] remove double-counting rnd_pos_by_record calls ha_rnd_pos, which does the counting --- sql/sql_class.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/sql_class.h b/sql/sql_class.h index ec94f6c45d2..9428f6f6ce5 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -5012,8 +5012,6 @@ inline int handler::ha_ft_read(uchar *buf) inline int handler::ha_rnd_pos_by_record(uchar *buf) { int error= rnd_pos_by_record(buf); - if (!error) - update_rows_read(); table->status=error ? STATUS_NOT_FOUND: 0; return error; } From 16c14d7ba05ed3ad6ce1f3a8e0447d6304a7e636 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 21 Jun 2018 09:43:05 +0200 Subject: [PATCH 200/203] mark ed25519 stable --- mysql-test/suite/plugins/r/auth_ed25519.result | 2 +- plugin/auth_ed25519/server_ed25519.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/plugins/r/auth_ed25519.result b/mysql-test/suite/plugins/r/auth_ed25519.result index a3b85a11dea..371d20befeb 100644 --- a/mysql-test/suite/plugins/r/auth_ed25519.result +++ b/mysql-test/suite/plugins/r/auth_ed25519.result @@ -32,7 +32,7 @@ PLUGIN_AUTHOR Sergei Golubchik PLUGIN_DESCRIPTION Elliptic curve ED25519 based authentication PLUGIN_LICENSE GPL LOAD_OPTION ON -PLUGIN_MATURITY Beta +PLUGIN_MATURITY Stable PLUGIN_AUTH_VERSION 1.0-alpha create user test1@localhost identified via ed25519 using 'ZIgUREUg5PVgQ6LskhXmO+eZLS0nC8be6HPjYWR4YJY'; show grants for test1@localhost; diff --git a/plugin/auth_ed25519/server_ed25519.c b/plugin/auth_ed25519/server_ed25519.c index ac443b43b09..7720a0a4074 100644 --- a/plugin/auth_ed25519/server_ed25519.c +++ b/plugin/auth_ed25519/server_ed25519.c @@ -102,7 +102,7 @@ maria_declare_plugin(ed25519) NULL, NULL, "1.0-alpha", - MariaDB_PLUGIN_MATURITY_BETA + MariaDB_PLUGIN_MATURITY_STABLE } maria_declare_plugin_end; From 8ca18294d59e4df82dacba69f9853e568ab2e0eb Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Sun, 18 Mar 2018 21:01:41 +0200 Subject: [PATCH 201/203] MDEV-14014 Multi-Slave Replication Fail: bogus data in log event MDEV-7257 made a dump thread to read from binlog concurrently with writers as long as the read bytes are below a water-mark (MYSQL_BIN_LOG::binlog_end_pos). However it appeared to be possible a dump thread reader reach out for bytes past the water mark through a feature of IO_CACHE that fills in the internal buffer and while doing so it could read what the reader is not supposed to see (the bytes above MYSQL_BIN_LOG::binlog_end_pos). The issue is fixed with constraining the IO_CACHE buffer fill to respect the watermark. An added unit test proves reading from file is bound to an external parameter passed to {IO_CACHE::end_of_file} cache member. --- sql/sql_repl.cc | 1 + unittest/sql/mf_iocache-t.cc | 37 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 569c3d2c4ef..db608de5147 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2546,6 +2546,7 @@ static int send_events(binlog_send_info *info, IO_CACHE* log, LOG_INFO* linfo, linfo->pos= my_b_tell(log); info->last_pos= my_b_tell(log); + log->end_of_file= end_pos; while (linfo->pos < end_pos) { if (should_stop(info)) diff --git a/unittest/sql/mf_iocache-t.cc b/unittest/sql/mf_iocache-t.cc index 1b04f8eb0d3..2cd5b678700 100644 --- a/unittest/sql/mf_iocache-t.cc +++ b/unittest/sql/mf_iocache-t.cc @@ -253,10 +253,43 @@ void mdev10259() } +void mdev14014() +{ + int res; + uchar buf_o[200]; + uchar buf_i[200]; + memset(buf_i, 0, sizeof( buf_i)); + memset(buf_o, FILL, sizeof(buf_o)); + + diag("MDEV-14014 Dump thread reads past last 'officially' written byte"); + + init_io_cache_encryption(); + + res= open_cached_file(&info, 0, 0, CACHE_SIZE, 0); + ok(res == 0, "open_cached_file" INFO_TAIL); + + res= my_b_write(&info, buf_o, sizeof(buf_o)); + ok(res == 0, "buffer is written" INFO_TAIL); + + res= my_b_flush_io_cache(&info, 1); + ok(res == 0, "flush" INFO_TAIL); + + res= reinit_io_cache(&info, READ_CACHE, 0, 0, 0); + ok(res == 0, "reinit READ_CACHE" INFO_TAIL); + + info.end_of_file= 100; + res= my_b_read(&info, buf_i, sizeof(buf_i)); + ok(res == 1 && buf_i[100] == 0 && buf_i[200-1] == 0, + "short read leaves buf_i[100..200-1] == 0"); + + close_cached_file(&info); +} + + int main(int argc __attribute__((unused)),char *argv[]) { MY_INIT(argv[0]); - plan(46); + plan(51); /* temp files with and without encryption */ encrypt_tmp_files= 1; @@ -272,6 +305,8 @@ int main(int argc __attribute__((unused)),char *argv[]) mdev10259(); encrypt_tmp_files= 0; + mdev14014(); + my_end(0); return exit_status(); } From 45cabf10175da1ae2d158ea17ccd6e19f461d6f4 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 28 Jun 2018 16:17:21 +0200 Subject: [PATCH 202/203] MDEV-16615 ASAN SEGV in handler::print_error or server crash after error upon CREATE TABLE table->in_use is not always set and a KILL signal can arrive anytime. --- mysql-test/r/max_statement_time.result | 3 +++ mysql-test/t/max_statement_time.test | 8 ++++++++ sql/handler.cc | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/max_statement_time.result b/mysql-test/r/max_statement_time.result index 44ee03b813a..a87a899b575 100644 --- a/mysql-test/r/max_statement_time.result +++ b/mysql-test/r/max_statement_time.result @@ -181,3 +181,6 @@ ERROR 70100: Query execution was interrupted (max_statement_time exceeded) set max_statement_time = 0; drop procedure pr; drop table t1; +SET max_statement_time= 1; +CREATE TABLE t ENGINE=InnoDB SELECT * FROM seq_1_to_50000; +ERROR 70100: Query execution was interrupted (max_statement_time exceeded) diff --git a/mysql-test/t/max_statement_time.test b/mysql-test/t/max_statement_time.test index 0882daff139..24b6d9311f2 100644 --- a/mysql-test/t/max_statement_time.test +++ b/mysql-test/t/max_statement_time.test @@ -5,6 +5,7 @@ --source include/not_embedded.inc --source include/have_innodb.inc +--source include/have_sequence.inc --source include/not_valgrind.inc --echo @@ -226,3 +227,10 @@ call pr(); set max_statement_time = 0; drop procedure pr; drop table t1; + +# +# MDEV-16615 ASAN SEGV in handler::print_error or server crash after error upon CREATE TABLE +# +SET max_statement_time= 1; +--error ER_STATEMENT_TIMEOUT +CREATE TABLE t ENGINE=InnoDB SELECT * FROM seq_1_to_50000; diff --git a/sql/handler.cc b/sql/handler.cc index 35b0814ef79..9c319b995da 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3413,8 +3413,8 @@ void handler::print_error(int error, myf errflag) break; case HA_ERR_ABORTED_BY_USER: { - DBUG_ASSERT(table->in_use->killed); - table->in_use->send_kill_message(); + DBUG_ASSERT(ha_thd()->killed); + ha_thd()->send_kill_message(); DBUG_VOID_RETURN; } case HA_ERR_WRONG_MRG_TABLE_DEF: From 04677f44c7a0faee366de265c92175daadf45bc9 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 Jun 2018 17:23:05 +0100 Subject: [PATCH 203/203] Innodb : do not use errno on Windows to print os_file_pwrite() error. Use GetLastError() instead. --- storage/innobase/os/os0file.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 568239b79ed..01b801461d6 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -5037,12 +5037,12 @@ os_file_write_func( << offset << ", " << n << " bytes should have been written," " only " << n_bytes << " were written." - " Operating system error number " << errno << "." + " Operating system error number " << IF_WIN(GetLastError(),errno) << "." " Check that your OS and file system" " support files of this size." " Check also that the disk is not full" " or a disk quota exceeded."; - +#ifndef _WIN32 if (strerror(errno) != NULL) { ib::error() @@ -5051,7 +5051,7 @@ os_file_write_func( } ib::info() << OPERATING_SYSTEM_ERROR_MSG; - +#endif os_has_said_disk_full = true; }