From ccec6b887b073dd509f5645aad7ab38ce5f972ac Mon Sep 17 00:00:00 2001 From: sjaakola Date: Wed, 27 May 2020 21:21:24 +0300 Subject: [PATCH 01/73] MDEV-22729 fixes for galera.galera_slave_replay test The test was changing wsrep_on option in node_3, which is native MariaDB server (i.e. not a cluster node). Native NariaDB server should not manipulate wsrep replication state, this problem is fixed. galera.galera_slave_replay test phase 2 will cause certification failure for async slave SQL handler thread. This certification failure is now monitored and required to happen in the test. The test phase 2, generates scenario, where async slave SQL handler faces certification failure and galera slave applier is paused when this happens. This makes the test vulnerable for anomaly described in MDEV-22632. Therefore the fix in this commit depends on MDEV-22632, and should be merged after the fix for MDEV-22632. --- .../suite/galera/r/galera_slave_replay.result | 14 +++------ .../suite/galera/t/galera_slave_replay.test | 29 +++++++------------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_slave_replay.result b/mysql-test/suite/galera/r/galera_slave_replay.result index e8dd6ae87b1..0b0199c4a02 100644 --- a/mysql-test/suite/galera/r/galera_slave_replay.result +++ b/mysql-test/suite/galera/r/galera_slave_replay.result @@ -4,9 +4,7 @@ connection node_2; connection node_1; connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; connection node_3; -SET GLOBAL wsrep_on=OFF; RESET MASTER; -SET GLOBAL wsrep_on=ON; connection node_2a; START SLAVE; connection node_3; @@ -72,20 +70,17 @@ SET AUTOCOMMIT=ON; START TRANSACTION; UPDATE t1 SET f2 = 'd' WHERE f1 = 3; connection node_2a; -SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync'; SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; -connection node_3; +connection node_1; UPDATE test.t1 SET f2 = 'e' WHERE f1 = 3; +connection node_2a; +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached"; connection node_3; COMMIT; connection node_2a; SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; connection node_2a; -SET GLOBAL wsrep_provider_options = 'dbug='; -SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync'; -SET DEBUG_SYNC = "RESET"; -connection node_2a; set session wsrep_sync_wait=15; SELECT COUNT(*) = 1 FROM test.t1 WHERE f2 = 'e'; COUNT(*) = 1 @@ -93,9 +88,8 @@ COUNT(*) = 1 set session wsrep_sync_wait=0; STOP SLAVE; RESET SLAVE; +SET DEBUG_SYNC = "RESET"; DROP TABLE t1; connection node_3; DROP TABLE t1; -SET GLOBAL wsrep_on=OFF; RESET MASTER; -SET GLOBAL wsrep_on=ON; diff --git a/mysql-test/suite/galera/t/galera_slave_replay.test b/mysql-test/suite/galera/t/galera_slave_replay.test index 37c4cbd5b43..bac394baf64 100644 --- a/mysql-test/suite/galera/t/galera_slave_replay.test +++ b/mysql-test/suite/galera/t/galera_slave_replay.test @@ -21,9 +21,7 @@ # --connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 --connection node_3 -SET GLOBAL wsrep_on=OFF; RESET MASTER; -SET GLOBAL wsrep_on=ON; --connection node_2a # @@ -156,35 +154,31 @@ UPDATE t1 SET f2 = 'd' WHERE f1 = 3; --let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; --source include/wait_condition.inc -# Block the commit ---let $galera_sync_point = commit_monitor_master_enter_sync ---source include/galera_set_sync_point.inc - # block applier SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; -# Inject a conflicting update from node 3 ---connection node_3 +# Inject a conflicting update from node 1 +--connection node_1 UPDATE test.t1 SET f2 = 'e' WHERE f1 = 3; +--connection node_2a +# wait until applier has reached the sync point +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached"; +--let $expected_cert_failures = `SELECT VARIABLE_VALUE+1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_cert_failures'` + # send the update from master --connection node_3 --error 0 COMMIT; --connection node_2a +--let $wait_condition = SELECT VARIABLE_VALUE = $expected_cert_failures FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_cert_failures' +--source include/wait_condition.inc -# release the applier +# release the applier from node 1 SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; - -# Unblock the async slave commit ---connection node_2a ---source include/galera_clear_sync_point.inc ---source include/galera_signal_sync_point.inc -SET DEBUG_SYNC = "RESET"; - --connection node_2a set session wsrep_sync_wait=15; @@ -193,11 +187,10 @@ set session wsrep_sync_wait=0; STOP SLAVE; RESET SLAVE; +SET DEBUG_SYNC = "RESET"; DROP TABLE t1; --connection node_3 DROP TABLE t1; -SET GLOBAL wsrep_on=OFF; RESET MASTER; -SET GLOBAL wsrep_on=ON; From 12f6db967bd1d83cf035aa281aee1a1d46b9b1a4 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Mon, 1 Jun 2020 12:34:33 +0300 Subject: [PATCH 02/73] MDEV-22763 backporting MDEV-20225 fix into 10.1 Backported the support for aborting and replaying stored procedure and fix for trigger key assigments from 10.4 version. Backported also two mtr tests: wsrep_sp_bf_abort and MDEV-20225 --- mysql-test/suite/galera/r/MDEV-20225.result | 16 + .../suite/galera/r/galera_sp_bf_abort.result | 294 +++++++++++++++ mysql-test/suite/galera/t/MDEV-20225.test | 49 +++ .../suite/galera/t/galera_sp_bf_abort.inc | 36 ++ .../suite/galera/t/galera_sp_bf_abort.test | 347 ++++++++++++++++++ sql/sp_head.cc | 89 +++++ sql/sql_trigger.cc | 16 +- sql/wsrep_hton.cc | 22 +- sql/wsrep_mysqld.cc | 9 +- sql/wsrep_thd.cc | 111 +++++- sql/wsrep_thd.h | 1 + 11 files changed, 978 insertions(+), 12 deletions(-) create mode 100644 mysql-test/suite/galera/r/MDEV-20225.result create mode 100644 mysql-test/suite/galera/r/galera_sp_bf_abort.result create mode 100644 mysql-test/suite/galera/t/MDEV-20225.test create mode 100644 mysql-test/suite/galera/t/galera_sp_bf_abort.inc create mode 100644 mysql-test/suite/galera/t/galera_sp_bf_abort.test diff --git a/mysql-test/suite/galera/r/MDEV-20225.result b/mysql-test/suite/galera/r/MDEV-20225.result new file mode 100644 index 00000000000..582353f10a6 --- /dev/null +++ b/mysql-test/suite/galera/r/MDEV-20225.result @@ -0,0 +1,16 @@ +CREATE TABLE t1 (f1 INT NOT NULL PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; +CREATE TABLE t2 (f1 INT NOT NULL PRIMARY KEY AUTO_INCREMENT, f2 INT) ENGINE=InnoDB; +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW INSERT INTO t2 VALUES (NULL, NEW.f1); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_slave_threads = 2; +SET GLOBAL debug_dbug = 'd,sync.mdev_20225'; +DROP TRIGGER tr1; +INSERT INTO t1 VALUES (NULL); +SET GLOBAL debug_dbug = 'RESET'; +SET DEBUG_SYNC = 'now SIGNAL signal.mdev_20225_continue'; +SET DEBUG_SYNC = 'RESET'; +SET GLOBAL wsrep_slave_threads = 1; +SHOW TRIGGERS; +Trigger Event Table Statement Timing Created sql_mode Definer character_set_client collation_connection Database Collation +DROP TABLE t1; +DROP TABLE t2; diff --git a/mysql-test/suite/galera/r/galera_sp_bf_abort.result b/mysql-test/suite/galera/r/galera_sp_bf_abort.result new file mode 100644 index 00000000000..205d73dd763 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sp_bf_abort.result @@ -0,0 +1,294 @@ +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)); +SET SESSION wsrep_sync_wait = 0; +CREATE PROCEDURE proc_update_insert() +BEGIN +UPDATE t1 SET f2 = 'b'; +INSERT INTO t1 VALUES (4, 'd'); +END| +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +INSERT INTO t1 VALUES (2, 'c'); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_replicate_sync'; +CALL proc_update_insert; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=after_replicate_sync'; +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +f1 f2 +1 b +2 c +3 b +4 d +wsrep_local_replays +1 +DELETE FROM t1; +CREATE PROCEDURE proc_update_insert_with_exit_handler() +BEGIN +DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN END; +UPDATE t1 SET f2 = 'b'; +INSERT INTO t1 VALUES (4, 'd'); +END| +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +INSERT INTO t1 VALUES (2, 'c'); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_replicate_sync'; +CALL proc_update_insert_with_exit_handler; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=after_replicate_sync'; +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +f1 f2 +1 b +2 c +3 b +4 d +wsrep_local_replays +1 +DELETE FROM t1; +CREATE PROCEDURE proc_update_insert_with_continue_handler() +BEGIN +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +UPDATE t1 SET f2 = 'b'; +INSERT INTO t1 VALUES (4, 'd'); +END| +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +INSERT INTO t1 VALUES (2, 'c'); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_replicate_sync'; +CALL proc_update_insert_with_continue_handler; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=after_replicate_sync'; +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +f1 f2 +1 b +2 c +3 b +4 d +wsrep_local_replays +1 +DELETE FROM t1; +CREATE PROCEDURE proc_update_insert_transaction() +BEGIN +START TRANSACTION; +UPDATE t1 SET f2 = 'b'; +INSERT INTO t1 VALUES (4, 'd'); +COMMIT; +END| +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +INSERT INTO t1 VALUES (2, 'c'); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_replicate_sync'; +CALL proc_update_insert_transaction; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=after_replicate_sync'; +Warnings: +Error 1317 Query execution was interrupted +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +f1 f2 +1 b +2 c +3 b +4 d +wsrep_local_replays +1 +DELETE FROM t1; +CREATE PROCEDURE proc_update_insert_transaction_with_continue_handler() +BEGIN +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +START TRANSACTION; +UPDATE t1 SET f2 = 'b'; +INSERT INTO t1 VALUES (4, 'd'); +COMMIT; +END| +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +INSERT INTO t1 VALUES (2, 'c'); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_replicate_sync'; +CALL proc_update_insert_transaction_with_continue_handler; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=after_replicate_sync'; +Warnings: +Error 1317 Query execution was interrupted +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +f1 f2 +1 b +2 c +3 b +4 d +wsrep_local_replays +1 +DELETE FROM t1; +CREATE PROCEDURE proc_update_insert_transaction_with_exit_handler() +BEGIN +DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN END; +START TRANSACTION; +UPDATE t1 SET f2 = 'b'; +INSERT INTO t1 VALUES (4, 'd'); +COMMIT; +END| +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +INSERT INTO t1 VALUES (2, 'c'); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_replicate_sync'; +CALL proc_update_insert_transaction_with_exit_handler; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=after_replicate_sync'; +Warnings: +Error 1317 Query execution was interrupted +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +f1 f2 +1 b +2 c +3 b +4 d +wsrep_local_replays +1 +DELETE FROM t1; +CREATE PROCEDURE proc_insert_insert_conflict() +BEGIN +INSERT INTO t1 VALUES (2, 'd'); +INSERT INTO t1 VALUES (4, 'd'); +END| +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +INSERT INTO t1 VALUES (2, 'c'); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_replicate_sync'; +CALL proc_insert_insert_conflict; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=after_replicate_sync'; +Got one of the listed errors +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +f1 f2 +1 a +2 c +3 a +wsrep_local_replays +1 +DELETE FROM t1; +CREATE PROCEDURE proc_insert_insert_conflict_with_exit_handler() +BEGIN +DECLARE EXIT HANDLER FOR SQLEXCEPTION SELECT "Conflict exit handler"; +INSERT INTO t1 VALUES (2, 'd'); +INSERT INTO t1 VALUES (4, 'd'); +END| +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +INSERT INTO t1 VALUES (2, 'c'); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_replicate_sync'; +CALL proc_insert_insert_conflict_with_exit_handler; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=after_replicate_sync'; +Conflict exit handler +Conflict exit handler +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +f1 f2 +1 a +2 c +3 a +wsrep_local_replays +1 +DELETE FROM t1; +CREATE PROCEDURE proc_insert_insert_conflict_with_continue_handler() +BEGIN +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION SELECT "Conflict continue handler"; +INSERT INTO t1 VALUES (2, 'd'); +INSERT INTO t1 VALUES (4, 'd'); +END| +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +INSERT INTO t1 VALUES (2, 'c'); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_replicate_sync'; +CALL proc_insert_insert_conflict_with_continue_handler; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=after_replicate_sync'; +Conflict continue handler +Conflict continue handler +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +f1 f2 +1 a +2 c +3 a +4 d +wsrep_local_replays +1 +DELETE FROM t1; +DROP PROCEDURE proc_update_insert; +DROP PROCEDURE proc_update_insert_with_continue_handler; +DROP PROCEDURE proc_update_insert_with_exit_handler; +DROP PROCEDURE proc_update_insert_transaction; +DROP PROCEDURE proc_update_insert_transaction_with_continue_handler; +DROP PROCEDURE proc_update_insert_transaction_with_exit_handler; +DROP PROCEDURE proc_insert_insert_conflict; +DROP PROCEDURE proc_insert_insert_conflict_with_exit_handler; +DROP PROCEDURE proc_insert_insert_conflict_with_continue_handler; +DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/MDEV-20225.test b/mysql-test/suite/galera/t/MDEV-20225.test new file mode 100644 index 00000000000..5fbd0965217 --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-20225.test @@ -0,0 +1,49 @@ +# +# MDEV-20225 - Verify that DROP TRIGGER gets keys assigned corresponding +# to all affected tables. +# + +--source include/galera_cluster.inc +--source include/have_innodb.inc + +CREATE TABLE t1 (f1 INT NOT NULL PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; +CREATE TABLE t2 (f1 INT NOT NULL PRIMARY KEY AUTO_INCREMENT, f2 INT) ENGINE=InnoDB; + +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW INSERT INTO t2 VALUES (NULL, NEW.f1); + +--connection node_2 +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_slave_threads = 2; +SET GLOBAL debug_dbug = 'd,sync.mdev_20225'; + +--let $galera_connection_name = node_1a +--let $galera_server_number = 1 +--source include/galera_connect.inc +DROP TRIGGER tr1; + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE = 'debug sync point: now' +--source include/wait_condition.inc + + +--connection node_1 +INSERT INTO t1 VALUES (NULL); +# We must rely on sleep here. If the bug is fixed, the second applier +# is not allowed to go past apply monitor which would trigger the bug, +# so there is no sync point or condition to wait. +--sleep 1 + +--connection node_2 +SET GLOBAL debug_dbug = 'RESET'; +SET DEBUG_SYNC = 'now SIGNAL signal.mdev_20225_continue'; +SET DEBUG_SYNC = 'RESET'; +SET GLOBAL wsrep_slave_threads = 1; + +--let $wait_condition = SELECT COUNT(*) = 1 FROM test.t1; +--source include/wait_condition.inc + +# Trigger should now be dropped on node_2. +SHOW TRIGGERS; + +DROP TABLE t1; +DROP TABLE t2; diff --git a/mysql-test/suite/galera/t/galera_sp_bf_abort.inc b/mysql-test/suite/galera/t/galera_sp_bf_abort.inc new file mode 100644 index 00000000000..7ca8ecf20a9 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sp_bf_abort.inc @@ -0,0 +1,36 @@ +# +# Issue an INSERT for gap between 1 and 3 to node_2 and wait until it hits +# apply monitor sync point on node_1 +# + +--connection node_1a +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_set_sync_point.inc + +--connection node_2 +--eval $galera_sp_bf_abort_conflict + +--connection node_1a +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc + +# Send a procedure to node_1 which should take a gap lock between +# rows 1 and 3. It does not conflict with INSERT from node_2 in +# certification. Park the UPDATE after replicate and let INSERT to +# continue applying, generating a BF abort. + +--let $galera_sync_point = after_replicate_sync +--source include/galera_set_sync_point.inc + +--connection node_1 +--send_eval CALL $galera_sp_bf_abort_proc + +--connection node_1a +--let $galera_sync_point = after_replicate_sync apply_monitor_slave_enter_sync +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc + +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_signal_sync_point.inc +--let $galera_sync_point = after_replicate_sync +--source include/galera_signal_sync_point.inc diff --git a/mysql-test/suite/galera/t/galera_sp_bf_abort.test b/mysql-test/suite/galera/t/galera_sp_bf_abort.test new file mode 100644 index 00000000000..484e2ca478d --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sp_bf_abort.test @@ -0,0 +1,347 @@ +# +# Test cases for stored procedure BF aborts. +# + +--source include/galera_cluster.inc +--source include/have_innodb.inc + +--connection node_1 + +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)); + +# Control connection for Galera sync point management +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +--connection node_1a +SET SESSION wsrep_sync_wait = 0; + +--connection node_1 +# +# Case 1a: Procedure does and UPDATE which will suffer BF abort +# but there is no actual conflict and non-conflicting INSERT. +# The expected outcome is that both UPDATE and INSERT will succedd +# and no errors are reported to the client, wsrep_local_replays is +# incremented by one. +# +DELIMITER |; +CREATE PROCEDURE proc_update_insert() +BEGIN + UPDATE t1 SET f2 = 'b'; + INSERT INTO t1 VALUES (4, 'd'); +END| +DELIMITER ;| + +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +--let $wsrep_local_replays_orig = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--let $galera_sp_bf_abort_proc = proc_update_insert +--let $galera_sp_bf_abort_conflict = INSERT INTO t1 VALUES (2, 'c') +--source galera_sp_bf_abort.inc +--connection node_1 +--reap +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +--let $wsrep_local_replays_curr = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_curr - $wsrep_local_replays_orig = 1 AS wsrep_local_replays; +--enable_query_log + +DELETE FROM t1; + +--connection node_1 +# +# Case 1b: Procedure does and UPDATE which will suffer BF abort +# but there is no actual conflict and non-conflicting INSERT. +# An EXIT HANDLER is declared for the procedure. +# The expected outcome is that both UPDATE and INSERT will succedd +# and no errors are reported to the client, wsrep_local_replays is +# incremented by one. +# +DELIMITER |; +CREATE PROCEDURE proc_update_insert_with_exit_handler() +BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN END; + UPDATE t1 SET f2 = 'b'; + INSERT INTO t1 VALUES (4, 'd'); +END| +DELIMITER ;| + +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +--let $wsrep_local_replays_orig = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--let $galera_sp_bf_abort_proc = proc_update_insert_with_exit_handler +--let $galera_sp_bf_abort_conflict = INSERT INTO t1 VALUES (2, 'c') +--source galera_sp_bf_abort.inc +--connection node_1 +--reap +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +--let $wsrep_local_replays_curr = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_curr - $wsrep_local_replays_orig = 1 AS wsrep_local_replays; +--enable_query_log + +DELETE FROM t1; + +--connection node_1 +# +# Case 1c: Procedure does and UPDATE which will suffer BF abort +# but there is no actual conflict and non-conflicting INSERT. +# A CONTINUE HANDLER is declared for the procedure. +# The expected outcome is that both UPDATE and INSERT will succedd +# and no errors are reported to the client, wsrep_local_replays is +# incremented by one. +# +DELIMITER |; +CREATE PROCEDURE proc_update_insert_with_continue_handler() +BEGIN + + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + UPDATE t1 SET f2 = 'b'; + INSERT INTO t1 VALUES (4, 'd'); +END| +DELIMITER ;| + +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +SET SESSION wsrep_sync_wait = 0; +--let $wsrep_local_replays_orig = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--let $galera_sp_bf_abort_proc = proc_update_insert_with_continue_handler +--let $galera_sp_bf_abort_conflict = INSERT INTO t1 VALUES (2, 'c') +--source galera_sp_bf_abort.inc +--connection node_1 +--reap +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +--let $wsrep_local_replays_curr = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_curr - $wsrep_local_replays_orig = 1 AS wsrep_local_replays; +--enable_query_log + +DELETE FROM t1; + +--connection node_1 +# +# Case 2a: UPDATE and INSERT are run inside a transaction and the transaction +# will be BF aborted on COMMIT. The expected outcome is that the transaction +# succeeds and no errors are reported to the client, wsrep_local_replays +# is incremented by one. +# + +DELIMITER |; +CREATE PROCEDURE proc_update_insert_transaction() +BEGIN + START TRANSACTION; + UPDATE t1 SET f2 = 'b'; + INSERT INTO t1 VALUES (4, 'd'); + COMMIT; +END| +DELIMITER ;| + +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +--let $wsrep_local_replays_orig = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--let $galera_sp_bf_abort_proc = proc_update_insert_transaction +--let $galera_sp_bf_abort_conflict = INSERT INTO t1 VALUES (2, 'c') +SET SESSION wsrep_sync_wait = 0; +--source galera_sp_bf_abort.inc +--connection node_1 +--reap +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +--let $wsrep_local_replays_curr = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_curr - $wsrep_local_replays_orig = 1 AS wsrep_local_replays; +--enable_query_log + +DELETE FROM t1; + +--connection node_1 +# +# Case 2b: UPDATE and INSERT are run inside a transaction and the transaction +# will be BF aborted on COMMIT. A CONTINUE HANDLER is declared for the +# procedure. The expected outcome is that the transaction +# succeeds and no errors are reported to the client, wsrep_local_replays +# is incremented by one. +# + +DELIMITER |; +CREATE PROCEDURE proc_update_insert_transaction_with_continue_handler() +BEGIN + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + START TRANSACTION; + UPDATE t1 SET f2 = 'b'; + INSERT INTO t1 VALUES (4, 'd'); + COMMIT; +END| +DELIMITER ;| + +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +--let $wsrep_local_replays_orig = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--let $galera_sp_bf_abort_proc = proc_update_insert_transaction_with_continue_handler +--let $galera_sp_bf_abort_conflict = INSERT INTO t1 VALUES (2, 'c') +SET SESSION wsrep_sync_wait = 0; +--source galera_sp_bf_abort.inc +--connection node_1 +--reap +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +--let $wsrep_local_replays_curr = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_curr - $wsrep_local_replays_orig = 1 AS wsrep_local_replays; +--enable_query_log + +DELETE FROM t1; + +--connection node_1 +# +# Case 2c: UPDATE and INSERT are run inside a transaction and the transaction +# will be BF aborted on COMMIT. An EXIT HANDLE is declared for the procedure. +# The expected outcome is that the transaction succeeds and no errors are +# reported to the client, wsrep_local_replays is incremented by one. +# + +DELIMITER |; +CREATE PROCEDURE proc_update_insert_transaction_with_exit_handler() +BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN END; + START TRANSACTION; + UPDATE t1 SET f2 = 'b'; + INSERT INTO t1 VALUES (4, 'd'); + COMMIT; +END| +DELIMITER ;| + +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +--let $wsrep_local_replays_orig = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--let $galera_sp_bf_abort_proc = proc_update_insert_transaction_with_exit_handler +--let $galera_sp_bf_abort_conflict = INSERT INTO t1 VALUES (2, 'c') +SET SESSION wsrep_sync_wait = 0; +--source galera_sp_bf_abort.inc +--connection node_1 +--reap +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +--let $wsrep_local_replays_curr = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_curr - $wsrep_local_replays_orig = 1 AS wsrep_local_replays; +--enable_query_log + +DELETE FROM t1; + +--connection node_1 + +# +# Case 3a: Two INSERTs are run inside stored procedure, this time +# the first INSERT will have a BF abort and real conflict. The expected outcome +# is that the INSERT fails and an error is reported to the client. +# wsrep_local_replays is not incremented. +# +# Notice that the resulting error code may be both +# ER_DUP_ENTRY (procedure will exit with cert failure conflict state and +# will be) or ER_LOCK_DEADLOCK depending on timing. +# +DELIMITER |; +CREATE PROCEDURE proc_insert_insert_conflict() +BEGIN + INSERT INTO t1 VALUES (2, 'd'); + INSERT INTO t1 VALUES (4, 'd'); +END| +DELIMITER ;| + +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +--let $wsrep_local_replays_orig = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--let $galera_sp_bf_abort_proc = proc_insert_insert_conflict +--let $galera_sp_bf_abort_conflict = INSERT INTO t1 VALUES (2, 'c') +SET SESSION wsrep_sync_wait = 0; +--source galera_sp_bf_abort.inc +--connection node_1 +--error ER_DUP_ENTRY,ER_LOCK_DEADLOCK, ER_ERROR_DURING_COMMIT +--reap +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +--let $wsrep_local_replays_curr = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_curr - $wsrep_local_replays_orig = 0 AS wsrep_local_replays; +--enable_query_log + +DELETE FROM t1; + +--connection node_1 + +# +# Case 3b: Two INSERTs are run inside stored procedure, this time +# the first INSERT will have a BF abort and real conflict. +# An EXIT HANDLER is declared for the procedure. The expected outcome +# is that the INSERT fails and an error is reported to the client. +# wsrep_local_replays is not incremented. +# +DELIMITER |; +CREATE PROCEDURE proc_insert_insert_conflict_with_exit_handler() +BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION SELECT "Conflict exit handler"; + INSERT INTO t1 VALUES (2, 'd'); + INSERT INTO t1 VALUES (4, 'd'); +END| +DELIMITER ;| + +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +--let $wsrep_local_replays_orig = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--let $galera_sp_bf_abort_proc = proc_insert_insert_conflict_with_exit_handler +--let $galera_sp_bf_abort_conflict = INSERT INTO t1 VALUES (2, 'c') +SET SESSION wsrep_sync_wait = 0; +--source galera_sp_bf_abort.inc +--connection node_1 +--reap +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +--let $wsrep_local_replays_curr = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_curr - $wsrep_local_replays_orig = 0 AS wsrep_local_replays; +--enable_query_log + +DELETE FROM t1; + +--connection node_1 + +# +# Case 3c: Two INSERTs are run inside stored procedure, this time +# the first INSERT will have a BF abort and real conflict. +# A CONTINUE HANDLER is declared for the procedure. The expected outcome +# is that the the first INSERT fails but the second is executed without +# errors. wsrep_local_replays is not incremented. +# +DELIMITER |; +CREATE PROCEDURE proc_insert_insert_conflict_with_continue_handler() +BEGIN + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION SELECT "Conflict continue handler"; + INSERT INTO t1 VALUES (2, 'd'); + INSERT INTO t1 VALUES (4, 'd'); +END| +DELIMITER ;| + +INSERT INTO t1 VALUES (1, 'a'), (3, 'a'); +--let $wsrep_local_replays_orig = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--let $galera_sp_bf_abort_proc = proc_insert_insert_conflict_with_continue_handler +--let $galera_sp_bf_abort_conflict = INSERT INTO t1 VALUES (2, 'c') +SET SESSION wsrep_sync_wait = 0; +--source galera_sp_bf_abort.inc +--connection node_1 +--reap +SET SESSION wsrep_sync_wait = default; +SELECT * FROM t1; +--let $wsrep_local_replays_curr = `SELECT VARIABLE_VALUE FROM information_schema.global_status WHERE VARIABLE_NAME = 'wsrep_local_replays'` +--disable_query_log +--eval SELECT $wsrep_local_replays_curr - $wsrep_local_replays_orig = 0 AS wsrep_local_replays; +--enable_query_log + +DELETE FROM t1; + + +DROP PROCEDURE proc_update_insert; +DROP PROCEDURE proc_update_insert_with_continue_handler; +DROP PROCEDURE proc_update_insert_with_exit_handler; +DROP PROCEDURE proc_update_insert_transaction; +DROP PROCEDURE proc_update_insert_transaction_with_continue_handler; +DROP PROCEDURE proc_update_insert_transaction_with_exit_handler; +DROP PROCEDURE proc_insert_insert_conflict; +DROP PROCEDURE proc_insert_insert_conflict_with_exit_handler; +DROP PROCEDURE proc_insert_insert_conflict_with_continue_handler; +DROP TABLE t1; diff --git a/sql/sp_head.cc b/sql/sp_head.cc index f940040b480..0428c0198a1 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -44,6 +44,9 @@ #include "transaction.h" // trans_commit_stmt #include "sql_audit.h" #include "debug_sync.h" +#ifdef WITH_WSREP +#include "wsrep_thd.h" +#endif /* WITH_WSREP */ /* Sufficient max length of printed destinations and frame offsets (all uints). @@ -1307,7 +1310,93 @@ sp_head::execute(THD *thd, bool merge_da_on_success) thd->m_digest= NULL; err_status= i->execute(thd, &ip); +#ifdef WITH_WSREP + if (m_type == TYPE_ENUM_PROCEDURE) + { + mysql_mutex_lock(&thd->LOCK_thd_data); + if (thd->wsrep_conflict_state == MUST_REPLAY) + { + wsrep_replay_sp_transaction(thd); + err_status= thd->get_stmt_da()->is_set(); + thd->wsrep_conflict_state= NO_CONFLICT; + } + else if (thd->wsrep_conflict_state == ABORTED || + thd->wsrep_conflict_state == CERT_FAILURE) + { + /* + If the statement execution was BF aborted or was aborted + due to certification failure, clean up transaction here + and reset conflict state to NO_CONFLICT and thd->killed + to THD::NOT_KILLED. Error handling is done based on err_status + below. Error must have been raised by wsrep hton code before + entering here. + */ + DBUG_ASSERT(err_status); + DBUG_ASSERT(thd->get_stmt_da()->is_error()); + thd->wsrep_conflict_state= NO_CONFLICT; + thd->killed= NOT_KILLED; + } + mysql_mutex_unlock(&thd->LOCK_thd_data); + } +#endif /* WITH_WSREP */ +#ifdef WITH_WSREP_NO + if (WSREP(thd)) + { + if (((thd->wsrep_trx().state() == wsrep::transaction::s_executing) && + (thd->is_fatal_error || thd->killed))) + { + WSREP_DEBUG("SP abort err status %d in sub %d trx state %d", + err_status, thd->in_sub_stmt, thd->wsrep_trx().state()); + err_status= 1; + thd->is_fatal_error= 1; + /* + SP was killed, and it is not due to a wsrep conflict. + We skip after_command hook at this point because + otherwise it clears the error, and cleans up the + whole transaction. For now we just return and finish + our handling once we are back to mysql_parse. + */ + WSREP_DEBUG("Skipping after_command hook for killed SP"); + } + else + { + const bool must_replay= wsrep_must_replay(thd); + if (must_replay) + { + WSREP_DEBUG("MUST_REPLAY set after SP, err_status %d trx state: %d", + err_status, thd->wsrep_trx().state()); + } + (void) wsrep_after_statement(thd); + /* + Reset the return code to zero if the transaction was + replayed succesfully. + */ + if (must_replay && !wsrep_current_error(thd)) + { + err_status= 0; + thd->get_stmt_da()->reset_diagnostics_area(); + } + /* + Final wsrep error status for statement is known only after + wsrep_after_statement() call. If the error is set, override + error in thd diagnostics area and reset wsrep client_state error + so that the error does not get propagated via client-server protocol. + */ + if (wsrep_current_error(thd)) + { + wsrep_override_error(thd, wsrep_current_error(thd), + wsrep_current_error_status(thd)); + thd->wsrep_cs().reset_error(); + /* Reset also thd->killed if it has been set during BF abort. */ + if (thd->killed == KILL_QUERY) + thd->killed= NOT_KILLED; + /* if failed transaction was not replayed, must return with error from here */ + if (!must_replay) err_status = 1; + } + } + } +#endif /* WITH_WSREP */ thd->m_digest= parent_digest; if (i->free_list) diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index c4d348ce400..2972ceecd8a 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -34,7 +34,9 @@ #include "sql_handler.h" // mysql_ha_rm_tables #include "sp_cache.h" // sp_invalidate_cache #include - +#ifdef WITH_WSREP +#include "debug_sync.h" +#endif /* WITH_WSREP */ /*************************************************************************/ template @@ -506,7 +508,7 @@ bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create) #ifdef WITH_WSREP if (thd->wsrep_exec_mode == LOCAL_STATE) - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, tables); #endif /* We should have only one table in table list. */ @@ -568,6 +570,16 @@ bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create) goto end; } +#ifdef WITH_WSREP + DBUG_EXECUTE_IF("sync.mdev_20225", + { + const char act[]= + "now " + "wait_for signal.mdev_20225_continue"; + DBUG_ASSERT(!debug_sync_set_action(thd, + STRING_WITH_LEN(act))); + };); +#endif /* WITH_WSREP */ result= (create ? table->triggers->create_trigger(thd, tables, &stmt_query): table->triggers->drop_trigger(thd, tables, &stmt_query)); diff --git a/sql/wsrep_hton.cc b/sql/wsrep_hton.cc index d8f82b13108..030c4244a30 100644 --- a/sql/wsrep_hton.cc +++ b/sql/wsrep_hton.cc @@ -37,6 +37,8 @@ extern "C" int thd_binlog_format(const MYSQL_THD thd); void wsrep_cleanup_transaction(THD *thd) { if (!WSREP(thd)) return; + DBUG_ASSERT(thd->wsrep_conflict_state != MUST_REPLAY && + thd->wsrep_conflict_state != REPLAYING); if (wsrep_emulate_bin_log) thd_binlog_trx_reset(thd); thd->wsrep_ws_handle.trx_id= WSREP_UNDEFINED_TRX_ID; @@ -136,7 +138,11 @@ void wsrep_post_commit(THD* thd, bool all) /* non-InnoDB statements may have populated events in stmt cache => cleanup */ - WSREP_DEBUG("cleanup transaction for LOCAL_STATE"); + if (thd->wsrep_conflict_state != MUST_REPLAY) + { + WSREP_DEBUG("cleanup transaction for LOCAL_STATE: %s", + WSREP_QUERY(thd)); + } /* Run post-rollback hook to clean up in the case if some keys were populated for the transaction in provider @@ -145,13 +151,18 @@ void wsrep_post_commit(THD* thd, bool all) rolls back to savepoint after first operation. */ if (all && thd->wsrep_conflict_state != MUST_REPLAY && - wsrep && wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) + thd->wsrep_conflict_state != REPLAYING && + wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) { WSREP_WARN("post_rollback fail: %llu %d", (long long)thd->thread_id, thd->get_stmt_da()->status()); } - wsrep_cleanup_transaction(thd); - break; + if (thd->wsrep_conflict_state != MUST_REPLAY && + thd->wsrep_conflict_state != REPLAYING) + { + wsrep_cleanup_transaction(thd); + } + break; } default: break; } @@ -575,7 +586,8 @@ wsrep_run_wsrep_commit(THD *thd, bool all) DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED); /* fall through */ case WSREP_TRX_FAIL: - WSREP_DEBUG("commit failed for reason: %d", rcode); + WSREP_DEBUG("commit failed for reason: %d conf %d", + rcode, thd->wsrep_conflict_state); DBUG_PRINT("wsrep", ("replicating commit fail")); thd->wsrep_query_state= QUERY_EXEC; diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index f38bf85cd1a..38f8ca413db 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -1570,7 +1570,6 @@ static bool wsrep_can_run_in_toi(THD *thd, const char *db, const char *table, case SQLCOM_CREATE_TRIGGER: - DBUG_ASSERT(!table_list); DBUG_ASSERT(first_table); if (find_temporary_table(thd, first_table)) @@ -1579,6 +1578,14 @@ static bool wsrep_can_run_in_toi(THD *thd, const char *db, const char *table, } return true; + case SQLCOM_DROP_TRIGGER: + DBUG_ASSERT(table_list); + if (find_temporary_table(thd, table_list)) + { + return false; + } + return true; + default: if (table && !find_temporary_table(thd, db, table)) { diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc index fad9e3f70c8..d37af157783 100644 --- a/sql/wsrep_thd.cc +++ b/sql/wsrep_thd.cc @@ -90,10 +90,10 @@ void wsrep_client_rollback(THD *thd) #define NUMBER_OF_FIELDS_TO_IDENTIFY_COORDINATOR 1 #define NUMBER_OF_FIELDS_TO_IDENTIFY_WORKER 2 -static rpl_group_info* wsrep_relay_group_init(const char* log_fname) +static rpl_group_info* wsrep_relay_group_init(THD *thd, const char* log_fname) { Relay_log_info* rli= new Relay_log_info(false); - + WSREP_DEBUG("wsrep_relay_group_init %s", log_fname); rli->no_storage= true; if (!rli->relay_log.description_event_for_exec) { @@ -123,7 +123,7 @@ static rpl_group_info* wsrep_relay_group_init(const char* log_fname) rli->mi = new Master_info(&connection_name, false); struct rpl_group_info *rgi= new rpl_group_info(rli); - rgi->thd= rli->sql_driver_thd= current_thd; + rgi->thd= rli->sql_driver_thd= thd; if ((rgi->deferred_events_collecting= rli->mi->rpl_filter->is_on())) { @@ -148,7 +148,7 @@ static void wsrep_prepare_bf_thd(THD *thd, struct wsrep_thd_shadow* shadow) else thd->variables.option_bits&= ~(OPTION_BIN_LOG); - if (!thd->wsrep_rgi) thd->wsrep_rgi= wsrep_relay_group_init("wsrep_relay"); + if (!thd->wsrep_rgi) thd->wsrep_rgi= wsrep_relay_group_init(thd, "wsrep_relay"); /* thd->system_thread_info.rpl_sql_info isn't initialized. */ thd->system_thread_info.rpl_sql_info= @@ -189,6 +189,109 @@ static void wsrep_return_from_bf_mode(THD *thd, struct wsrep_thd_shadow* shadow) thd->set_row_count_func(shadow->row_count_func); } +void wsrep_replay_sp_transaction(THD* thd) +{ + DBUG_ENTER("wsrep_replay_sp_transaction"); + mysql_mutex_assert_owner(&thd->LOCK_thd_data); + DBUG_ASSERT(thd->wsrep_conflict_state == MUST_REPLAY); + DBUG_ASSERT(wsrep_thd_trx_seqno(thd) > 0); + + WSREP_DEBUG("replaying SP transaction %llu", thd->thread_id); + close_thread_tables(thd); + if (thd->locked_tables_mode && thd->lock) + { + WSREP_DEBUG("releasing table lock for replaying (%u)", + thd->thread_id); + thd->locked_tables_list.unlock_locked_tables(thd); + thd->variables.option_bits&= ~(OPTION_TABLE_LOCK); + } + thd->mdl_context.release_transactional_locks(); + + mysql_mutex_unlock(&thd->LOCK_thd_data); + THD *replay_thd= new THD(true); + replay_thd->thread_stack= thd->thread_stack; + + struct wsrep_thd_shadow shadow; + wsrep_prepare_bf_thd(replay_thd, &shadow); + WSREP_DEBUG("replaying set for %p rgi %p", replay_thd, replay_thd->wsrep_rgi); replay_thd->wsrep_trx_meta= thd->wsrep_trx_meta; + replay_thd->wsrep_ws_handle= thd->wsrep_ws_handle; + replay_thd->wsrep_ws_handle.trx_id= WSREP_UNDEFINED_TRX_ID; + replay_thd->wsrep_conflict_state= REPLAYING; + + replay_thd->variables.option_bits|= OPTION_BEGIN; + replay_thd->server_status|= SERVER_STATUS_IN_TRANS; + + thd->reset_globals(); + replay_thd->store_globals(); + wsrep_status_t rcode= wsrep->replay_trx(wsrep, + &replay_thd->wsrep_ws_handle, + (void*) replay_thd); + + wsrep_return_from_bf_mode(replay_thd, &shadow); + replay_thd->reset_globals(); + delete replay_thd; + + mysql_mutex_lock(&thd->LOCK_thd_data); + + thd->store_globals(); + + switch (rcode) + { + case WSREP_OK: + { + thd->wsrep_conflict_state= NO_CONFLICT; + thd->killed= NOT_KILLED; + wsrep_status_t rcode= wsrep->post_commit(wsrep, &thd->wsrep_ws_handle); + if (rcode != WSREP_OK) + { + WSREP_WARN("Post commit failed for SP replay: thd: %u error: %d", + thd->thread_id, rcode); + } + /* As replaying the transaction was successful, an error must not + be returned to client, so we need to reset the error state of + the diagnostics area */ + thd->get_stmt_da()->reset_diagnostics_area(); + break; + } + case WSREP_TRX_FAIL: + { + thd->wsrep_conflict_state= ABORTED; + wsrep_status_t rcode= wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle); + if (rcode != WSREP_OK) + { + WSREP_WARN("Post rollback failed for SP replay: thd: %u error: %d", + thd->thread_id, rcode); + } + if (thd->get_stmt_da()->is_set()) + { + thd->get_stmt_da()->reset_diagnostics_area(); + } + my_error(ER_LOCK_DEADLOCK, MYF(0)); + break; + } + default: + WSREP_ERROR("trx_replay failed for: %d, schema: %s, query: %s", + rcode, + (thd->db ? thd->db : "(null)"), + WSREP_QUERY(thd)); + /* we're now in inconsistent state, must abort */ + mysql_mutex_unlock(&thd->LOCK_thd_data); + unireg_abort(1); + break; + } + + wsrep_cleanup_transaction(thd); + + mysql_mutex_lock(&LOCK_wsrep_replaying); + wsrep_replaying--; + WSREP_DEBUG("replaying decreased: %d, thd: %u", + wsrep_replaying, thd->thread_id); + mysql_cond_broadcast(&COND_wsrep_replaying); + mysql_mutex_unlock(&LOCK_wsrep_replaying); + + DBUG_VOID_RETURN; +} + void wsrep_replay_transaction(THD *thd) { DBUG_ENTER("wsrep_replay_transaction"); diff --git a/sql/wsrep_thd.h b/sql/wsrep_thd.h index 6ce14a4eb0e..8e82a0fbe21 100644 --- a/sql/wsrep_thd.h +++ b/sql/wsrep_thd.h @@ -25,6 +25,7 @@ int wsrep_show_bf_aborts (THD *thd, SHOW_VAR *var, char *buff, enum enum_var_type scope); void wsrep_client_rollback(THD *thd); +void wsrep_replay_sp_transaction(THD* thd); void wsrep_replay_transaction(THD *thd); void wsrep_create_appliers(long threads); void wsrep_create_rollbacker(); From 21e79331c8c89e397d5a1ca4a4b8a70ad7c0377a Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 13 Jun 2020 16:45:55 +0300 Subject: [PATCH 03/73] MDEV-22779: Crash: Prepared Statement with a '?' parameter inside a re-used CTE When a prepared statement parameter '?' is used in a CTE that is used multiple times, the following happens: - The CTE definition is re-parsed multiple times. - There are multiple Item_param objects referring to the same "?" in the original query. - Prepared_statement::param has a pointer to the first of them, the others are "clones". - When prepared statement parameter gets the value, it should be passed over to clones with param->sync_clones() call. This call is made in insert_params(), etc. It was not made in insert_params_with_log(). This would cause Item_param to not have any value which would confuse the query optimizer. Added the missing call. --- sql/sql_prepare.cc | 2 ++ sql/sql_select.cc | 1 + tests/mysql_client_test.c | 52 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 6ac85f1331e..79d18fcb799 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -903,6 +903,8 @@ static bool insert_params_with_log(Prepared_statement *stmt, uchar *null_array, if (param->convert_str_value(thd)) DBUG_RETURN(1); /* out of memory */ + + param->sync_clones(); } if (acc.finalize()) DBUG_RETURN(1); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b558445540d..4cca2d67eb8 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -5997,6 +5997,7 @@ static void optimize_keyuse(JOIN *join, DYNAMIC_ARRAY *keyuse_array) uint n_tables= my_count_bits(map); if (n_tables == 1) // Only one table { + DBUG_ASSERT(!(map & PSEUDO_TABLE_BITS)); // Must be a real table Table_map_iterator it(map); int tablenr= it.next_bit(); DBUG_ASSERT(tablenr != Table_map_iterator::BITMAP_END); diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index c544e20f2fe..33655b80662 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -19843,6 +19843,57 @@ static void test_bulk_replace() } #endif + +static void test_ps_params_in_ctes() +{ + int rc; + const char *query; + MYSQL_BIND ps_params[1]; + int int_data[1]; + MYSQL_STMT *stmt; + + rc= mysql_query(mysql, "create table t1(a int, b int, key(a))"); + myquery(rc); + + rc= mysql_query(mysql, "insert into t1 (a) values " + "(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)"); + myquery(rc); + + query= + "explain " + "with T as " + "( " + " select * from t1 where t1.a=? limit 2 " + ") " + "select * from T as TA, T as TB;"; + + stmt= mysql_stmt_init(mysql); + check_stmt(stmt); + + rc= mysql_stmt_prepare(stmt, query, (uint) strlen(query)); + check_execute(stmt, rc); + + int_data[0]=2; + + ps_params[0].buffer_type= MYSQL_TYPE_LONG; + ps_params[0].buffer= (char *) &int_data[0]; + ps_params[0].length= 0; + ps_params[0].is_null= 0; + + rc= mysql_stmt_bind_param(stmt, ps_params); + check_execute(stmt, rc); + + rc= mysql_stmt_execute(stmt); + check_execute(stmt, rc); + + rc= mysql_stmt_store_result(stmt); + check_execute(stmt, rc); + + rc= mysql_query(mysql, "drop table t1"); + myquery(rc); +} + + static struct my_tests_st my_tests[]= { { "disable_query_logs", disable_query_logs }, { "test_view_sp_list_fields", test_view_sp_list_fields }, @@ -20127,6 +20178,7 @@ static struct my_tests_st my_tests[]= { { "test_bulk_delete", test_bulk_delete }, { "test_bulk_replace", test_bulk_replace }, #endif + { "test_ps_params_in_ctes", test_ps_params_in_ctes }, { 0, 0 } }; From e623d247872c1736029cffb0b7ff055a698900c6 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 13 Jun 2020 23:28:09 +0300 Subject: [PATCH 04/73] MDEV-22779: Crash: Prepared Statement ..., part #2. For the sake of completeness, call sync_clones in reset_stmt_params, too. --- sql/sql_prepare.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 79d18fcb799..f0c9f818f87 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -3125,7 +3125,10 @@ static void reset_stmt_params(Prepared_statement *stmt) Item_param **item= stmt->param_array; Item_param **end= item + stmt->param_count; for (;item < end ; ++item) + { (**item).reset(); + (**item).sync_clones(); + } } From 7710f28eecc3e9761d7431ae7ae4e88564d706dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 15 Jun 2020 09:29:17 +0300 Subject: [PATCH 05/73] Add missing include as test requires galera debug library --- mysql-test/suite/galera/t/MW-388.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/suite/galera/t/MW-388.test b/mysql-test/suite/galera/t/MW-388.test index 40522f30abb..670674bbae5 100644 --- a/mysql-test/suite/galera/t/MW-388.test +++ b/mysql-test/suite/galera/t/MW-388.test @@ -1,5 +1,7 @@ --source include/galera_cluster.inc +--source include/have_debug.inc --source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc --connection node_1 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(255)) Engine=InnoDB; From 93cee30309588b62312debba714eec06ea5b2063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Mon, 15 Jun 2020 16:01:41 +0300 Subject: [PATCH 06/73] Check for krb5_xfree instead of krb5_free_unparsed_name Use krb5_xfree if available, otherwise default to krb5_free_unparsed_name. --- plugin/auth_gssapi/CMakeLists.txt | 6 +++--- plugin/auth_gssapi/gssapi_server.cc | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugin/auth_gssapi/CMakeLists.txt b/plugin/auth_gssapi/CMakeLists.txt index 7d9e58e165f..c708606856d 100644 --- a/plugin/auth_gssapi/CMakeLists.txt +++ b/plugin/auth_gssapi/CMakeLists.txt @@ -21,9 +21,9 @@ ELSE() SET(CMAKE_REQUIRED_INCLUDES ${GSSAPI_INCS}) SET(CMAKE_REQUIRED_LIBRARIES ${GSSAPI_LIBS}) INCLUDE(CheckCXXSymbolExists) - CHECK_CXX_SYMBOL_EXISTS(krb5_free_unparsed_name "krb5.h" HAVE_KRB5_FREE_UNPARSED_NAME) - IF(HAVE_KRB5_FREE_UNPARSED_NAME) - ADD_DEFINITIONS(-DHAVE_KRB5_FREE_UNPARSED_NAME=1) + CHECK_CXX_SYMBOL_EXISTS(krb5_xfree "krb5.h" HAVE_KRB5_XFREE) + IF(HAVE_KRB5_XFREE) + ADD_DEFINITIONS(-DHAVE_KRB5_XFREE=1) ENDIF() ELSE() diff --git a/plugin/auth_gssapi/gssapi_server.cc b/plugin/auth_gssapi/gssapi_server.cc index 50c34ecc573..c1c4fa13e00 100644 --- a/plugin/auth_gssapi/gssapi_server.cc +++ b/plugin/auth_gssapi/gssapi_server.cc @@ -31,7 +31,7 @@ static void log_error( OM_uint32 major, OM_uint32 minor, const char *msg) Generate default principal service name formatted as principal name "mariadb/server.fqdn@REALM" */ #include -#ifndef HAVE_KRB5_FREE_UNPARSED_NAME +#ifdef HAVE_KRB5_XFREE #define krb5_free_unparsed_name(a,b) krb5_xfree(b) #endif static char* get_default_principal_name() From bf74f7f9ff40f8a17e739e9ab6f5c906ccb9e892 Mon Sep 17 00:00:00 2001 From: Sujatha Date: Mon, 15 Jun 2020 15:57:01 +0530 Subject: [PATCH 07/73] MDEV-20428: "Start binlog_dump" message doesn't indicate GTID position Problem: ======= The "Start binlog_dump" message hasn't been updated to include the slave's requested GTID position: 20:05:05 139836760311552 [Note] Start binlog_dump to slave_server(2), pos(, 4) For diagnostic purposes, it would be helpful if the GTID position were included. Fix: === Imporve "Start binlog_dump" print message to include "using_gtid" and "GTID position" requested by slave. Ex: [Note] Start binlog_dump to slave_server(2), pos(, 4), using_gtid(1), gtid('1-1-201,2-2-100') [Note] Start binlog_dump to slave_server(3), pos('mariadb-bin.004142', 507988273), using_gtid(0), gtid('') --- ...l_binlog_dump_slave_gtid_state_info.result | 34 +++++ ...rpl_binlog_dump_slave_gtid_state_info.test | 121 ++++++++++++++++++ sql/sql_repl.cc | 8 +- 3 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_binlog_dump_slave_gtid_state_info.result create mode 100644 mysql-test/suite/rpl/t/rpl_binlog_dump_slave_gtid_state_info.test diff --git a/mysql-test/suite/rpl/r/rpl_binlog_dump_slave_gtid_state_info.result b/mysql-test/suite/rpl/r/rpl_binlog_dump_slave_gtid_state_info.result new file mode 100644 index 00000000000..87ed91ced01 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_binlog_dump_slave_gtid_state_info.result @@ -0,0 +1,34 @@ +include/master-slave.inc +[connection master] +SET GLOBAL LOG_WARNINGS=2; +include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=current_pos; +include/start_slave.inc +"Test Case 1: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('')" +FOUND /using_gtid\(1\), gtid\(\'\'\)/ in mysqld.1.err +include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=no; +include/start_slave.inc +"Test Case 2: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(0), gtid('')" +FOUND /using_gtid\(0\), gtid\(\'\'\)/ in mysqld.1.err +CREATE TABLE t (f INT) ENGINE=INNODB; +INSERT INTO t VALUES(10); +include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=slave_pos; +include/start_slave.inc +"Test Case 3: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('0-1-2')" +FOUND /using_gtid\(1\), gtid\(\'0-1-2\'\)/ in mysqld.1.err +SET @@SESSION.gtid_domain_id=10; +INSERT INTO t VALUES(20); +include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=slave_pos; +include/start_slave.inc +"Test Case 4: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('0-1-2,10-1-1')" +FOUND /using_gtid\(1\), gtid\(\'0-1-2,10-1-1\'\)/ in mysqld.1.err +"===== Clean up =====" +include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=no; +include/start_slave.inc +DROP TABLE t; +SET GLOBAL LOG_WARNINGS=default; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_binlog_dump_slave_gtid_state_info.test b/mysql-test/suite/rpl/t/rpl_binlog_dump_slave_gtid_state_info.test new file mode 100644 index 00000000000..e2cc4b002cc --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_binlog_dump_slave_gtid_state_info.test @@ -0,0 +1,121 @@ +# ==== Purpose ==== +# +# Test verifies that Start binlog_dump message will report GTID position +# requested by slave when log_warnings > 1. +# +# ==== Implementation ==== +# +# Steps: +# 0 - Have LOG_WARNINGS=2 +# 1 - On a fresh slave server which has not replicated any GTIDs execute +# "CHANGE MASTER TO MASTER_USE_GTID=current_pos;" command. Start the +# slave. +# 2 - In Master error log verify that pattern "using_gtid(1), gtid('')" is +# present. +# 3 - On slave server do STOP SLAVE and execute "CHANGE MASTER TO +# MASTER_USE_GTID=no;" command. Start the slave threads. +# 4 - In Master error log verify that pattern "using_gtid(0), gtid('')" is +# present. +# 5- Execute a DDL and DML on master server. This will generated two GTIDs +# on the master server ('0-1-2'). Sync the slave server with master. +# 6 - On slave do STOP SLAVE and execute "CHANGE MASTER TO +# MASTER_USE_GTID=slave_pos;" command. Start slave threads. +# 7 - In Master error verify that pattern "using_gtid(1), gtid('0-1-2')" is +# present. +# 8 - On Master change domain ID to 10 and execute a DML operation. It will +# generate a GTID 10-1-1. +# 9 - On slave do STOP SLAVE and execute "CHANGE MASTER TO +# MASTER_USE_GTID=slave_pos;" command. Start slave threads. +# 10 -In Master error verify that pattern "using_gtid(1), +# gtid('0-1-2,10-1-1')" is present. +# +# ==== References ==== +# +# MDEV-20428: "Start binlog_dump" message doesn't indicate GTID position +# + +--source include/have_binlog_format_mixed.inc +--source include/have_innodb.inc +--source include/master-slave.inc + +--connection master +SET GLOBAL LOG_WARNINGS=2; + +--connection slave +--source include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=current_pos; +--source include/start_slave.inc + +--connection master +# Check error log for correct messages. +let $log_error_= `SELECT @@GLOBAL.log_error`; +if(!$log_error_) +{ + # MySQL Server on windows is started with --console and thus + # does not know the location of its .err log, use default location + let $log_error_ = $MYSQLTEST_VARDIR/log/mysqld.1.err; +} +--echo "Test Case 1: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('')" +--let SEARCH_FILE=$log_error_ +--let SEARCH_RANGE=-50000 +--let SEARCH_PATTERN=using_gtid\(1\), gtid\(\'\'\) +--source include/search_pattern_in_file.inc + +--connection slave +--source include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=no; +--source include/start_slave.inc + +--connection master +--echo "Test Case 2: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(0), gtid('')" +--let SEARCH_FILE=$log_error_ +--let SEARCH_RANGE=-50000 +--let SEARCH_PATTERN=using_gtid\(0\), gtid\(\'\'\) +--source include/search_pattern_in_file.inc +CREATE TABLE t (f INT) ENGINE=INNODB; +INSERT INTO t VALUES(10); +save_master_pos; + +--connection slave +sync_with_master; + +--connection slave +--source include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=slave_pos; +--source include/start_slave.inc + +--connection master +--echo "Test Case 3: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('0-1-2')" +--let SEARCH_FILE=$log_error_ +--let SEARCH_RANGE=-50000 +--let SEARCH_PATTERN=using_gtid\(1\), gtid\(\'0-1-2\'\) +--source include/search_pattern_in_file.inc +SET @@SESSION.gtid_domain_id=10; +INSERT INTO t VALUES(20); +save_master_pos; + +--connection slave +sync_with_master; + +--connection slave +--source include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=slave_pos; +--source include/start_slave.inc + +--connection master +--echo "Test Case 4: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('0-1-2,10-1-1')" +--let SEARCH_FILE=$log_error_ +--let SEARCH_RANGE=-50000 +--let SEARCH_PATTERN=using_gtid\(1\), gtid\(\'0-1-2,10-1-1\'\) +--source include/search_pattern_in_file.inc + +--echo "===== Clean up =====" +--connection slave +--source include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=no; +--source include/start_slave.inc + +--connection master +DROP TABLE t; +SET GLOBAL LOG_WARNINGS=default; +--source include/rpl_end.inc diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 851bff5fd24..38c2b9b5b8e 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2027,9 +2027,13 @@ static int init_binlog_sender(binlog_send_info *info, }); if (global_system_variables.log_warnings > 1) + { sql_print_information( - "Start binlog_dump to slave_server(%lu), pos(%s, %lu)", - thd->variables.server_id, log_ident, (ulong)*pos); + "Start binlog_dump to slave_server(%lu), pos(%s, %lu), " + "using_gtid(%d), gtid('%s')", thd->variables.server_id, + log_ident, (ulong)*pos, info->using_gtid_state, + connect_gtid_state.c_ptr_quick()); + } #ifndef DBUG_OFF if (opt_sporadic_binlog_dump_fail && (binlog_dump_count++ % 2)) From 29a5829c37071e10c85fa4d8be9b62f25b27d8cc Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 16 Jun 2020 12:27:09 +0530 Subject: [PATCH 08/73] MDEV-22901 Accessing btr_search_sys->hash_tables when buffer pool resize happens - btr_get_search_table() is accessing btr_search_sys->hash_tables, which is not safe if the buffer pool resize happens concurrently. --- storage/innobase/btr/btr0sea.cc | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index f24c143c24a..90b5ab31c64 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1485,20 +1485,12 @@ btr_search_build_page_hash_index( btr_search_check_free_space_in_heap(index); - hash_table_t* table = btr_get_search_table(index); rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto exit_func; } - table = btr_get_search_table(index); - if (block->index && ((block->curr_n_fields != n_fields) - || (block->curr_n_bytes != n_bytes) - || (block->curr_left_side != left_side))) { - goto exit_func; - } - /* This counter is decremented every time we drop page hash index entries and is incremented here. Since we can rebuild hash index for a page that is already hashed, we @@ -1507,6 +1499,10 @@ btr_search_build_page_hash_index( if (!block->index) { assert_block_ahi_empty(block); index->search_info->ref_count++; + } else if (block->curr_n_fields != n_fields + || block->curr_n_bytes != n_bytes + || block->curr_left_side != left_side) { + goto exit_func; } block->n_hash_helps = 0; @@ -1516,9 +1512,11 @@ btr_search_build_page_hash_index( block->curr_left_side = unsigned(left_side); block->index = index; - for (i = 0; i < n_cached; i++) { - - ha_insert_for_fold(table, folds[i], block, recs[i]); + { + hash_table_t* table = btr_get_search_table(index); + for (i = 0; i < n_cached; i++) { + ha_insert_for_fold(table, folds[i], block, recs[i]); + } } MONITOR_INC(MONITOR_ADAPTIVE_HASH_PAGE_ADDED); From 9f37323f063d4efbbab4a235d1e2417d7a646053 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 16 Jun 2020 12:30:04 +0530 Subject: [PATCH 09/73] MDEV-22901 Accessing btr_search_sys->hash_tables when buffer pool resize happens btr_search_build_page_hash_index(): Reduce the scope of the variable 'i' --- storage/innobase/btr/btr0sea.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 90b5ab31c64..32f5ae672e8 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1348,7 +1348,6 @@ btr_search_build_page_hash_index( ulint n_recs; ulint* folds; const rec_t** recs; - ulint i; mem_heap_t* heap = NULL; rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; rec_offs* offsets = offsets_; @@ -1514,7 +1513,7 @@ btr_search_build_page_hash_index( { hash_table_t* table = btr_get_search_table(index); - for (i = 0; i < n_cached; i++) { + for (ulint i = 0; i < n_cached; i++) { ha_insert_for_fold(table, folds[i], block, recs[i]); } } From b633b6a9d8278a3e0d9baaf83264fd007820c918 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Tue, 16 Jun 2020 10:43:53 +0300 Subject: [PATCH 10/73] MDEV-22906 Disallow system_versioning_asof in DML system_versioning_asof does not influence on multi-delete, multi-update, insert-select, replace-select. --- mysql-test/suite/versioning/r/sysvars.result | 29 ++++++++++++++++++++ mysql-test/suite/versioning/t/sysvars.test | 24 ++++++++++++++++ sql/sql_select.cc | 5 +++- 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/versioning/r/sysvars.result b/mysql-test/suite/versioning/r/sysvars.result index 9cb4911b321..cd858d6cf69 100644 --- a/mysql-test/suite/versioning/r/sysvars.result +++ b/mysql-test/suite/versioning/r/sysvars.result @@ -146,3 +146,32 @@ show status like "Feature_system_versioning"; Variable_name Value Feature_system_versioning 2 drop table t; +# +# MDEV-22906 Disallow system_versioning_asof in DML +# +create or replace table t1 (x int) with system versioning; +create or replace table t2 (y int); +insert into t1 values (1); +insert into t2 values (1); +set system_versioning_asof= '1970-01-01 00:00:00'; +delete t1, t2 from t1 join t2 where t1.x = t2.y; +select * from t1 for system_time as of timestamp now(6); +x +insert into t1 values (1); +insert into t2 values (1); +update t1, t2 set x= 2, y= 2 where x = y; +select * from t1 for system_time as of timestamp now(6); +x +2 +replace t2 select x + 1 from t1; +select * from t2; +y +2 +3 +insert t2 select x + 2 from t1; +select * from t2; +y +2 +3 +4 +drop tables t1, t2; diff --git a/mysql-test/suite/versioning/t/sysvars.test b/mysql-test/suite/versioning/t/sysvars.test index 1999ddd13fc..2f62262fc8e 100644 --- a/mysql-test/suite/versioning/t/sysvars.test +++ b/mysql-test/suite/versioning/t/sysvars.test @@ -103,3 +103,27 @@ select * from t for system_time between '1970-01-01 00:00' and current_timestamp show status like "Feature_system_versioning"; drop table t; + +--echo # +--echo # MDEV-22906 Disallow system_versioning_asof in DML +--echo # +create or replace table t1 (x int) with system versioning; +create or replace table t2 (y int); +insert into t1 values (1); +insert into t2 values (1); +set system_versioning_asof= '1970-01-01 00:00:00'; +delete t1, t2 from t1 join t2 where t1.x = t2.y; +select * from t1 for system_time as of timestamp now(6); + +insert into t1 values (1); +insert into t2 values (1); +update t1, t2 set x= 2, y= 2 where x = y; +select * from t1 for system_time as of timestamp now(6); + +replace t2 select x + 1 from t1; +select * from t2; + +insert t2 select x + 2 from t1; +select * from t2; + +drop tables t1, t2; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 05da4a1e750..1f35754a8fe 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -778,9 +778,12 @@ int SELECT_LEX::vers_setup_conds(THD *thd, TABLE_LIST *tables) } bool is_select= false; + bool use_sysvar= false; switch (thd->lex->sql_command) { case SQLCOM_SELECT: + use_sysvar= true; + /* fall through */ case SQLCOM_INSERT_SELECT: case SQLCOM_REPLACE_SELECT: case SQLCOM_DELETE_MULTI: @@ -824,7 +827,7 @@ int SELECT_LEX::vers_setup_conds(THD *thd, TABLE_LIST *tables) } // propagate system_time from sysvar - if (!vers_conditions.is_set() && is_select) + if (!vers_conditions.is_set() && use_sysvar) { if (vers_conditions.init_from_sysvar(thd)) DBUG_RETURN(-1); From fb0d18e4128d82ac92c6024cb9d5e4e3c9a6da98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 16 Jun 2020 12:12:36 +0300 Subject: [PATCH 11/73] Add global ignore for Sending JOIN failed warning. --- mysql-test/suite/galera/suite.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/suite.pm b/mysql-test/suite/galera/suite.pm index 96580150c7d..cc35f65423c 100644 --- a/mysql-test/suite/galera/suite.pm +++ b/mysql-test/suite/galera/suite.pm @@ -65,7 +65,7 @@ push @::global_suppressions, qr|WSREP: JOIN message from member .* in non-primary configuration. Ignored.|, qr(WSREP: Failed to remove page file .*), qr(WSREP: wsrep_sst_method is set to 'mysqldump' yet mysqld bind_address is set to .*), - qr(WSREP: .*Transport endpoint is not connected.*), + qr|WSREP: Sending JOIN failed: -107 \(Transport endpoint is not connected\). Will retry in new primary component.|, ); bless { }; From b46b7144d1999d4950a812486f36f2f0d6ab645d Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 17 Jun 2020 11:19:50 +0400 Subject: [PATCH 12/73] MDEV-21695 Server crashes in TABLE::evaluate_update_default_function upon UPDATE on temporary table copy_data_between_tables() sets to->s->default_fields to 0, as a part of the code disabling ON UPDATE actions for all old fields (so ON UPDATE is enable only for new fields during copying). After the actual copying, copy_data_between_tables() did not restore to->s->default_fields to the original value. As a result, the TABLE_SHARE to->s was left in a wrong state after copy_data_between_tables() and further open_table_from_share() using this TABLE_SHARE did not populate TABLE::default_field, which further made TABLE::evaluate_update_default_function() crash on access to NULL pointer. Fix: Changing copy_data_between_tables() to restore to->s->default_fields to its original value after the copying loop. --- mysql-test/r/temp_table.result | 18 ++++++++++++++++++ mysql-test/t/temp_table.test | 19 +++++++++++++++++++ sql/sql_table.cc | 2 ++ 3 files changed, 39 insertions(+) diff --git a/mysql-test/r/temp_table.result b/mysql-test/r/temp_table.result index 78ca7a66682..cd6621ac487 100644 --- a/mysql-test/r/temp_table.result +++ b/mysql-test/r/temp_table.result @@ -584,3 +584,21 @@ ALTER TABLE t1 CHANGE no_such_col1 col1 BIGINT NULL; ERROR 42S22: Unknown column 'no_such_col1' in 't1' TRUNCATE TABLE t1; DROP TEMPORARY TABLE t1; +# +# MDEV-21695 Server crashes in TABLE::evaluate_update_default_function upon UPDATE on temporary table +# +SET timestamp=UNIX_TIMESTAMP('2001-01-01 10:20:30'); +CREATE TEMPORARY TABLE t1 (a DATETIME ON UPDATE CURRENT_TIMESTAMP); +ALTER TABLE t1 ADD b INT; +INSERT INTO t1 (b) VALUES (1),(2); +ALTER TABLE t1 CHANGE COLUMN x xx INT; +ERROR 42S22: Unknown column 'x' in 't1' +UPDATE t1 SET b = 3; +SELECT * FROM t1; +a b +2001-01-01 10:20:30 3 +2001-01-01 10:20:30 3 +DROP TEMPORARY TABLE t1; +# +# End of 10.2 tests +# diff --git a/mysql-test/t/temp_table.test b/mysql-test/t/temp_table.test index bd3bba34f89..dc5fe7f3cd0 100644 --- a/mysql-test/t/temp_table.test +++ b/mysql-test/t/temp_table.test @@ -639,3 +639,22 @@ ALTER TABLE t1 CHANGE no_such_col1 col1 BIGINT NULL; # was not dropped during the first TRUNCATE due to extra table handles. TRUNCATE TABLE t1; DROP TEMPORARY TABLE t1; + +--echo # +--echo # MDEV-21695 Server crashes in TABLE::evaluate_update_default_function upon UPDATE on temporary table +--echo # + +SET timestamp=UNIX_TIMESTAMP('2001-01-01 10:20:30'); +CREATE TEMPORARY TABLE t1 (a DATETIME ON UPDATE CURRENT_TIMESTAMP); +ALTER TABLE t1 ADD b INT; +INSERT INTO t1 (b) VALUES (1),(2); +--error ER_BAD_FIELD_ERROR +ALTER TABLE t1 CHANGE COLUMN x xx INT; +UPDATE t1 SET b = 3; +SELECT * FROM t1; +DROP TEMPORARY TABLE t1; + + +--echo # +--echo # End of 10.2 tests +--echo # diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 87d84b1abc6..7cce3bcc323 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -10057,6 +10057,7 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, sql_mode_t save_sql_mode= thd->variables.sql_mode; ulonglong prev_insert_id, time_to_report_progress; Field **dfield_ptr= to->default_field; + uint save_to_s_default_fields= to->s->default_fields; DBUG_ENTER("copy_data_between_tables"); /* Two or 3 stages; Sorting, copying data and update indexes */ @@ -10338,6 +10339,7 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, *copied= found_count; *deleted=delete_count; to->file->ha_release_auto_increment(); + to->s->default_fields= save_to_s_default_fields; if (!cleanup_done) { From e30c4cfc7a23d64aa602a6b751b51b18fb730a44 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 12 Jun 2020 09:38:35 +0530 Subject: [PATCH 13/73] MDEV-22811 DDL fails to drop and re-create FTS index Problem: ======== - InnoDB clears the fts resource when last FTS index is being dropped if the table has user defined FTS_DOC_ID. While creating the new fts index, InnoDB expects to have FTS resources. Fix: === fts_drop_index(): Removed the fts resource clear. fts_clear_all(): Clear the fts resource when there are no new fts index to be added. commit_cache_norebuild(), row_merge_drop_indexes(): Tries to call fts resource after removing associated fts index from table object --- .../suite/innodb_fts/r/innodb-fts-ddl.result | 7 +++ .../suite/innodb_fts/t/innodb-fts-ddl.test | 8 +++ storage/innobase/fts/fts0fts.cc | 51 ++++++++++--------- storage/innobase/handler/handler0alter.cc | 1 + storage/innobase/include/fts0fts.h | 6 +++ storage/innobase/row/row0merge.cc | 2 + 6 files changed, 51 insertions(+), 24 deletions(-) diff --git a/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result b/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result index 983f254cf90..f23813aed48 100644 --- a/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result +++ b/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result @@ -227,3 +227,10 @@ id title body 1 MySQL Tutorial DBMS stands for DataBase ... 3 Optimizing MySQL In this tutorial we will show ... DROP TABLE articles; +# +# MDEV-22811 DDL fails to drop and re-create FTS index +# +CREATE TABLE t1 (FTS_DOC_ID BIGINT UNSIGNED PRIMARY KEY, +f1 VARCHAR(200),FULLTEXT fidx(f1))engine=innodb; +ALTER TABLE t1 DROP index fidx, ADD FULLTEXT INDEX(f1); +DROP TABLE t1; diff --git a/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test b/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test index 10dc1462c98..cd292803fb3 100644 --- a/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test +++ b/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test @@ -270,3 +270,11 @@ SELECT * FROM articles WHERE MATCH (title, body) AGAINST ('Tutorial' IN NATURAL LANGUAGE MODE); DROP TABLE articles; + +--echo # +--echo # MDEV-22811 DDL fails to drop and re-create FTS index +--echo # +CREATE TABLE t1 (FTS_DOC_ID BIGINT UNSIGNED PRIMARY KEY, + f1 VARCHAR(200),FULLTEXT fidx(f1))engine=innodb; +ALTER TABLE t1 DROP index fidx, ADD FULLTEXT INDEX(f1); +DROP TABLE t1; diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index dd4938dd019..739b402314c 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -805,6 +805,29 @@ fts_check_cached_index( return(TRUE); } +/** Clear all fts resources when there is no internal DOC_ID +and there are no new fts index to add. +@param[in,out] table table where fts is to be freed +@param[in] trx transaction to drop all fts tables */ +void fts_clear_all(dict_table_t *table, trx_t *trx) +{ + if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID) || + !table->fts || + !ib_vector_is_empty(table->fts->indexes)) + return; + + for (const dict_index_t *index= dict_table_get_first_index(table); + index; index= dict_table_get_next_index(index)) + if (index->type & DICT_FTS) + return; + + fts_optimize_remove_table(table); + + fts_drop_tables(trx, table); + fts_free(table); + DICT_TF2_FLAG_UNSET(table, DICT_TF2_FTS); +} + /*******************************************************************//** Drop auxiliary tables related to an FTS index @return DB_SUCCESS or error number */ @@ -821,9 +844,10 @@ fts_drop_index( ut_a(indexes); if ((ib_vector_size(indexes) == 1 - && (index == static_cast( - ib_vector_getp(table->fts->indexes, 0)))) - || ib_vector_is_empty(indexes)) { + && (index == static_cast( + ib_vector_getp(table->fts->indexes, 0))) + && DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) + || ib_vector_is_empty(indexes)) { doc_id_t current_doc_id; doc_id_t first_doc_id; @@ -833,27 +857,6 @@ fts_drop_index( DICT_TF2_FLAG_UNSET(table, DICT_TF2_FTS); - /* If Doc ID column is not added internally by FTS index, - we can drop all FTS auxiliary tables. Otherwise, we will - need to keep some common table such as CONFIG table, so - as to keep track of incrementing Doc IDs */ - if (!DICT_TF2_FLAG_IS_SET( - table, DICT_TF2_FTS_HAS_DOC_ID)) { - - err = fts_drop_tables(trx, table); - - err = fts_drop_index_tables(trx, index); - - while (index->index_fts_syncing - && !trx_is_interrupted(trx)) { - DICT_BG_YIELD(trx); - } - - fts_free(table); - - return(err); - } - while (index->index_fts_syncing && !trx_is_interrupted(trx)) { DICT_BG_YIELD(trx); diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index d057afef9b5..db161ba50d8 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -7922,6 +7922,7 @@ commit_cache_norebuild( dict_index_remove_from_cache(index->table, index); } + fts_clear_all(ctx->old_table, trx); trx_commit_for_mysql(trx); } diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index c2a89c25d16..b7d3451e887 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -983,4 +983,10 @@ fts_trx_t* fts_trx_create( trx_t* trx); +/** Clear all fts resources when there is no internal DOC_ID +and there are no new fts index to add. +@param[in,out] table table where fts is to be freed +@param[in] trx transaction to drop all fts tables */ +void fts_clear_all(dict_table_t *table, trx_t *trx); + #endif /*!< fts0fts.h */ diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index e44b9aae626..5cbfe0e3ddb 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3815,6 +3815,7 @@ row_merge_drop_indexes( ut_error; } + fts_clear_all(table, trx); return; } @@ -3867,6 +3868,7 @@ row_merge_drop_indexes( } } + fts_clear_all(table, trx); table->drop_aborted = FALSE; ut_d(dict_table_check_for_dup_indexes(table, CHECK_ALL_COMPLETE)); } From 4c3cbe23928029288ee1e6fd2fdb957f78a3240f Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Fri, 22 May 2020 21:15:17 +0530 Subject: [PATCH 14/73] MDEV-22665: Print ranges in the optimizer trace created for non-indexed columns when optimizer_use_condition_selectivity >2 Now the optimizer trace shows the ranges constructed while getting estimates from EITS --- mysql-test/main/opt_trace.result | 96 +++++++++++++++++++++++- mysql-test/main/opt_trace.test | 18 ++++- sql/field.cc | 40 ++++++++++ sql/field.h | 1 + sql/opt_range.cc | 123 ++++++++++++++++++++++--------- 5 files changed, 240 insertions(+), 38 deletions(-) diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result index be358e69c47..6da22802cca 100644 --- a/mysql-test/main/opt_trace.result +++ b/mysql-test/main/opt_trace.result @@ -108,6 +108,7 @@ select * from v1 { "selectivity_for_columns": [ { "column_name": "a", + "ranges": ["1 <= a <= 1"], "selectivity_from_histogram": 0.5 } ], @@ -253,6 +254,7 @@ select * from (select * from t1 where t1.a=1)q { "selectivity_for_columns": [ { "column_name": "a", + "ranges": ["1 <= a <= 1"], "selectivity_from_histogram": 0.5 } ], @@ -403,6 +405,7 @@ select * from v2 { "selectivity_for_columns": [ { "column_name": "a", + "ranges": ["1 <= a <= 1"], "selectivity_from_histogram": 0.5 } ], @@ -1416,10 +1419,12 @@ EXPLAIN SELECT MIN(d) FROM t1 where b=2 and c=3 group by a { "selectivity_for_columns": [ { "column_name": "b", + "ranges": ["2 <= b <= 2"], "selectivity_from_histogram": 0.2891 }, { "column_name": "c", + "ranges": ["3 <= c <= 3"], "selectivity_from_histogram": 0.2891 } ], @@ -2091,10 +2096,12 @@ explain select * from t1 where a=1 and b=2 order by c limit 1 { "selectivity_for_columns": [ { "column_name": "a", + "ranges": ["1 <= a <= 1"], "selectivity_from_histogram": 0.1797 }, { "column_name": "b", + "ranges": ["2 <= b <= 2"], "selectivity_from_histogram": 0.0156 } ], @@ -3310,10 +3317,12 @@ explain select * from t1 where pk = 2 and a=5 and b=1 { "selectivity_for_columns": [ { "column_name": "a", + "ranges": ["5 <= a <= 5"], "selectivity_from_histogram": 0.1 }, { "column_name": "b", + "ranges": ["1 <= b <= 1"], "selectivity_from_histogram": 0.1 } ], @@ -8465,5 +8474,90 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans')) ] ] DROP TABLE t1,t2; -# End of 10.4 tests +# +# MDEV-22665: Print ranges in the optimizer trace created for non-indexed columns when +# optimizer_use_condition_selectivity >2 +# +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 SELECT seq, seq from seq_1_to_100; +SET optimizer_trace=1; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +EXPLAIN EXTENDED SELECT * from t1 WHERE a between 1 and 5 and b <= 5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 0.22 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` between 1 and 5 and `test`.`t1`.`b` <= 5 +select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; +JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) +[ + + [ + + { + "column_name": "a", + "ranges": + [ + "1 <= a <= 5" + ], + "selectivity_from_histogram": 0.0469 + }, + + { + "column_name": "b", + "ranges": + [ + "NULL < b <= 5" + ], + "selectivity_from_histogram": 0.0469 + } + ] +] +EXPLAIN EXTENDED SELECT * from t1 WHERE a != 5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` <> 5 +select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; +JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) +[ + + [ + + { + "column_name": "a", + "ranges": + [ + "NULL < a < 5", + "5 < a" + ], + "selectivity_from_histogram": 1 + } + ] +] +EXPLAIN EXTENDED SELECT * from t1 WHERE b >= 10 and b < 25; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 15.62 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`b` >= 10 and `test`.`t1`.`b` < 25 +select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; +JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) +[ + + [ + + { + "column_name": "b", + "ranges": + [ + "10 <= b < 25" + ], + "selectivity_from_histogram": 0.1562 + } + ] +] +drop table t1; set optimizer_trace='enabled=off'; +# End of 10.4 tests diff --git a/mysql-test/main/opt_trace.test b/mysql-test/main/opt_trace.test index d1a8fedc635..8633ed5b020 100644 --- a/mysql-test/main/opt_trace.test +++ b/mysql-test/main/opt_trace.test @@ -594,6 +594,22 @@ EXPLAIN SELECT * FROM t1, t2 WHERE t1.a=t2.a ORDER BY t2.b; select JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; DROP TABLE t1,t2; ---echo # End of 10.4 tests +--echo # +--echo # MDEV-22665: Print ranges in the optimizer trace created for non-indexed columns when +--echo # optimizer_use_condition_selectivity >2 +--echo # + +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 SELECT seq, seq from seq_1_to_100; +SET optimizer_trace=1; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +EXPLAIN EXTENDED SELECT * from t1 WHERE a between 1 and 5 and b <= 5; +select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; +EXPLAIN EXTENDED SELECT * from t1 WHERE a != 5; +select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; +EXPLAIN EXTENDED SELECT * from t1 WHERE b >= 10 and b < 25; +select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; +drop table t1; set optimizer_trace='enabled=off'; +--echo # End of 10.4 tests diff --git a/sql/field.cc b/sql/field.cc index 3f4e1e73029..21145cdf32a 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -11400,6 +11400,46 @@ void Field_blob::print_key_value(String *out, uint32 length) } +/* + @brief Print value of the key part + + @param + out Output string + key value of the key + length Length of field in bytes, + excluding NULL flag and length bytes +*/ + + +void +Field::print_key_part_value(String *out, const uchar* key, uint32 length) +{ + StringBuffer<128> tmp(system_charset_info); + uint null_byte= 0; + if (real_maybe_null()) + { + /* + Byte 0 of key is the null-byte. If set, key is NULL. + Otherwise, print the key value starting immediately after the + null-byte + */ + if (*key) + { + out->append(STRING_WITH_LEN("NULL")); + return; + } + null_byte++; // Skip null byte + } + + set_key_image(key + null_byte, length); + print_key_value(&tmp, length); + if (charset() == &my_charset_bin) + out->append(tmp.ptr(), tmp.length(), tmp.charset()); + else + tmp.print(out, system_charset_info); +} + + void Field::print_key_value_binary(String *out, const uchar* key, uint32 length) { out->append_semi_hex((const char*)key, length, charset()); diff --git a/sql/field.h b/sql/field.h index 7393def96b1..1d421abe880 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1410,6 +1410,7 @@ public: bool set_warning(Sql_condition::enum_warning_level, unsigned int code, int cuted_increment, ulong current_row=0) const; virtual void print_key_value(String *out, uint32 length); + void print_key_part_value(String *out, const uchar *key, uint32 length); void print_key_value_binary(String *out, const uchar* key, uint32 length); protected: bool set_warning(unsigned int code, int cuted_increment) const diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 690657616b8..ecbf5d6afcb 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -442,6 +442,13 @@ static void print_range(String *out, const KEY_PART_INFO *key_part, KEY_MULTI_RANGE *range, uint n_key_parts); +static +void print_range_for_non_indexed_field(String *out, Field *field, + KEY_MULTI_RANGE *range); + +static void print_min_range_operator(String *out, const ha_rkey_function flag); +static void print_max_range_operator(String *out, const ha_rkey_function flag); + /* SEL_IMERGE is a list of possible ways to do index merge, i.e. it is @@ -3174,6 +3181,7 @@ static double records_in_column_ranges(PARAM *param, uint idx, SEL_ARG *tree) { + THD *thd= param->thd; SEL_ARG_RANGE_SEQ seq; KEY_MULTI_RANGE range; range_seq_t seq_it; @@ -3200,6 +3208,8 @@ double records_in_column_ranges(PARAM *param, uint idx, seq_it= seq_if.init((void *) &seq, 0, flags); + Json_writer_array range_trace(thd, "ranges"); + while (!seq_if.next(seq_it, &range)) { key_range *min_endp, *max_endp; @@ -3216,6 +3226,13 @@ double records_in_column_ranges(PARAM *param, uint idx, if (range.start_key.flag == HA_READ_BEFORE_KEY) range_flag |= NEAR_MAX; + if (unlikely(thd->trace_started())) + { + StringBuffer<128> range_info(system_charset_info); + print_range_for_non_indexed_field(&range_info, field, &range); + range_trace.add(range_info.c_ptr_safe(), range_info.length()); + } + rows= get_column_range_cardinality(field, min_endp, max_endp, range_flag); if (DBL_MAX == rows) { @@ -15804,6 +15821,37 @@ void QUICK_GROUP_MIN_MAX_SELECT::dbug_dump(int indent, bool verbose) #endif /* !DBUG_OFF */ + +/* + @brief Print the comparison operator for the min range +*/ + +static void print_min_range_operator(String *out, const ha_rkey_function flag) +{ + if (flag == HA_READ_AFTER_KEY) + out->append(STRING_WITH_LEN(" < ")); + else if (flag == HA_READ_KEY_EXACT || flag == HA_READ_KEY_OR_NEXT) + out->append(STRING_WITH_LEN(" <= ")); + else + out->append(STRING_WITH_LEN(" ? ")); +} + + +/* + @brief Print the comparison operator for the max range +*/ + +static void print_max_range_operator(String *out, const ha_rkey_function flag) +{ + if (flag == HA_READ_BEFORE_KEY) + out->append(STRING_WITH_LEN(" < ")); + else if (flag == HA_READ_AFTER_KEY) + out->append(STRING_WITH_LEN(" <= ")); + else + out->append(STRING_WITH_LEN(" ? ")); +} + + static void print_range(String *out, const KEY_PART_INFO *key_part, KEY_MULTI_RANGE *range, uint n_key_parts) @@ -15832,30 +15880,55 @@ void print_range(String *out, const KEY_PART_INFO *key_part, { print_key_value(out, key_part, range->start_key.key, range->start_key.length); - if (range->start_key.flag == HA_READ_AFTER_KEY) - out->append(STRING_WITH_LEN(" < ")); - else if (range->start_key.flag == HA_READ_KEY_EXACT || - range->start_key.flag == HA_READ_KEY_OR_NEXT) - out->append(STRING_WITH_LEN(" <= ")); - else - out->append(STRING_WITH_LEN(" ? ")); + print_min_range_operator(out, range->start_key.flag); } print_keyparts_name(out, key_part, n_key_parts, keypart_map); if (range->end_key.length) { - if (range->end_key.flag == HA_READ_BEFORE_KEY) - out->append(STRING_WITH_LEN(" < ")); - else if (range->end_key.flag == HA_READ_AFTER_KEY) - out->append(STRING_WITH_LEN(" <= ")); - else - out->append(STRING_WITH_LEN(" ? ")); + print_max_range_operator(out, range->end_key.flag); print_key_value(out, key_part, range->end_key.key, range->end_key.length); } } + +/* + @brief Print range created for non-indexed columns + + @param + out output string + field field for which the range is printed + range range for the field +*/ + +static +void print_range_for_non_indexed_field(String *out, Field *field, + KEY_MULTI_RANGE *range) +{ + TABLE *table= field->table; + my_bitmap_map *old_sets[2]; + dbug_tmp_use_all_columns(table, old_sets, table->read_set, table->write_set); + + if (range->start_key.length) + { + field->print_key_part_value(out, range->start_key.key, field->key_length()); + print_min_range_operator(out, range->start_key.flag); + } + + out->append(field->field_name); + + if (range->end_key.length) + { + print_max_range_operator(out, range->end_key.flag); + field->print_key_part_value(out, range->end_key.key, field->key_length()); + } + dbug_tmp_restore_column_maps(table->read_set, table->write_set, old_sets); +} + + + /* Add ranges to the trace @@ -15927,30 +16000,8 @@ static void print_key_value(String *out, const KEY_PART_INFO *key_part, field= key_part->field; store_length= key_part->store_length; - if (field->real_maybe_null()) - { - /* - Byte 0 of key is the null-byte. If set, key is NULL. - Otherwise, print the key value starting immediately after the - null-byte - */ - if (*key) - { - out->append(STRING_WITH_LEN("NULL")); - goto next; - } - key++; // Skip null byte - store_length--; - } + field->print_key_part_value(out, key, key_part->length); - field->set_key_image(key, key_part->length); - field->print_key_value(&tmp, key_part->length); - if (field->charset() == &my_charset_bin) - out->append(tmp.ptr(), tmp.length(), tmp.charset()); - else - tmp.print(out, system_charset_info); - - next: if (key + store_length < key_end) out->append(STRING_WITH_LEN(",")); } From 26907e7ef18964dad7b6b1fc19c7d1e65e915020 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 19 Jun 2020 16:04:45 +0400 Subject: [PATCH 15/73] MDEV-22941 Assertion `idx < array.elements' failed in Dynamic_array::at The code in fill_schema_schemata() did not take into account that make_db_list() can leave empty db_names if the requested database name was too long, so the call for db_names.at(0) crashed on assert. - Moving the code testing if the database directory exists into a separate function verify_database_directory_exists() - Modifying the test to check if db_names is not empty --- mysql-test/r/information_schema.result | 10 +++++++ mysql-test/t/information_schema.test | 12 ++++++++ sql/sql_show.cc | 40 +++++++++++++++++--------- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index 631dd8eabf5..dae46f9ff47 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -2101,3 +2101,13 @@ column_name c1 c2 DROP TABLE tt1, tt2; +# +# MDEV-13242 Wrong results for queries with row constructors and information_schema +# +SELECT SCHEMA_NAME from information_schema.schemata where schema_name='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +SCHEMA_NAME +SELECT SCHEMA_NAME from information_schema.schemata where schema_name=REPEAT('a',193); +SCHEMA_NAME +# +# End of 10.1 tests +# diff --git a/mysql-test/t/information_schema.test b/mysql-test/t/information_schema.test index 5b3fa7b653c..0ab2930e3f5 100644 --- a/mysql-test/t/information_schema.test +++ b/mysql-test/t/information_schema.test @@ -1936,3 +1936,15 @@ SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (t SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name='tt1' AND column_name='c1') OR (table_name='tt2' AND column_name='c2'); SELECT column_name FROM information_schema.columns WHERE (table_name, column_name) IN (('tt1','c1'),('tt2', 'c2')) ORDER BY column_name; DROP TABLE tt1, tt2; + +--echo # +--echo # MDEV-13242 Wrong results for queries with row constructors and information_schema +--echo # + +SELECT SCHEMA_NAME from information_schema.schemata where schema_name='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +SELECT SCHEMA_NAME from information_schema.schemata where schema_name=REPEAT('a',193); + + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 4f217159e5c..70cab968f43 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -4953,6 +4953,29 @@ bool store_schema_shemata(THD* thd, TABLE *table, LEX_STRING *db_name, } +/* + Check if the specified database exists on disk. + + @param dbname - the database name + @retval true - on error, the database directory does not exists + @retval false - on success, the database directory exists +*/ +static bool verify_database_directory_exists(const LEX_STRING &dbname) +{ + DBUG_ENTER("verity_database_exists"); + char path[FN_REFLEN + 16]; + uint path_len; + MY_STAT stat_info; + if (!dbname.str[0]) + DBUG_RETURN(true); // Empty database name: does not exits. + path_len= build_table_filename(path, sizeof(path) - 1, dbname.str, "", "", 0); + path[path_len - 1]= 0; + if (!mysql_file_stat(key_file_misc, path, &stat_info, MYF(0))) + DBUG_RETURN(true); // The database directory was not found: does not exists. + DBUG_RETURN(false); // The database directory was found. +} + + int fill_schema_schemata(THD *thd, TABLE_LIST *tables, COND *cond) { /* @@ -4981,19 +5004,10 @@ int fill_schema_schemata(THD *thd, TABLE_LIST *tables, COND *cond) If we have lookup db value we should check that the database exists */ if(lookup_field_vals.db_value.str && !lookup_field_vals.wild_db_value && - db_names.at(0) != &INFORMATION_SCHEMA_NAME) - { - char path[FN_REFLEN+16]; - uint path_len; - MY_STAT stat_info; - if (!lookup_field_vals.db_value.str[0]) - DBUG_RETURN(0); - path_len= build_table_filename(path, sizeof(path) - 1, - lookup_field_vals.db_value.str, "", "", 0); - path[path_len-1]= 0; - if (!mysql_file_stat(key_file_misc, path, &stat_info, MYF(0))) - DBUG_RETURN(0); - } + (!db_names.elements() /* The database name was too long */|| + (db_names.at(0) != &INFORMATION_SCHEMA_NAME && + verify_database_directory_exists(lookup_field_vals.db_value)))) + DBUG_RETURN(0); for (size_t i=0; i < db_names.elements(); i++) { From 727252ff1bb65c7abddc5a9acb17304695c09265 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 20 Jun 2020 01:00:36 +0200 Subject: [PATCH 16/73] MDEV-22950 : fix race condition in dbug FreeState() zeros init_settings.out_file, which another thread can be using --- dbug/dbug.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbug/dbug.c b/dbug/dbug.c index 21f86ded0a5..b0e1b0eaae6 100644 --- a/dbug/dbug.c +++ b/dbug/dbug.c @@ -483,6 +483,7 @@ static int DbugParse(CODE_STATE *cs, const char *control) rel= control[0] == '+' || control[0] == '-'; if ((!rel || (!stack->out_file && !stack->next))) { + LockIfInitSettings(cs); FreeState(cs, 0); stack->flags= 0; stack->delay= 0; @@ -490,10 +491,9 @@ static int DbugParse(CODE_STATE *cs, const char *control) stack->sub_level= 0; stack->out_file= sstderr; stack->functions= NULL; - LockIfInitSettings(cs); stack->keywords= NULL; - UnlockIfInitSettings(cs); stack->processes= NULL; + UnlockIfInitSettings(cs); } else if (!stack->out_file) { From b1b9803cb800334962970a8783c782253021e964 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 20 Jun 2020 01:01:50 +0200 Subject: [PATCH 17/73] Disable dtrace probes on Windows. Apparently, in Win10, dtrace is avaialable, but it does not work with MariaDB user probes --- cmake/dtrace.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/dtrace.cmake b/cmake/dtrace.cmake index 4e0f7501045..d3dded99497 100644 --- a/cmake/dtrace.cmake +++ b/cmake/dtrace.cmake @@ -43,7 +43,8 @@ MACRO(CHECK_DTRACE) IF(DTRACE AND NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT BUGGY_GCC_NO_DTRACE_MODULES AND NOT BUGGY_LINUX_DTRACE - AND NOT CMAKE_SYSTEM_NAME MATCHES "SunOS") + AND NOT CMAKE_SYSTEM_NAME MATCHES "SunOS" + AND NOT WIN32) SET(ENABLE_DTRACE ON CACHE BOOL "Enable dtrace") ENDIF() # On GNU/Hurd, dtrace is not supported From 804ed12e0e98012b9307a5bf74b41e76dd1ae67b Mon Sep 17 00:00:00 2001 From: Sachin Date: Fri, 17 Apr 2020 16:32:51 +0530 Subject: [PATCH 18/73] MDEV-22179 rr(record and replay) support for mtr This feature adds the support for rr in mtr. These 2 options are added --rr run the mysqld in rr record mode --rr_option= run the rr with custom record option, for multiple options use --rr_option= for each option. For example ./mtr main.view --rr_option=-h --rr_option=-u --rr_option=-c=23 --boot-rr run the mysqld performing bootstrap in rr record mode Recording are stored in mysql-test/var/rr folder. To run recording please run rr replay var/rr/mysql-X Limitations Restart will create a new recording. Repeat will work on same recording , So might be harder to debug. If test create the multiple instance of mariadb all will be stored in var/rr --- mysql-test/mysql-test-run.pl | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 2d4f3df3f5d..5fa087bd28f 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -261,8 +261,10 @@ our %gprof_dirs; our $glob_debugger= 0; our $opt_gdb; +our $opt_rr; our $opt_client_gdb; my $opt_boot_gdb; +my $opt_boot_rr; our $opt_dbx; our $opt_client_dbx; my $opt_boot_dbx; @@ -333,6 +335,7 @@ my $opt_callgrind; my %mysqld_logs; my $opt_debug_sync_timeout= 300; # Default timeout for WAIT_FOR actions. my $warn_seconds = 60; +my @rr_record_args; sub testcase_timeout ($) { my ($tinfo)= @_; @@ -1316,10 +1319,13 @@ sub command_line_setup { 'debug-common' => \$opt_debug_common, 'debug-server' => \$opt_debug_server, 'gdb=s' => \$opt_gdb, + 'rr' => \$opt_rr, + 'rr_option=s' => \@rr_record_args, 'client-gdb' => \$opt_client_gdb, 'manual-gdb' => \$opt_manual_gdb, 'manual-lldb' => \$opt_manual_lldb, 'boot-gdb' => \$opt_boot_gdb, + 'boot-rr' => \$opt_boot_rr, 'manual-debug' => \$opt_manual_debug, 'ddd' => \$opt_ddd, 'client-ddd' => \$opt_client_ddd, @@ -1782,8 +1788,8 @@ sub command_line_setup { # -------------------------------------------------------------------------- # Check debug related options # -------------------------------------------------------------------------- - if ( $opt_gdb || $opt_client_gdb || $opt_ddd || $opt_client_ddd || - $opt_manual_gdb || $opt_manual_lldb || $opt_manual_ddd || + if ( $opt_gdb || $opt_client_gdb || $opt_ddd || $opt_client_ddd || $opt_rr || + @rr_record_args || $opt_manual_gdb || $opt_manual_lldb || $opt_manual_ddd || $opt_manual_debug || $opt_dbx || $opt_client_dbx || $opt_manual_dbx || $opt_debugger || $opt_client_debugger ) { @@ -2440,6 +2446,7 @@ sub environment_setup { $ENV{'MYSQL_TEST_DIR'}= $glob_mysql_test_dir; $ENV{'DEFAULT_MASTER_PORT'}= $mysqld_variables{'port'}; $ENV{'MYSQL_TMP_DIR'}= $opt_tmpdir; + $ENV{'_RR_TRACE_DIR'}= "$opt_vardir/rr"; $ENV{'MYSQLTEST_VARDIR'}= $opt_vardir; $ENV{'MYSQL_BINDIR'}= $bindir; $ENV{'MYSQL_SHAREDIR'}= $path_language; @@ -2749,6 +2756,7 @@ sub setup_vardir() { # Create var/tmp and tmp - they might be different mkpath("$opt_vardir/tmp"); mkpath($opt_tmpdir) if ($opt_tmpdir ne "$opt_vardir/tmp"); + mkpath("$opt_vardir/rr"); # On some operating systems, there is a limit to the length of a # UNIX domain socket's path far below PATH_MAX. @@ -3398,6 +3406,10 @@ sub mysql_install_db { ddd_arguments(\$args, \$exe_mysqld_bootstrap, $mysqld->name(), $bootstrap_sql_file); } + if ($opt_boot_rr) { + $args= ["record", "$exe_mysqld_bootstrap", @$args]; + $exe_mysqld_bootstrap= "rr"; + } my $path_sql= my_find_file($install_basedir, ["mysql", "sql/share", "share/mariadb", @@ -5368,6 +5380,18 @@ sub mysqld_start ($$) { # Indicate the exe should not be started $exe= undef; } + elsif ( $opt_rr || @rr_record_args) + { + if (@rr_record_args) + { + $args= ["record", @rr_record_args, "$exe", @$args]; + } + else + { + $args= ["record", "$exe", @$args]; + } + $exe= "rr"; + } else { # Default to not wait until pid file has been created @@ -6552,6 +6576,12 @@ Options for strace stracer= Specify name and path to the trace program to use. Default is "strace". Example: $0 --stracer=ktrace. +Options for rr(Record and Replay) + rr Run the "mysqld" executables using rr. Default run + option is "rr record mysqld mysqld_options" + boot-rr Start bootstrap server in rr + rr_option=ARG Option to give rr record, can be specified more then once + Misc options user=USER User for connecting to mysqld(default: $opt_user) comment=STR Write STR to the output From 009ef36d9ae01a44ddf5e1af35f54740838e1e70 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Tue, 16 Jun 2020 22:59:00 +0300 Subject: [PATCH 19/73] MDEV-22179 rr support for mtr review * --rr-arg instead of --rr_option * Bootstrap saved to rr.bootstrap * Replication slaves are saved to rr.N dirs * Perl coding fixes --- mysql-test/mysql-test-run.pl | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 5fa087bd28f..93cf00498b4 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1320,7 +1320,7 @@ sub command_line_setup { 'debug-server' => \$opt_debug_server, 'gdb=s' => \$opt_gdb, 'rr' => \$opt_rr, - 'rr_option=s' => \@rr_record_args, + 'rr-arg=s' => \@rr_record_args, 'client-gdb' => \$opt_client_gdb, 'manual-gdb' => \$opt_manual_gdb, 'manual-lldb' => \$opt_manual_lldb, @@ -1785,11 +1785,16 @@ sub command_line_setup { mtr_error("Coverage test needs the source - please use source dist"); } + if ( @rr_record_args ) + { + $opt_rr= 1; + } + # -------------------------------------------------------------------------- # Check debug related options # -------------------------------------------------------------------------- if ( $opt_gdb || $opt_client_gdb || $opt_ddd || $opt_client_ddd || $opt_rr || - @rr_record_args || $opt_manual_gdb || $opt_manual_lldb || $opt_manual_ddd || + $opt_manual_gdb || $opt_manual_lldb || $opt_manual_ddd || $opt_manual_debug || $opt_dbx || $opt_client_dbx || $opt_manual_dbx || $opt_debugger || $opt_client_debugger ) { @@ -2446,7 +2451,6 @@ sub environment_setup { $ENV{'MYSQL_TEST_DIR'}= $glob_mysql_test_dir; $ENV{'DEFAULT_MASTER_PORT'}= $mysqld_variables{'port'}; $ENV{'MYSQL_TMP_DIR'}= $opt_tmpdir; - $ENV{'_RR_TRACE_DIR'}= "$opt_vardir/rr"; $ENV{'MYSQLTEST_VARDIR'}= $opt_vardir; $ENV{'MYSQL_BINDIR'}= $bindir; $ENV{'MYSQL_SHAREDIR'}= $path_language; @@ -2756,7 +2760,6 @@ sub setup_vardir() { # Create var/tmp and tmp - they might be different mkpath("$opt_vardir/tmp"); mkpath($opt_tmpdir) if ($opt_tmpdir ne "$opt_vardir/tmp"); - mkpath("$opt_vardir/rr"); # On some operating systems, there is a limit to the length of a # UNIX domain socket's path far below PATH_MAX. @@ -3407,8 +3410,11 @@ sub mysql_install_db { $bootstrap_sql_file); } if ($opt_boot_rr) { - $args= ["record", "$exe_mysqld_bootstrap", @$args]; + $args= ["record", @rr_record_args, $exe_mysqld_bootstrap, @$args]; $exe_mysqld_bootstrap= "rr"; + my $rr_dir= "$opt_vardir/rr.bootstrap"; + $ENV{'_RR_TRACE_DIR'}= $rr_dir; + mkpath($rr_dir); } my $path_sql= my_find_file($install_basedir, @@ -5380,17 +5386,13 @@ sub mysqld_start ($$) { # Indicate the exe should not be started $exe= undef; } - elsif ( $opt_rr || @rr_record_args) + elsif ( $opt_rr ) { - if (@rr_record_args) - { - $args= ["record", @rr_record_args, "$exe", @$args]; - } - else - { - $args= ["record", "$exe", @$args]; - } + $args= ["record", @rr_record_args, "$exe", @$args]; $exe= "rr"; + my $rr_dir= "$opt_vardir/rr". $mysqld->after('mysqld'); + $ENV{'_RR_TRACE_DIR'}= $rr_dir; + mkpath($rr_dir); } else { @@ -6580,7 +6582,7 @@ Options for rr(Record and Replay) rr Run the "mysqld" executables using rr. Default run option is "rr record mysqld mysqld_options" boot-rr Start bootstrap server in rr - rr_option=ARG Option to give rr record, can be specified more then once + rr-arg=ARG Option to give rr record, can be specified more then once Misc options user=USER User for connecting to mysqld(default: $opt_user) From ccc4eb8530727d5b9701f46ef4d79660b872a66e Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sun, 3 May 2020 01:05:15 +0300 Subject: [PATCH 20/73] MDEV-22438 add a function similar to std::make_scope_exit() The idea was borrowed from http://wg21.link/p0052 scope_exit class is a helper, its name is hidden from user in the namespace detail. Alternative implementation of scope_exit with std::function looks slower on goldbolt.org as it may require allocation, etc. scope_exit doesn't need to own a callable, so beeing a pointer is enough. And std::decay produces such a pointer from callable. --- include/scope.h | 66 +++++++++++++++++++++++++++ storage/innobase/handler/ha_innodb.cc | 6 +-- 2 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 include/scope.h diff --git a/include/scope.h b/include/scope.h new file mode 100644 index 00000000000..b9e2e96ae5c --- /dev/null +++ b/include/scope.h @@ -0,0 +1,66 @@ +/* + Copyright (c) 2020, MariaDB + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +#pragma once + +#include +#include + +namespace detail +{ + +template class scope_exit +{ +public: + template + explicit scope_exit(F &&f) : function_(std::forward(f)) + { + } + + scope_exit(scope_exit &&rhs) + : function_(std::move(rhs.function_)), engaged_(rhs.engaged_) + { + rhs.release(); + } + + scope_exit(const scope_exit &)= delete; + scope_exit &operator=(scope_exit &&)= delete; + scope_exit &operator=(const scope_exit &)= delete; + + void release() { engaged_= false; } + + ~scope_exit() + { + if (engaged_) + function_(); + } + +private: + Callable function_; + bool engaged_= true; +}; + +} // end namespace detail + +template +detail::scope_exit::type> +make_scope_exit(Callable &&f) +{ + return detail::scope_exit::type>( + std::forward(f)); +} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 777e93997e0..06d63fbe84a 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -55,6 +55,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include #include #include "field.h" +#include "scope.h" // MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system; // MYSQL_PLUGIN_IMPORT extern char mysql_unpacked_real_data_home[]; @@ -10881,6 +10882,7 @@ create_table_info_t::create_table_def() } heap = mem_heap_create(1000); + auto _ = make_scope_exit([heap]() { mem_heap_free(heap); }); ut_d(bool have_vers_start = false); ut_d(bool have_vers_end = false); @@ -10941,7 +10943,6 @@ create_table_info_t::create_table_def() " must be below 256." " Unsupported code " ULINTPF ".", charset_no); - mem_heap_free(heap); dict_mem_table_free(table); DBUG_RETURN(ER_CANT_CREATE_TABLE); @@ -10972,7 +10973,6 @@ create_table_info_t::create_table_def() field->field_name.str); err_col: dict_mem_table_free(table); - mem_heap_free(heap); ut_ad(trx_state_eq(m_trx, TRX_STATE_NOT_STARTED)); DBUG_RETURN(HA_ERR_GENERIC); } @@ -11100,8 +11100,6 @@ err_col: DBUG_SUICIDE();); } - mem_heap_free(heap); - DBUG_EXECUTE_IF("ib_create_err_tablespace_exist", err = DB_TABLESPACE_EXISTS;); From b80b52394d41a4c334642ae8b3af16f76f6fac57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 22 Jun 2020 13:25:25 +0300 Subject: [PATCH 21/73] Test case cleanups. --- mysql-test/suite/galera/r/galera_as_slave_gtid.result | 1 + .../suite/galera/r/galera_as_slave_gtid_auto_engine.result | 1 + .../suite/galera/r/galera_as_slave_gtid_myisam.result | 3 ++- mysql-test/suite/galera/suite.pm | 3 ++- mysql-test/suite/galera/t/galera_as_slave_gtid.inc | 7 +++++++ mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test | 7 +++++-- 6 files changed, 18 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_as_slave_gtid.result b/mysql-test/suite/galera/r/galera_as_slave_gtid.result index 3f4137a3b28..41d9085ccac 100644 --- a/mysql-test/suite/galera/r/galera_as_slave_gtid.result +++ b/mysql-test/suite/galera/r/galera_as_slave_gtid.result @@ -9,6 +9,7 @@ INSERT INTO t1 VALUES(1); SELECT LENGTH(@@global.gtid_binlog_state) > 1; LENGTH(@@global.gtid_binlog_state) > 1 1 +connection node_1; connection node_2; gtid_binlog_state_equal 1 diff --git a/mysql-test/suite/galera/r/galera_as_slave_gtid_auto_engine.result b/mysql-test/suite/galera/r/galera_as_slave_gtid_auto_engine.result index 6c84c1ecd31..cff5bb0a5d0 100644 --- a/mysql-test/suite/galera/r/galera_as_slave_gtid_auto_engine.result +++ b/mysql-test/suite/galera/r/galera_as_slave_gtid_auto_engine.result @@ -9,6 +9,7 @@ INSERT INTO t1 VALUES(1); SELECT LENGTH(@@global.gtid_binlog_state) > 1; LENGTH(@@global.gtid_binlog_state) > 1 1 +connection node_1; connection node_2; gtid_binlog_state_equal 1 diff --git a/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result b/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result index 7e8da675ad4..b498f334bf8 100644 --- a/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result +++ b/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result @@ -31,7 +31,6 @@ DROP TABLE t1; connection node_1; connection node_2; connection node_3; -connection node_3; RESET MASTER; connection node_1; STOP SLAVE; @@ -43,3 +42,5 @@ connection node_2; SET GLOBAL WSREP_ON=OFF; reset master; SET GLOBAL WSREP_ON=ON; +connection node_3; +RESET MASTER; diff --git a/mysql-test/suite/galera/suite.pm b/mysql-test/suite/galera/suite.pm index 4dd11f5d7a4..671fd1688c9 100644 --- a/mysql-test/suite/galera/suite.pm +++ b/mysql-test/suite/galera/suite.pm @@ -62,7 +62,8 @@ push @::global_suppressions, qr(WSREP: Ignoring error*), qr(WSREP: Failed to remove page file .*), qr(WSREP: wsrep_sst_method is set to 'mysqldump' yet mysqld bind_address is set to .*), - qr(WSREP: .*Transport endpoint is not connected.*), + qr|WSREP: Sending JOIN failed: -107 \(Transport endpoint is not connected\). Will retry in new primary component.|, + qr|WSREP: Trying to continue unpaused monitor|, ); sub skip_combinations { diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid.inc b/mysql-test/suite/galera/t/galera_as_slave_gtid.inc index f5222b4322b..461e7833a94 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_gtid.inc +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid.inc @@ -27,6 +27,13 @@ INSERT INTO t1 VALUES(1); SELECT LENGTH(@@global.gtid_binlog_state) > 1; --let $gtid_binlog_state_node1 = `SELECT @@global.gtid_binlog_state;` +--connection node_1 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc + --connection node_2 --let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test index 004f5c825bb..8787f864a99 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test @@ -59,6 +59,8 @@ SELECT COUNT(*) AS EXPECT_0 FROM t1; --connection node_3 DROP TABLE t1; +--sleep 1 + --connection node_1 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc @@ -70,8 +72,6 @@ DROP TABLE t1; --connection node_3 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc - ---connection node_3 RESET MASTER; --connection node_1 @@ -85,3 +85,6 @@ SET GLOBAL WSREP_ON=ON; SET GLOBAL WSREP_ON=OFF; reset master; SET GLOBAL WSREP_ON=ON; + +--connection node_3 +RESET MASTER; From 30903c37432e518e09b87610f6edd0a1d039669c Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 22 Jun 2020 15:43:53 +0400 Subject: [PATCH 22/73] MDEV-22976 CAST(JSON_EXTRACT() AS DECIMAL) does not handle boolean values Item_func_json_extract did not implement val_decimal(), so CAST(JSON_EXTRACT('{"x":true}', '$.x') AS DECIMAL) erroneously returned 0 with a warning because of convertion from the string "true" to decimal. Implementing val_decimal(), so boolean values are correctly handled. --- mysql-test/r/func_json.result | 13 +++++++++++++ mysql-test/t/func_json.test | 13 +++++++++++++ sql/item_jsonfunc.cc | 35 +++++++++++++++++++++++++++++++++++ sql/item_jsonfunc.h | 1 + 4 files changed, 62 insertions(+) diff --git a/mysql-test/r/func_json.result b/mysql-test/r/func_json.result index 94adf310fec..caf46eb5145 100644 --- a/mysql-test/r/func_json.result +++ b/mysql-test/r/func_json.result @@ -916,5 +916,18 @@ NULL Warnings: Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' # +# MDEV-22976 CAST(JSON_EXTRACT() AS DECIMAL) does not handle boolean values +# +SELECT +CAST(JSON_EXTRACT('{"x":true}', '$.x') AS DOUBLE) AS cf, +CAST(JSON_EXTRACT('{"x":true}', '$.x') AS DECIMAL) AS cd; +cf cd +1 1 +SELECT +CAST(JSON_EXTRACT('{"x":false}', '$.x') AS DOUBLE) AS cf, +CAST(JSON_EXTRACT('{"x":false}', '$.x') AS DECIMAL) AS cd; +cf cd +0 0 +# # End of 10.2 tests # diff --git a/mysql-test/t/func_json.test b/mysql-test/t/func_json.test index 2eaaf218e5e..c9adbd945b1 100644 --- a/mysql-test/t/func_json.test +++ b/mysql-test/t/func_json.test @@ -540,6 +540,19 @@ SELECT JSON_MERGE_PATCH('{}'); SELECT JSON_MERGE_PATCH('{', '[1,2,3]'); SELECT JSON_MERGE_PATCH('{"a":"b"}', '[1,'); +--echo # +--echo # MDEV-22976 CAST(JSON_EXTRACT() AS DECIMAL) does not handle boolean values +--echo # + +SELECT + CAST(JSON_EXTRACT('{"x":true}', '$.x') AS DOUBLE) AS cf, + CAST(JSON_EXTRACT('{"x":true}', '$.x') AS DECIMAL) AS cd; + +SELECT + CAST(JSON_EXTRACT('{"x":false}', '$.x') AS DOUBLE) AS cf, + CAST(JSON_EXTRACT('{"x":false}', '$.x') AS DECIMAL) AS cd; + + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 4db4c563d97..b3c8366907a 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -962,6 +962,41 @@ double Item_func_json_extract::val_real() } +my_decimal *Item_func_json_extract::val_decimal(my_decimal *to) +{ + json_value_types type; + char *value; + int value_len; + + if (read_json(NULL, &type, &value, &value_len) != NULL) + { + switch (type) + { + case JSON_VALUE_STRING: + case JSON_VALUE_NUMBER: + { + my_decimal *res= decimal_from_string_with_check(to, collation.collation, + value, + value + value_len); + null_value= res == NULL; + return res; + } + case JSON_VALUE_TRUE: + int2my_decimal(E_DEC_FATAL_ERROR, 1, false/*unsigned_flag*/, to); + return to; + case JSON_VALUE_OBJECT: + case JSON_VALUE_ARRAY: + case JSON_VALUE_FALSE: + case JSON_VALUE_NULL: + break; + }; + } + int2my_decimal(E_DEC_FATAL_ERROR, 0, false/*unsigned_flag*/, to); + return to; +} + + + bool Item_func_json_contains::fix_length_and_dec() { a2_constant= args[1]->const_item(); diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index d7f804a1b60..0277cb15154 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -171,6 +171,7 @@ public: String *val_str(String *); longlong val_int(); double val_real(); + my_decimal *val_decimal(my_decimal *); uint get_n_paths() const { return arg_count - 1; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } From e72e6663e1ec7ca91120ee4c40021771b117f2a2 Mon Sep 17 00:00:00 2001 From: Sachin Date: Tue, 23 Jun 2020 00:00:39 +0530 Subject: [PATCH 23/73] Remove rpl_parallel2 from disabled.def --- mysql-test/suite/rpl/disabled.def | 1 - 1 file changed, 1 deletion(-) diff --git a/mysql-test/suite/rpl/disabled.def b/mysql-test/suite/rpl/disabled.def index 9b123cda2aa..acdd3afcf9c 100644 --- a/mysql-test/suite/rpl/disabled.def +++ b/mysql-test/suite/rpl/disabled.def @@ -15,6 +15,5 @@ rpl_spec_variables : BUG#11755836 2009-10-27 jasonh rpl_spec_variables fa rpl_partition_archive : MDEV-5077 2013-09-27 svoj Cannot exchange partition with archive table rpl_row_binlog_max_cache_size : MDEV-11092 rpl_row_index_choice : MDEV-11666 -rpl_parallel2 : fails after MDEV-16172 rpl_semi_sync_after_sync : fails after MDEV-16172 rpl_slave_grp_exec: MDEV-10514 From 37c88445e30d52c965bcb19b19fa710c3eb4fad9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 31 Mar 2020 07:57:53 +1100 Subject: [PATCH 24/73] mtr: use env for perl On FreeBSD, perl isn't in /usr/bin, its in /usr/local/bin or elsewhere in the path. Like storage/{maria/unittest/,}ma_test_* , we use /usr/bin/env to find perl and run it. --- cmake/configure.pl | 2 +- dbug/dbug_add_tags.pl | 2 +- dbug/remove_function_from_trace.pl | 2 +- debian/additions/mysqlreport | 2 +- extra/yassl/include/openssl/generate_prefix_files.pl | 2 +- mysql-test/lib/process-purecov-annotations.pl | 2 +- mysql-test/lib/t/SafeProcessStress.pl | 2 +- mysql-test/lib/t/copytree.t | 2 +- mysql-test/lib/t/dummyd.pl | 2 +- mysql-test/lib/t/rmtree.t | 2 +- mysql-test/lib/t/testMyConfig.t | 2 +- mysql-test/lib/t/testMyConfigFactory.t | 2 +- mysql-test/lib/t/test_child.pl | 2 +- mysql-test/lib/v1/mysql-test-run.pl | 2 +- mysql-test/mtr.out-of-source | 2 +- mysql-test/mysql-stress-test.pl | 2 +- mysql-test/mysql-test-run.pl | 2 +- mysql-test/std_data/checkDBI_DBD-mysql.pl | 2 +- mysql-test/suite/engines/rr_trx/run_stress_tx_rr.pl | 2 +- mysql-test/suite/funcs_1/lib/DataGen_local.pl | 2 +- mysql-test/suite/funcs_1/lib/DataGen_modify.pl | 2 +- mysql-test/suite/funcs_2/lib/gen_charset_utf8.pl | 2 +- mysql-test/suite/rpl/extension/checksum.pl | 2 +- plugin/handler_socket/client/hspool_test.pl | 2 +- plugin/handler_socket/client/hstest.pl | 2 +- .../perl-Net-HandlerSocket/lib/Net/HandlerSocket/Pool.pm | 2 +- plugin/handler_socket/regtest/test_01_lib/test01.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test02.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test03.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test04.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test05.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test06.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test07.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test08.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test09.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test10.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test11.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test12.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test13.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test14.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test15.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test16.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test17.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test18.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test19.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test20.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test21.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test22.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test23.pl | 2 +- plugin/handler_socket/regtest/test_01_lib/test24.pl | 2 +- scripts/convert-debug-for-diff.sh | 2 +- scripts/mytop.sh | 2 +- sql-bench/as3ap.sh | 2 +- sql-bench/bench-count-distinct.sh | 2 +- sql-bench/bench-init.pl.sh | 2 +- sql-bench/compare-results.sh | 2 +- sql-bench/copy-db.sh | 2 +- sql-bench/crash-me.sh | 2 +- sql-bench/graph-compare-results.sh | 2 +- sql-bench/innotest1.sh | 2 +- sql-bench/innotest1a.sh | 2 +- sql-bench/innotest1b.sh | 2 +- sql-bench/innotest2.sh | 2 +- sql-bench/innotest2a.sh | 2 +- sql-bench/innotest2b.sh | 2 +- sql-bench/run-all-tests.sh | 2 +- sql-bench/server-cfg.sh | 2 +- sql-bench/test-ATIS.sh | 2 +- sql-bench/test-alter-table.sh | 2 +- sql-bench/test-big-tables.sh | 2 +- sql-bench/test-connect.sh | 2 +- sql-bench/test-create.sh | 2 +- sql-bench/test-insert.sh | 2 +- sql-bench/test-select.sh | 2 +- sql-bench/test-table-elimination.sh | 2 +- sql-bench/test-transactions.sh | 2 +- sql-bench/test-wisconsin.sh | 2 +- storage/myisam/ftbench/Ecompare.pl | 2 +- storage/myisam/ftbench/Ecreate.pl | 2 +- storage/myisam/ftbench/Ereport.pl | 2 +- tests/big_record.pl | 2 +- tests/drop_test.pl | 2 +- tests/export.pl | 2 +- tests/fork2_test.pl | 2 +- tests/fork_big.pl | 2 +- tests/fork_big2.pl | 2 +- tests/grant.pl | 2 +- tests/index_corrupt.pl | 2 +- tests/insert_and_repair.pl | 2 +- tests/lock_test.pl | 2 +- tests/mail_to_db.pl | 2 +- tests/pmail.pl | 2 +- tests/rename_test.pl | 2 +- tests/table_types.pl | 2 +- tests/test_delayed_insert.pl | 2 +- tests/truncate.pl | 2 +- unittest/unit.pl | 2 +- 97 files changed, 97 insertions(+), 97 deletions(-) diff --git a/cmake/configure.pl b/cmake/configure.pl index 70122664ad4..45847aa576f 100644 --- a/cmake/configure.pl +++ b/cmake/configure.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved. # diff --git a/dbug/dbug_add_tags.pl b/dbug/dbug_add_tags.pl index 7be8fb9b18d..f117bdcd65b 100755 --- a/dbug/dbug_add_tags.pl +++ b/dbug/dbug_add_tags.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # Copyright (c) 2002 MySQL AB, 2009 Sun Microsystems, Inc. # Use is subject to license terms. diff --git a/dbug/remove_function_from_trace.pl b/dbug/remove_function_from_trace.pl index 380df168caf..67d7fa54b6a 100755 --- a/dbug/remove_function_from_trace.pl +++ b/dbug/remove_function_from_trace.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl die < Date: Mon, 22 Jun 2020 12:18:12 +0300 Subject: [PATCH 25/73] MDEV-20928 : Galera test failure on galera.galera_var_innodb_disallow_writes: Result length mismatch Add wait_conditions to force desired execution. --- mysql-test/suite/galera/disabled.def | 1 - .../galera_var_innodb_disallow_writes.result | 18 +++++++------- .../t/galera_var_innodb_disallow_writes.test | 24 ++++++++++++------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 0cbde628e45..125ea03474f 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -23,7 +23,6 @@ galera_parallel_simple : MDEV-20318 galera.galera_parallel_simple fails galera_shutdown_nonprim : MDEV-21493 galera.galera_shutdown_nonprim galera_ssl_upgrade : MDEV-19950 Galera test failure on galera_ssl_upgrade galera_sst_mariabackup_encrypt_with_key : MDEV-21484 galera_sst_mariabackup_encrypt_with_key -galera_var_innodb_disallow_writes : MDEV-20928 galera.galera_var_innodb_disallow_writes galera_var_node_address : MDEV-20485 Galera test failure galera_wan : MDEV-17259 Test failure on galera.galera_wan partition : MDEV-19958 Galera test failure on galera.partition diff --git a/mysql-test/suite/galera/r/galera_var_innodb_disallow_writes.result b/mysql-test/suite/galera/r/galera_var_innodb_disallow_writes.result index 4db4e539c50..54cebbee40c 100644 --- a/mysql-test/suite/galera/r/galera_var_innodb_disallow_writes.result +++ b/mysql-test/suite/galera/r/galera_var_innodb_disallow_writes.result @@ -1,3 +1,4 @@ +connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; connection node_1a; SET SESSION wsrep_sync_wait = 0; connection node_1; @@ -5,19 +6,20 @@ CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB; SET GLOBAL innodb_disallow_writes=ON; INSERT INTO t1 VALUES (1);; connection node_1a; -SELECT COUNT(*) = 1 FROM t1; -COUNT(*) = 1 +SELECT COUNT(*) AS EXPECT_0 FROM t1; +EXPECT_0 0 -SELECT COUNT(*) = 1 FROM t1; -COUNT(*) = 1 +SELECT COUNT(*) AS EXPECT_0 FROM t1; +EXPECT_0 0 SET GLOBAL innodb_disallow_writes=OFF; connection node_1; -SELECT COUNT(*) = 1 FROM t1; -COUNT(*) = 1 +SELECT COUNT(*) AS EXPECT_1 FROM t1; +EXPECT_1 1 connection node_2; -SELECT COUNT(*) = 1 FROM t1; -COUNT(*) = 1 +SELECT COUNT(*) AS EXPECT_1 FROM t1; +EXPECT_1 1 DROP TABLE t1; +disconnect node_1a; diff --git a/mysql-test/suite/galera/t/galera_var_innodb_disallow_writes.test b/mysql-test/suite/galera/t/galera_var_innodb_disallow_writes.test index a1154ebce22..3754cff0123 100644 --- a/mysql-test/suite/galera/t/galera_var_innodb_disallow_writes.test +++ b/mysql-test/suite/galera/t/galera_var_innodb_disallow_writes.test @@ -3,12 +3,9 @@ # --source include/galera_cluster.inc ---source include/have_innodb.inc # Open a separate connection to be used to run SHOW PROCESSLIST ---let $galera_connection_name = node_1a ---let $galera_server_number = 1 ---source include/galera_connect.inc +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 --connection node_1a SET SESSION wsrep_sync_wait = 0; @@ -18,18 +15,27 @@ SET GLOBAL innodb_disallow_writes=ON; --send INSERT INTO t1 VALUES (1); --connection node_1a -SELECT COUNT(*) = 1 FROM t1; -let $wait_condition = SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO = 'INSERT INTO t1 VALUES (1)' AND State = 'query end'; +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; --source include/wait_condition.inc -SELECT COUNT(*) = 1 FROM t1; +SELECT COUNT(*) AS EXPECT_0 FROM t1; +let $wait_condition = SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO = 'INSERT INTO t1 VALUES (1)'; +--source include/wait_condition.inc +SELECT COUNT(*) AS EXPECT_0 FROM t1; SET GLOBAL innodb_disallow_writes=OFF; --connection node_1 --reap -SELECT COUNT(*) = 1 FROM t1; +SELECT COUNT(*) AS EXPECT_1 FROM t1; --connection node_2 -SELECT COUNT(*) = 1 FROM t1; +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc +SELECT COUNT(*) AS EXPECT_1 FROM t1; DROP TABLE t1; + +--disconnect node_1a + From 5d7e067cce8577e6a8b3764d634a95f17763c3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 17 Jun 2020 12:58:33 +0300 Subject: [PATCH 26/73] MDEV-22125 : galera.galera_drop_multi MTR failed: InnoDB: MySQL is trying to drop database `fts`.`` though there are still open handles MDEV-22140 galera.galera_drop_database MTR failed: InnoDB: MySQL is trying to drop database `fts`.`` though there are still open handles Add wait conditions to wait that all operations are done in both nodes. --- .../galera/r/galera_drop_database.result | 25 +++++++++++++++++++ .../suite/galera/t/galera_drop_database.test | 20 +++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_drop_database.result b/mysql-test/suite/galera/r/galera_drop_database.result index 86135625a71..914cdffceed 100644 --- a/mysql-test/suite/galera/r/galera_drop_database.result +++ b/mysql-test/suite/galera/r/galera_drop_database.result @@ -11,9 +11,33 @@ INSERT INTO fts_t2 (f2) SELECT 'foobarbaz' FROM ten AS a1, ten AS a2, ten AS a3; DROP TABLE ten; UPDATE fts_t1 SET f2 = 'abcd'; UPDATE fts_t2 SET f2 = 'efjh'; +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t1; +EXPECT_1000 +1000 +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t2; +EXPECT_1000 +1000 +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t1 where f2 = 'abcd'; +EXPECT_1000 +1000 +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t2 where f2 = 'efjh'; +EXPECT_1000 +1000 connection node_2; connection node_1; connection node_2; +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t1; +EXPECT_1000 +1000 +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t2; +EXPECT_1000 +1000 +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t1 where f2 = 'abcd'; +EXPECT_1000 +1000 +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t2 where f2 = 'efjh'; +EXPECT_1000 +1000 connection node_1; USE fts; DROP TABLE fts_t1; @@ -21,3 +45,4 @@ DROP TABLE fts_t2; SHOW TABLES; Tables_in_fts DROP DATABASE fts; +connection node_2; diff --git a/mysql-test/suite/galera/t/galera_drop_database.test b/mysql-test/suite/galera/t/galera_drop_database.test index 12d9efea2f9..8dc73c1ce38 100644 --- a/mysql-test/suite/galera/t/galera_drop_database.test +++ b/mysql-test/suite/galera/t/galera_drop_database.test @@ -23,6 +23,10 @@ INSERT INTO fts_t2 (f2) SELECT 'foobarbaz' FROM ten AS a1, ten AS a2, ten AS a3; DROP TABLE ten; UPDATE fts_t1 SET f2 = 'abcd'; UPDATE fts_t2 SET f2 = 'efjh'; +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t1; +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t2; +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t1 where f2 = 'abcd'; +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t2 where f2 = 'efjh'; # Restart the second node: --connection node_2 @@ -34,6 +38,14 @@ UPDATE fts_t2 SET f2 = 'efjh'; --connection node_2 --source include/wait_until_ready.inc +--let $wait_condition = SELECT COUNT(*) = 1000 FROM fts_t1 where f2 = 'abcd'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1000 FROM fts_t2 where f2 = 'efjh'; +--source include/wait_condition.inc +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t1; +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t2; +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t1 where f2 = 'abcd'; +SELECT COUNT(*) AS EXPECT_1000 FROM fts_t2 where f2 = 'efjh'; # Drop the tables and database after nodes restarted: --connection node_1 @@ -43,5 +55,13 @@ DROP TABLE fts_t2; SHOW TABLES; DROP DATABASE fts; +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'fts_t1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'fts_t2'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'fts'; +--source include/wait_condition.inc + # Restore original auto_increment_offset values. --source include/auto_increment_offset_restore.inc From 3ddb0805361834988acfc1e969b38dc1af19b622 Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko Date: Sun, 14 Jun 2020 22:13:45 +0300 Subject: [PATCH 27/73] Fix include statements in galera_ipv6_mariabackup_section and galera_ipv6_mariabackup MTR tests --- mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test | 2 +- .../suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test index 808dad0dbb2..863ce65b579 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test @@ -1,7 +1,7 @@ --source include/galera_cluster.inc --source include/check_ipv6.inc --source include/have_innodb.inc ---source include/have_mariabackup.inc +--source ../galera/include/have_mariabackup.inc # Confirm that initial handshake happened over ipv6 diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test index 72f4fde299b..78fcf0873fc 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test @@ -1,7 +1,7 @@ --source include/galera_cluster.inc --source include/check_ipv6.inc --source include/have_innodb.inc ---source include/have_mariabackup.inc +--source ../galera/include/have_mariabackup.inc # Confirm that initial handshake happened over ipv6 From 51c8289ed6fab4c2495289172b1dd0a2b51914f2 Mon Sep 17 00:00:00 2001 From: MikkoJaakola Date: Mon, 15 Jun 2020 16:39:41 +0300 Subject: [PATCH 28/73] MDEV-21759 galera.galera_parallel_autoinc_manytrx sporadic failures. The galera.galera_parallel_autoinc_manytrx mtr test opens and runs test scenario through 3 connections to node 1 and one connection to node 2. In the test initialization phase, the test creates two tables 't1' and 'ten' and then creates a stored procedure 'p1' to operate on these tables. These 3 create DDL statements are issued through same connection to node 1. In the next test phase, the mtr script uses send command to launch the call for the p1 stored procedure through all 3 connections to node 1 and through one connection to node 2. As the mtr send command is asynchronous, this test phase is non blocking and fast operation. Now, if the replication between nodes is slow, it may happen that the initialization phase DDL statements have not been received or have not been fully applied in node 2. Therefore there is no guarantee that the test tables and the stored procedure have been created in node 2. Yet, the test is trying to call p1 in node 2. In the failure case error logs, there is error message "MTR failed: query 'reap' failed: 1305: PROCEDURE test.p1 does not exist" The reap command through connection to node 2, is the first place where test execution may observe that test tables and/or stored procedure are not yet created in node 2. The fix in this commit adds a wait condition in connection to node 2, to wait until the stored procedure is created before calling the stored procedure. The wait is implemented by looking in information_schema.routines for the p1 stored procedure. --- .../suite/galera/r/galera_parallel_autoinc_manytrx.result | 2 ++ mysql-test/suite/galera/t/galera_parallel_autoinc_manytrx.test | 2 ++ 2 files changed, 4 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_parallel_autoinc_manytrx.result b/mysql-test/suite/galera/r/galera_parallel_autoinc_manytrx.result index 0fd0f0b505a..445b818746a 100644 --- a/mysql-test/suite/galera/r/galera_parallel_autoinc_manytrx.result +++ b/mysql-test/suite/galera/r/galera_parallel_autoinc_manytrx.result @@ -16,6 +16,8 @@ COMMIT; SET current_num = current_num + 1; END WHILE; END| +call p1(1000); +connection node_1; connection node_1a; connection node_1b; connection node_2; diff --git a/mysql-test/suite/galera/t/galera_parallel_autoinc_manytrx.test b/mysql-test/suite/galera/t/galera_parallel_autoinc_manytrx.test index b82a160657d..587fbe10fe4 100644 --- a/mysql-test/suite/galera/t/galera_parallel_autoinc_manytrx.test +++ b/mysql-test/suite/galera/t/galera_parallel_autoinc_manytrx.test @@ -57,6 +57,8 @@ send call p1(1000); --connection node_2 --disable_query_log +--let $wait_condition = select count(*)=1 from information_schema.routines WHERE routine_name='p1'; +--source include/wait_condition.inc send call p1(1000); --connection node_1 From eba918977793f0995d2f4f7707fc5dd891da4064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 22 Jun 2020 13:25:25 +0300 Subject: [PATCH 29/73] Test case cleanups. --- .../suite/galera/galera_3nodes_as_slave.cnf | 3 + .../galera/r/galera_as_slave_nonprim.result | 15 +++ mysql-test/suite/galera/suite.pm | 1 + .../suite/galera/t/galera_as_slave_gtid.inc | 93 +++++++++++++++++++ .../galera/t/galera_as_slave_gtid_myisam.test | 3 +- 5 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/galera/t/galera_as_slave_gtid.inc diff --git a/mysql-test/suite/galera/galera_3nodes_as_slave.cnf b/mysql-test/suite/galera/galera_3nodes_as_slave.cnf index ac1ca34e242..28fa5095a56 100644 --- a/mysql-test/suite/galera/galera_3nodes_as_slave.cnf +++ b/mysql-test/suite/galera/galera_3nodes_as_slave.cnf @@ -18,6 +18,7 @@ server-id=1 #sst_port=@OPT.port log-slave-updates +wsrep-on=1 innodb-autoinc-lock-mode=2 default-storage-engine=innodb @@ -39,6 +40,7 @@ server-id=2 #sst_port=@OPT.port log-slave-updates +wsrep-on=1 innodb-autoinc-lock-mode=2 default-storage-engine=innodb @@ -60,6 +62,7 @@ server-id=3 #sst_port=@OPT.port log-slave-updates +wsrep-on=1 innodb-autoinc-lock-mode=2 default-storage-engine=innodb diff --git a/mysql-test/suite/galera/r/galera_as_slave_nonprim.result b/mysql-test/suite/galera/r/galera_as_slave_nonprim.result index 365ea31f292..6d0bcd4204a 100644 --- a/mysql-test/suite/galera/r/galera_as_slave_nonprim.result +++ b/mysql-test/suite/galera/r/galera_as_slave_nonprim.result @@ -1,13 +1,27 @@ +connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connection node_2; START SLAVE; SET SESSION wsrep_sync_wait = 0; +connection node_1; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB; +connection node_2; SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1'; +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connection node_3; +connection node_1; INSERT INTO t1 VALUES (1),(2),(3),(4),(5); +connection node_2; +connection node_3; expected_error 1 +connection node_2; SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0'; +connection node_3; +connection node_2; START SLAVE; +connection node_1; DROP TABLE t1; +connection node_2; STOP SLAVE; RESET SLAVE ALL; CALL mtr.add_suppression("Slave SQL: Error 'Unknown command' on query"); @@ -15,4 +29,5 @@ CALL mtr.add_suppression("Slave: Unknown command Error_code: 1047"); CALL mtr.add_suppression("Transport endpoint is not connected"); CALL mtr.add_suppression("Slave SQL: Error in Xid_log_event: Commit could not be completed, 'Deadlock found when trying to get lock; try restarting transaction', Error_code: 1213"); CALL mtr.add_suppression("Slave SQL: Node has dropped from cluster, Error_code: 1047"); +connection node_1; RESET MASTER; diff --git a/mysql-test/suite/galera/suite.pm b/mysql-test/suite/galera/suite.pm index cc35f65423c..8ef5965834f 100644 --- a/mysql-test/suite/galera/suite.pm +++ b/mysql-test/suite/galera/suite.pm @@ -66,6 +66,7 @@ push @::global_suppressions, qr(WSREP: Failed to remove page file .*), qr(WSREP: wsrep_sst_method is set to 'mysqldump' yet mysqld bind_address is set to .*), qr|WSREP: Sending JOIN failed: -107 \(Transport endpoint is not connected\). Will retry in new primary component.|, + qr|WSREP: Trying to continue unpaused monitor|, ); bless { }; diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid.inc b/mysql-test/suite/galera/t/galera_as_slave_gtid.inc new file mode 100644 index 00000000000..461e7833a94 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid.inc @@ -0,0 +1,93 @@ +# +# Test Galera as a slave to a MariaDB master using GTIDs +# +# suite/galera/galera_2nodes_as_slave.cnf describes the setup of the nodes +# suite/galera/t/galera_as_slave_gtid.cnf has the GTID options +# +# In addition to performing DDL and DML, we check that the gtid of the master is preserved inside the cluster +# + +--source include/have_innodb.inc +--source include/galera_cluster.inc + +# As node #3 is not a Galera node, and galera_cluster.inc does not open connetion to it +# we open the node_3 connection here +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 + +--connection node_2 +--disable_query_log +--eval CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_USER='root', MASTER_PORT=$NODE_MYPORT_3; +--enable_query_log +START SLAVE; + +--connection node_3 +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES(1); + +SELECT LENGTH(@@global.gtid_binlog_state) > 1; +--let $gtid_binlog_state_node1 = `SELECT @@global.gtid_binlog_state;` + +--connection node_1 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc + +--disable_query_log + +--eval SELECT '$gtid_binlog_state_node1' = @@global.gtid_binlog_state AS gtid_binlog_state_equal; +#--eval SELECT GTID_SUBSET('$gtid_executed_node1', @@global.gtid_executed) AS gtid_executed_equal; + +--enable_query_log + +--connection node_1 +SELECT COUNT(*) = 1 FROM t1; + +--disable_query_log +--eval SELECT '$gtid_binlog_state_node1' = @@global.gtid_binlog_state AS gtid_binlog_state_equal; +#--eval SELECT GTID_SUBSET('$gtid_executed_node1', @@global.gtid_executed) AS gtid_executed_equal; +--enable_query_log + +--connection node_3 +DROP TABLE t1; + +# +# Unfortunately without the sleep below the following statement fails with "query returned no rows", which +# is difficult to understand given that it is an aggregate query. A "query execution was interrupted" +# warning is also reported by MTR, which is also weird. +# + +--sleep 1 + +--connection node_1 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +STOP SLAVE; +RESET SLAVE ALL; + +--echo #cleanup +--connection node_1 +set global wsrep_on=OFF; +reset master; +set global wsrep_on=ON; + +--connection node_2 +set global wsrep_on=OFF; +reset master; +set global wsrep_on=ON; + +--connection node_3 +reset master; diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test index 3710e6d65be..940c9c0667d 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test @@ -53,6 +53,8 @@ SELECT COUNT(*) = 0 FROM t1; --enable_query_log --echo #cleanup +--sleep 1 + --connection node_1 DROP TABLE t1; @@ -63,7 +65,6 @@ DROP TABLE t1; --connection node_3 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc - --connection node_1 reset master; From 7bd11fb46fad3d7db6acbc11ad86aed3c83381c6 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 23 Jun 2020 12:56:08 +0200 Subject: [PATCH 30/73] MDEV-22729: additional changes after merge --- mysql-test/suite/galera/t/galera_slave_replay.test | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_slave_replay.test b/mysql-test/suite/galera/t/galera_slave_replay.test index bac394baf64..f1500eeaeaa 100644 --- a/mysql-test/suite/galera/t/galera_slave_replay.test +++ b/mysql-test/suite/galera/t/galera_slave_replay.test @@ -7,6 +7,7 @@ # --source include/have_innodb.inc +--source include/have_log_bin.inc --source include/have_debug.inc --source include/have_debug_sync.inc --source include/galera_have_debug_sync.inc @@ -31,10 +32,10 @@ RESET MASTER; # -# nodes 1 and 2 form a galera cluster, node 2 operates as slave for native MariaDB naster in node 3 +# nodes 1 and 2 form a galera cluster, node 2 operates as slave for native MariaDB master in node 3 # --disable_query_log ---eval CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_USER='root', MASTER_PORT=$NODE_MYPORT_3; +--eval CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_USER='root', MASTER_PORT=$NODE_MYPORT_3; --enable_query_log START SLAVE; From e0793d386517f4ff9c0267830d558f91c75263aa Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Tue, 23 Jun 2020 13:42:11 +0200 Subject: [PATCH 31/73] Fix result of merge. --- sql/sql_class.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sql/sql_class.h b/sql/sql_class.h index 7ca3896a69d..4d14b42b065 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3408,11 +3408,7 @@ public: inline bool is_error() const { return m_stmt_da->is_error(); } /// Returns Diagnostics-area for the current statement. - Diagnostics_area *get_stmt_da() - { return m_stmt_da; } - - /// Returns Diagnostics-area for the current statement. - const Diagnostics_area *get_stmt_da() const + Diagnostics_area *get_stmt_da() const { return m_stmt_da; } /// Sets Diagnostics-area for the current statement. From 697273554f0952160baa10606c02be1f6cf3c6c5 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 16 Jun 2020 01:29:51 +0300 Subject: [PATCH 32/73] MDEV-22866: Crash in join optimizer with constant outer join nest Starting from 10.3, the optimizer is able to detect that entire outer join nests are constants (because of "Impossible ON") and remove them (see mark_join_nest_as_const) However, this was not properly accounted for in NESTED_JOIN structure and the way check_interleaving_with_nj() uses its n_tables member to check if the join prefix order is allowed. (The result was that the optimizer could conclude that no join prefix is allowed and fail an assertion) --- mysql-test/main/join_outer.result | 31 +++++++++++++++++++++++ mysql-test/main/join_outer.test | 35 ++++++++++++++++++++++++++ mysql-test/main/join_outer_jcl6.result | 31 +++++++++++++++++++++++ sql/sql_select.cc | 18 +++++++++---- sql/table.h | 6 +++-- 5 files changed, 114 insertions(+), 7 deletions(-) diff --git a/mysql-test/main/join_outer.result b/mysql-test/main/join_outer.result index 7a230ccd15a..796d01a0996 100644 --- a/mysql-test/main/join_outer.result +++ b/mysql-test/main/join_outer.result @@ -2752,3 +2752,34 @@ WHERE t3.pk IN (2); drop view v4; drop table t1,t2,t3,t4; SET optimizer_switch=@org_optimizer_switch; +# +# MDEV-22866: Crash in join optimizer with constant outer join nest +# +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (3),(4); +CREATE TABLE t3 (c INT, KEY(c)) ENGINE=MyISAM; +CREATE TABLE t4 (d INT, KEY(d)) ENGINE=MyISAM; +INSERT INTO t4 VALUES (5),(6); +CREATE TABLE t5 (e INT) ENGINE=MyISAM; +INSERT INTO t5 VALUES (7),(8); +CREATE TABLE t6 (f INT) ENGINE=MyISAM; +INSERT INTO t6 VALUES (9),(10); +SELECT * +FROM +t1 +LEFT JOIN ( +t2 LEFT JOIN ( +t3 JOIN +t4 ON t3.c = t4.d and t3.c >2 and t3.c<0 +) ON t2.b >= t4.d +) ON t1.a <= t2.b +LEFT JOIN t5 ON t2.b = t5.e +LEFT JOIN t6 ON t3.c = t6.f; +a b c d e f +1 3 NULL NULL NULL NULL +2 3 NULL NULL NULL NULL +1 4 NULL NULL NULL NULL +2 4 NULL NULL NULL NULL +drop table t1,t2,t3,t4,t5,t6; diff --git a/mysql-test/main/join_outer.test b/mysql-test/main/join_outer.test index 2e5fc65ebb6..f835d8af5a8 100644 --- a/mysql-test/main/join_outer.test +++ b/mysql-test/main/join_outer.test @@ -2252,3 +2252,38 @@ drop view v4; drop table t1,t2,t3,t4; SET optimizer_switch=@org_optimizer_switch; + +--echo # +--echo # MDEV-22866: Crash in join optimizer with constant outer join nest +--echo # + +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1),(2); + +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (3),(4); + +CREATE TABLE t3 (c INT, KEY(c)) ENGINE=MyISAM; + +CREATE TABLE t4 (d INT, KEY(d)) ENGINE=MyISAM; +INSERT INTO t4 VALUES (5),(6); + +CREATE TABLE t5 (e INT) ENGINE=MyISAM; +INSERT INTO t5 VALUES (7),(8); + +CREATE TABLE t6 (f INT) ENGINE=MyISAM; +INSERT INTO t6 VALUES (9),(10); + +SELECT * +FROM + t1 + LEFT JOIN ( + t2 LEFT JOIN ( + t3 JOIN + t4 ON t3.c = t4.d and t3.c >2 and t3.c<0 + ) ON t2.b >= t4.d + ) ON t1.a <= t2.b + LEFT JOIN t5 ON t2.b = t5.e + LEFT JOIN t6 ON t3.c = t6.f; + +drop table t1,t2,t3,t4,t5,t6; diff --git a/mysql-test/main/join_outer_jcl6.result b/mysql-test/main/join_outer_jcl6.result index d3276de88ea..4f66b004cfc 100644 --- a/mysql-test/main/join_outer_jcl6.result +++ b/mysql-test/main/join_outer_jcl6.result @@ -2759,3 +2759,34 @@ WHERE t3.pk IN (2); drop view v4; drop table t1,t2,t3,t4; SET optimizer_switch=@org_optimizer_switch; +# +# MDEV-22866: Crash in join optimizer with constant outer join nest +# +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (3),(4); +CREATE TABLE t3 (c INT, KEY(c)) ENGINE=MyISAM; +CREATE TABLE t4 (d INT, KEY(d)) ENGINE=MyISAM; +INSERT INTO t4 VALUES (5),(6); +CREATE TABLE t5 (e INT) ENGINE=MyISAM; +INSERT INTO t5 VALUES (7),(8); +CREATE TABLE t6 (f INT) ENGINE=MyISAM; +INSERT INTO t6 VALUES (9),(10); +SELECT * +FROM +t1 +LEFT JOIN ( +t2 LEFT JOIN ( +t3 JOIN +t4 ON t3.c = t4.d and t3.c >2 and t3.c<0 +) ON t2.b >= t4.d +) ON t1.a <= t2.b +LEFT JOIN t5 ON t2.b = t5.e +LEFT JOIN t6 ON t3.c = t6.f; +a b c d e f +1 3 NULL NULL NULL NULL +2 3 NULL NULL NULL NULL +1 4 NULL NULL NULL NULL +2 4 NULL NULL NULL NULL +drop table t1,t2,t3,t4,t5,t6; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1f35754a8fe..ee5144fafe6 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -15826,10 +15826,15 @@ static uint build_bitmap_for_nested_joins(List *join_list, /** - Set NESTED_JOIN::counter=0 in all nested joins in passed list. + Set NESTED_JOIN::counter and n_tables in all nested joins in passed list. - Recursively set NESTED_JOIN::counter=0 for all nested joins contained in - the passed join_list. + For all nested joins contained in the passed join_list (including its + children), set: + - nested_join->counter=0 + - nested_join->n_tables= {number of non-degenerate direct children}. + + Non-degenerate means non-const base table or a join nest that has a + non-degenerate child. @param join_list List of nested joins to process. It may also contain base tables which will be ignored. @@ -15852,8 +15857,11 @@ static uint reset_nj_counters(JOIN *join, List *join_list) if (!nested_join->n_tables) is_eliminated_nest= TRUE; } - if ((table->nested_join && !is_eliminated_nest) || - (!table->nested_join && (table->table->map & ~join->eliminated_tables))) + const table_map removed_tables= join->eliminated_tables | + join->const_table_map; + + if ((table->nested_join && !is_eliminated_nest) || + (!table->nested_join && (table->table->map & ~removed_tables))) n++; } DBUG_RETURN(n); diff --git a/sql/table.h b/sql/table.h index 2bad6cbf32f..d289b6e0ab2 100644 --- a/sql/table.h +++ b/sql/table.h @@ -2932,9 +2932,11 @@ typedef struct st_nested_join Before each use the counters are zeroed by reset_nj_counters. */ uint counter; + /* - Number of elements in join_list that were not (or contain table(s) that - weren't) removed by table elimination. + Number of elements in join_list that participate in the join plan choice: + - Base tables that were not removed by table elimination + - Join nests that were not removed by mark_join_nest_as_const */ uint n_tables; nested_join_map nj_map; /* Bit used to identify this nested join*/ From 8e58eeba789bc2760603dfdc07ed4d9e648c4dbb Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko Date: Wed, 29 Apr 2020 13:41:46 +0300 Subject: [PATCH 33/73] MTR tests to test Galera fix for node joining over several configuration changes. This requires Galera commit 065e484144c5999709ae8fd19844da72bb785073 --- .../r/galera_join_with_cc_A.result | 40 +++ .../r/galera_join_with_cc_B.result | 50 +++ .../r/galera_join_with_cc_C.result | 55 ++++ .../t/galera_join_with_cc_A.test | 262 +++++++++++++++ .../t/galera_join_with_cc_B.test | 273 ++++++++++++++++ .../t/galera_join_with_cc_C.test | 299 ++++++++++++++++++ 6 files changed, 979 insertions(+) create mode 100644 mysql-test/suite/galera_3nodes/r/galera_join_with_cc_A.result create mode 100644 mysql-test/suite/galera_3nodes/r/galera_join_with_cc_B.result create mode 100644 mysql-test/suite/galera_3nodes/r/galera_join_with_cc_C.result create mode 100644 mysql-test/suite/galera_3nodes/t/galera_join_with_cc_A.test create mode 100644 mysql-test/suite/galera_3nodes/t/galera_join_with_cc_B.test create mode 100644 mysql-test/suite/galera_3nodes/t/galera_join_with_cc_C.test diff --git a/mysql-test/suite/galera_3nodes/r/galera_join_with_cc_A.result b/mysql-test/suite/galera_3nodes/r/galera_join_with_cc_A.result new file mode 100644 index 00000000000..0461f1f1feb --- /dev/null +++ b/mysql-test/suite/galera_3nodes/r/galera_join_with_cc_A.result @@ -0,0 +1,40 @@ +CREATE TABLE t1 (pk INT PRIMARY KEY, node INT) ENGINE=innodb; +INSERT INTO t1 VALUES (1, 1); +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (2, 3); +SET wsrep_sync_wait = 0; +SET wsrep_on = OFF; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_shift_to_joining'; +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +INSERT INTO t1 VALUES (3, 2); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (4, 3); +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +INSERT INTO t1 VALUES (5, 2); +SET GLOBAL wsrep_provider_options = 'dbug=d,before_send_state_request'; +SET GLOBAL wsrep_provider_options = 'signal=after_shift_to_joining'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (6, 3); +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +INSERT INTO t1 VALUES (7, 2); +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (8, 3); +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +SET GLOBAL wsrep_provider_options = 'dbug=d,process_primary_configuration'; +SET GLOBAL wsrep_provider_options = 'signal=before_send_state_request'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +INSERT INTO t1 VALUES (9, 2); +SET GLOBAL wsrep_provider_options = 'signal=process_primary_configuration'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options = 'signal=process_primary_configuration'; +SET GLOBAL wsrep_provider_options = 'dbug='; +DROP TABLE t1; +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); diff --git a/mysql-test/suite/galera_3nodes/r/galera_join_with_cc_B.result b/mysql-test/suite/galera_3nodes/r/galera_join_with_cc_B.result new file mode 100644 index 00000000000..d878f60ca6b --- /dev/null +++ b/mysql-test/suite/galera_3nodes/r/galera_join_with_cc_B.result @@ -0,0 +1,50 @@ +CREATE TABLE t1 (pk INT PRIMARY KEY, node INT) ENGINE=innodb; +INSERT INTO t1 VALUES (1, 1); +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (2, 3); +SET wsrep_sync_wait = 0; +SET wsrep_on = OFF; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_shift_to_joining'; +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +INSERT INTO t1 VALUES (3, 2); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (4, 3); +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +INSERT INTO t1 VALUES (5, 2); +SET GLOBAL wsrep_provider_options = 'dbug=d,before_send_state_request'; +SET GLOBAL wsrep_provider_options = 'signal=after_shift_to_joining'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=before_send_state_request'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_debug_sync_waiters'; +VARIABLE_NAME VARIABLE_VALUE +WSREP_DEBUG_SYNC_WAITERS after_shift_to_joining +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (6, 3); +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +INSERT INTO t1 VALUES (7, 2); +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (8, 3); +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +SET GLOBAL wsrep_provider_options = 'dbug=d,process_primary_configuration'; +SET GLOBAL wsrep_provider_options = 'signal=after_shift_to_joining'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +INSERT INTO t1 VALUES (9, 2); +SET GLOBAL wsrep_provider_options = 'signal=process_primary_configuration'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_debug_sync_waiters'; +VARIABLE_NAME VARIABLE_VALUE +WSREP_DEBUG_SYNC_WAITERS process_primary_configuration +SET GLOBAL wsrep_provider_options = 'signal=process_primary_configuration'; +SET GLOBAL wsrep_provider_options = 'dbug='; +DROP TABLE t1; +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); diff --git a/mysql-test/suite/galera_3nodes/r/galera_join_with_cc_C.result b/mysql-test/suite/galera_3nodes/r/galera_join_with_cc_C.result new file mode 100644 index 00000000000..df0a924029c --- /dev/null +++ b/mysql-test/suite/galera_3nodes/r/galera_join_with_cc_C.result @@ -0,0 +1,55 @@ +CREATE TABLE t1 (pk INT PRIMARY KEY, node INT) ENGINE=innodb; +INSERT INTO t1 VALUES (1, 1); +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (2, 3); +SET wsrep_sync_wait = 0; +SET wsrep_on = OFF; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_shift_to_joining'; +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +INSERT INTO t1 VALUES (3, 2); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (4, 3); +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +INSERT INTO t1 VALUES (5, 2); +SET GLOBAL wsrep_provider_options = 'dbug=d,before_send_state_request'; +SET GLOBAL wsrep_provider_options = 'signal=after_shift_to_joining'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +SET GLOBAL wsrep_provider_options = 'dbug=d,after_shift_to_joining'; +SET GLOBAL wsrep_provider_options = 'signal=before_send_state_request'; +4 +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_debug_sync_waiters'; +VARIABLE_NAME VARIABLE_VALUE +WSREP_DEBUG_SYNC_WAITERS +INSERT INTO t1 VALUES (6, 3); +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +INSERT INTO t1 VALUES (7, 2); +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,process_primary_configuration'; +SET GLOBAL wsrep_provider_options = 'signal=after_shift_to_joining'; +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; +INSERT INTO t1 VALUES (8, 3); +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +INSERT INTO t1 VALUES (9, 2); +SET GLOBAL wsrep_provider_options = 'signal=process_primary_configuration'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=process_primary_configuration'; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 0; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=after_shift_to_joining'; +DROP TABLE t1; +call mtr.add_suppression("WSREP: Send action {\(.*\), STATE_REQUEST} returned -107 \\(Transport endpoint is not connected\\)"); +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); diff --git a/mysql-test/suite/galera_3nodes/t/galera_join_with_cc_A.test b/mysql-test/suite/galera_3nodes/t/galera_join_with_cc_A.test new file mode 100644 index 00000000000..e181b6831af --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/galera_join_with_cc_A.test @@ -0,0 +1,262 @@ +# +# Tests handling of several configuration changes while a joiner gets +# state transfer +# +# Variant A: sending of state transfer request delayed until two more +# primary configuration changes happen +# +# Refs codersihp/galera-bugs#454 +# +--source include/have_innodb.inc +--source include/galera_cluster.inc +--source suite/galera/include/galera_have_debug_sync.inc + +--let $galera_connection_name = node_3 +--let $galera_server_number = 3 +--source include/galera_connect.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +CREATE TABLE t1 (pk INT PRIMARY KEY, node INT) ENGINE=innodb; +INSERT INTO t1 VALUES (1, 1); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc + +# +# Isolate node_1 and update cluster state to force node 1 into joiner mode +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (2, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 2 FROM t1; +--source include/wait_condition.inc + +# +# Now reconnect node_1 but first make it block before sending state transfer +# request +# +# THIS IS PC1 +# +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +SET wsrep_sync_wait = 0; +SET wsrep_on = OFF; +--let $galera_sync_point = after_shift_to_joining +--source include/galera_set_sync_point.inc + +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (3, 2); + +--connection node_1a +--source include/galera_wait_sync_point.inc +# +# At this point every node thinks that node_1 is in a JOINER state +# + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 3 FROM t1; +--source include/wait_condition.inc + +# +# Now that node_1 sent state request and became JOINER isolate node_1 again +# and commit one more action, so that node_1 loses JOINER state +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (4, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 4 FROM t1; +--source include/wait_condition.inc + +# +# Reconnect node_1 again +# +# THIS IS PC2 +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +# +# After this point node_1 is no longer JOINER and is required to start the +# whole procedure over because it missed some actions (4th insert into t1) +# + +INSERT INTO t1 VALUES (5, 2); + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 5 FROM t1; +--source include/wait_condition.inc + +# +# Now let node_1 continue with IST and finish processing PC1, but make it +# block when processing PC2 just before sending state transfer request +# +--connection node_1a +--let $galera_sync_point = before_send_state_request +--source include/galera_set_sync_point.inc +--let $galera_sync_point = after_shift_to_joining +--source include/galera_signal_sync_point.inc +--let $galera_sync_point = before_send_state_request +--source include/galera_wait_sync_point.inc + +# since PC1 has been processed node_1 must have 3 rows in t1 +# 2 were there before PC1 and one was added while in PC1 +--let $wait_condition = SELECT COUNT(*) = 3 FROM t1; +--source include/wait_condition.inc + +# +# Now disconnect and reconnect node_1 again to get PC3 +# It still is blocked before sending state transfer request in PC2. +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (6, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 6 FROM t1; +--source include/wait_condition.inc + +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +# +# This is PC3 +# + +INSERT INTO t1 VALUES (7, 2); + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 7 FROM t1; +--source include/wait_condition.inc + +# +# Now disconnect and reconnect node_1 again to get PC4 and allow node_1 +# to send state transfer request to be delivered in PC4 (and thus get +# updated to PC4 seqno in state transfer. +# Note that node_1 still processes PC2. +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (8, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 8 FROM t1; +--source include/wait_condition.inc + +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +# +# This is PC4. node_1 is still processing PC2, waiting to send state trasfer +# request +# +--connection node_1a +--let $galera_sync_point = process_primary_configuration +--source include/galera_set_sync_point.inc +--let $galera_sync_point = before_send_state_request +--source include/galera_signal_sync_point.inc +# sent STR from PC2 into PC4 + +--let $galera_sync_point = process_primary_configuration +--source include/galera_wait_sync_point.inc +# +# Now node_1 is processing PC3, but should have completed state transfer from +# PC4 and thus must have 8 rows in t1 +# +--let $wait_condition = SELECT COUNT(*) = 8 FROM t1; +--source include/wait_condition.inc + +--connection node_2 +INSERT INTO t1 VALUES (9, 2); + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 9 FROM t1; +--source include/wait_condition.inc + +--connection node_1a +#--let $galera_sync_point = process_primary_configuration +--source include/galera_signal_sync_point.inc +--source include/galera_wait_sync_point.inc +# +# Now node_1 is processing PC4, still must have 8 rows in t1 +# +--let $wait_condition = SELECT COUNT(*) = 8 FROM t1; +--source include/wait_condition.inc + +#--let $galera_sync_point = process_primary_configuration +--source include/galera_signal_sync_point.inc +--source include/galera_clear_sync_point.inc + +--let $wait_condition = SELECT COUNT(*) = 9 FROM t1; +--source include/wait_condition.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +DROP TABLE t1; + +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); + +--connection node_2 +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); + +--connection node_3 +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); diff --git a/mysql-test/suite/galera_3nodes/t/galera_join_with_cc_B.test b/mysql-test/suite/galera_3nodes/t/galera_join_with_cc_B.test new file mode 100644 index 00000000000..7618f5a112c --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/galera_join_with_cc_B.test @@ -0,0 +1,273 @@ +# +# Tests handling of several configuration changes while a joiner gets +# state transfer +# +# Variant B: sending of state transfer request is immediate but completion +# of IST delayed until two more primary configuration changes happen +# +# Refs codersihp/galera-bugs#454 +# +--source include/have_innodb.inc +--source include/galera_cluster.inc +--source suite/galera/include/galera_have_debug_sync.inc + +--let $galera_connection_name = node_3 +--let $galera_server_number = 3 +--source include/galera_connect.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +CREATE TABLE t1 (pk INT PRIMARY KEY, node INT) ENGINE=innodb; +INSERT INTO t1 VALUES (1, 1); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc + +# +# Isolate node_1 and update cluster state to force node 1 into joiner mode +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (2, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 2 FROM t1; +--source include/wait_condition.inc + +# +# Now reconnect node_1 but first make it block before sending state transfer +# request +# +# THIS IS PC1 +# +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +SET wsrep_sync_wait = 0; +SET wsrep_on = OFF; +--let $galera_sync_point = after_shift_to_joining +--source include/galera_set_sync_point.inc + +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (3, 2); + +--connection node_1a +--source include/galera_wait_sync_point.inc +# +# At this point every node thinks that node_1 is in a JOINER state +# + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 3 FROM t1; +--source include/wait_condition.inc + +# +# Now that node_1 sent state request and became JOINER isolate node_1 again +# and commit one more action, so that node_1 loses JOINER state +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (4, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 4 FROM t1; +--source include/wait_condition.inc + +# +# Reconnect node_1 again +# +# THIS IS PC2 +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +# +# After this point node_1 is no longer JOINER and is required to start the +# whole procedure over because it missed some actions (4th insert into t1) +# + +INSERT INTO t1 VALUES (5, 2); + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 5 FROM t1; +--source include/wait_condition.inc + +# +# Now let node_1 continue with IST and finish processing PC1, but make it +# block when processing PC2 right after progressing to JOINER state and +# before IST happens. +# +--connection node_1a +--let $galera_sync_point = before_send_state_request +--source include/galera_set_sync_point.inc +--let $galera_sync_point = after_shift_to_joining +--source include/galera_signal_sync_point.inc +--let $galera_sync_point = before_send_state_request +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc +# Here node_1 is processing PC2 just before sending state request + +# since PC1 has been processed node_1 must have 3 rows in t1 +# 2 were there before PC1 and one was added while in PC1 +--let $wait_condition = SELECT COUNT(*) = 3 FROM t1; +--source include/wait_condition.inc + +# Proceed to sending state transfer request and block right after +--source include/galera_signal_sync_point.inc # before_send_state_request +--let $galera_sync_point = after_shift_to_joining +--source include/galera_wait_sync_point.inc +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_debug_sync_waiters'; + +# +# Now disconnect and reconnect node_1 again to get PC3 +# It is blocked right after shifting to JOINING state. +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (6, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 6 FROM t1; +--source include/wait_condition.inc + +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +# +# This is PC3 +# + +INSERT INTO t1 VALUES (7, 2); + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 7 FROM t1; +--source include/wait_condition.inc + +# +# Now disconnect and reconnect node_1 again to get PC4 and allow node_1 +# to continue with IST. +# Note that node_1 still processes PC2 and is joining. +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (8, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 8 FROM t1; +--source include/wait_condition.inc + +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +# +# This is PC4. node_1 is still processing PC2, waiting to send state trasfer +# request +# +--connection node_1a +--let $galera_sync_point = process_primary_configuration +--source include/galera_set_sync_point.inc +--let $galera_sync_point = after_shift_to_joining +--source include/galera_signal_sync_point.inc +# continue with IST prepared for in PC2 + +--let $galera_sync_point = process_primary_configuration +--source include/galera_wait_sync_point.inc +# +# Now node_1 is processing PC3, and should have finished state transfer +# State tranfer request was dilivered before PC3, so node_1 should have +# received IST up to 4 rows in t1 (what was there before PC2) plus one more +# INSERT while in PC2. +# +--let $wait_condition = SELECT COUNT(*) = 5 FROM t1; +--source include/wait_condition.inc + +--connection node_2 +INSERT INTO t1 VALUES (9, 2); + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 9 FROM t1; +--source include/wait_condition.inc + +--connection node_1a +#--let $galera_sync_point = process_primary_configuration +--source include/galera_signal_sync_point.inc +--source include/galera_wait_sync_point.inc +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_debug_sync_waiters'; +# +# Now node_1 is processing PC4, still must have 8 rows in t1 +# +--let $wait_condition = SELECT COUNT(*) = 8 FROM t1; +--source include/wait_condition.inc + +#--let $galera_sync_point = process_primary_configuration +--source include/galera_signal_sync_point.inc +--source include/galera_clear_sync_point.inc + +--let $wait_condition = SELECT COUNT(*) = 9 FROM t1; +--source include/wait_condition.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +DROP TABLE t1; + +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); + +--connection node_2 +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); + +--connection node_3 +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); diff --git a/mysql-test/suite/galera_3nodes/t/galera_join_with_cc_C.test b/mysql-test/suite/galera_3nodes/t/galera_join_with_cc_C.test new file mode 100644 index 00000000000..68dea95854c --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/galera_join_with_cc_C.test @@ -0,0 +1,299 @@ +# +# Tests handling of several configuration changes while a joiner gets +# state transfer +# +# Variant C: sending of state transfer request is scheduled while in non-PRIM +# +# Refs codersihp/galera-bugs#454 +# +--source include/have_innodb.inc +--source include/galera_cluster.inc +--source suite/galera/include/galera_have_debug_sync.inc + +--let $galera_connection_name = node_3 +--let $galera_server_number = 3 +--source include/galera_connect.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +CREATE TABLE t1 (pk INT PRIMARY KEY, node INT) ENGINE=innodb; +INSERT INTO t1 VALUES (1, 1); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc + +# +# Isolate node_1 and update cluster state to force node 1 into joiner mode +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (2, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 2 FROM t1; +--source include/wait_condition.inc + +# +# Now reconnect node_1 but first make it block before sending state transfer +# request +# +# THIS IS PC1 +# +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +SET wsrep_sync_wait = 0; +SET wsrep_on = OFF; +--let $galera_sync_point = after_shift_to_joining +--source include/galera_set_sync_point.inc + +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (3, 2); + +--connection node_1a +--source include/galera_wait_sync_point.inc +# +# At this point every node thinks that node_1 is in a JOINER state +# + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 3 FROM t1; +--source include/wait_condition.inc + +# +# Now that node_1 sent state request and became JOINER isolate node_1 again +# and commit one more action, so that node_1 loses JOINER state +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (4, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 4 FROM t1; +--source include/wait_condition.inc + +# +# Reconnect node_1 again +# +# THIS IS PC2 +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +# +# After this point node_1 is no longer JOINER and is required to start the +# whole procedure over because it missed some actions (4th insert into t1) +# + +INSERT INTO t1 VALUES (5, 2); + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 5 FROM t1; +--source include/wait_condition.inc + +# +# Now let node_1 continue with IST and finish processing PC1, but make it +# block when processing PC2 right after progressing to JOINER state and +# before IST happens. +# +--connection node_1a +--let $galera_sync_point = before_send_state_request +--source include/galera_set_sync_point.inc +--let $galera_sync_point = after_shift_to_joining +--source include/galera_signal_sync_point.inc +--let $galera_sync_point = before_send_state_request +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc +# Here we are processing PC2 just before sending state request + +# since PC1 has been processed node_1 must have 3 rows in t1 +# 2 were there before PC1 and one was added while in PC1 +--let $wait_condition = SELECT COUNT(*) = 3 FROM t1; +--source include/wait_condition.inc + +# +# Now disconnect and reconnect node_1 again to get PC3 +# It still is blocked before sending state transfer request in PC2. +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +# Proceed to sending state transfer request +--connection node_1a +--let $galera_sync_point = after_shift_to_joining +--source include/galera_set_sync_point.inc +--let $galera_sync_point = before_send_state_request +--source include/galera_signal_sync_point.inc # before_send_state_request +# +# node_1 proceeds to sending state transfer request, it will be delivered only +# in the next PC which is PC3. Only then the node will shift to JOINING +# +--echo 4 +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_debug_sync_waiters'; + +--connection node_3 +INSERT INTO t1 VALUES (6, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 6 FROM t1; +--source include/wait_condition.inc + +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +# +# This is PC3 +# + +INSERT INTO t1 VALUES (7, 2); + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 7 FROM t1; +--source include/wait_condition.inc + +# node_1 is stiil processing PC2, it was let to send state transfer request +# while in non-PRIM. Now it should be able to complete it and shift to +# JOINING. Make it block on next PC(3) and continue to receive IST +--connection node_1a +--let $galera_sync_point = after_shift_to_joining +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc # won't need it any more +--let $galera_sync_point = process_primary_configuration +--source include/galera_set_sync_point.inc +--let $galera_sync_point = after_shift_to_joining +--source include/galera_signal_sync_point.inc +# continue with IST while still processing PC2 + +# +# Now disconnect and reconnect node_1 again to generate PC4. +# +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +INSERT INTO t1 VALUES (8, 3); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 8 FROM t1; +--source include/wait_condition.inc + +--connection node_1 +SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; + +--connection node_2 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +# +# This is PC4. node_1 should complete IST, complete PC2, and continue +# with the next item in queue +# +--connection node_1a +--let $galera_sync_point = process_primary_configuration +--source include/galera_wait_sync_point.inc + +# +# Now node_1 is blocked before processing PC3, and should have finished state +# transfer started while in PC2. +# State tranfer request was dilivered in PC3 to donor, so node_1 should have +# received IST up to 6 rows in t1 (what was there before PC3). +# +--let $wait_condition = SELECT COUNT(*) = 6 FROM t1; +--source include/wait_condition.inc + +--connection node_2 +INSERT INTO t1 VALUES (9, 2); + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 9 FROM t1; +--source include/wait_condition.inc + +--connection node_1a +# continue with processing PC3 +--let $galera_sync_point = process_primary_configuration +--source include/galera_signal_sync_point.inc + +# wait for row that follows PC3 +--let $wait_condition = SELECT COUNT(*) = 7 FROM t1; +--source include/wait_condition.inc + +# wait till PC4 +--let $galera_sync_point = process_primary_configuration +--source include/galera_wait_sync_point.inc +# +# Now node_1 is processing PC4, still must have 7 rows in t1 +# +--let $wait_condition = SELECT COUNT(*) = 7 FROM t1; +--source include/wait_condition.inc + +# Continue with PC4 +--let $galera_sync_point = process_primary_configuration +--source include/galera_clear_sync_point.inc +--source include/galera_signal_sync_point.inc +--let $galera_sync_point = after_shift_to_joining +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc +--source include/galera_signal_sync_point.inc + +--let $wait_condition = SELECT COUNT(*) = 9 FROM t1; +--source include/wait_condition.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +DROP TABLE t1; + +call mtr.add_suppression("WSREP: Send action {\(.*\), STATE_REQUEST} returned -107 \\(Transport endpoint is not connected\\)"); +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); + +--connection node_2 +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); + +--connection node_3 +call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required."); From fe0cf85d5adb2a9cb619d7137f971b7579d85d02 Mon Sep 17 00:00:00 2001 From: Vincent Milum Jr Date: Tue, 11 Feb 2020 10:27:59 -0800 Subject: [PATCH 34/73] MDEV-21709 ZFS snapdir=visible breaks Galera rsync SST replcation Fix for Galera rsync SST with the specific conditions listed in MDEV-21709 Exclude needs to be on receiving side too --- scripts/wsrep_sst_rsync.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index ef32e77e20d..ce970ed67e1 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -198,6 +198,7 @@ fi # New filter - exclude everything except dirs (schemas) and innodb files FILTER="-f '- /lost+found' + -f '- /.zfs' -f '- /.fseventsd' -f '- /.Trashes' -f '+ /wsrep_sst_binlog.tar' @@ -354,7 +355,7 @@ EOF [ "$OS" = "Linux" ] && count=$(grep -c processor /proc/cpuinfo) [ "$OS" = "Darwin" -o "$OS" = "FreeBSD" ] && count=$(sysctl -n hw.ncpu) - find . -maxdepth 1 -mindepth 1 -type d -not -name "lost+found" \ + find . -maxdepth 1 -mindepth 1 -type d -not -name "lost+found" -not -name ".zfs" \ -print0 | xargs -I{} -0 -P $count \ rsync ${STUNNEL:+--rsh="$STUNNEL"} \ --owner --group --perms --links --specials \ @@ -437,6 +438,7 @@ timeout = 300 $SILENT [$MODULE] path = $WSREP_SST_OPT_DATA + exclude = .zfs [$MODULE-log_dir] path = $WSREP_LOG_DIR [$MODULE-data_dir] From 9fb8d87d2d34b817aebdabe8f6c9c82909abf4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 24 Jun 2020 09:38:54 +0300 Subject: [PATCH 35/73] Test fixes. --- mysql-test/suite/galera/t/MW-388.test | 2 ++ mysql-test/suite/galera/t/MW-86-wait1.test | 1 + mysql-test/suite/galera/t/MW-86-wait8.test | 2 ++ mysql-test/suite/galera/t/galera_query_cache_sync_wait.test | 2 ++ mysql-test/suite/galera/t/galera_var_retry_autocommit.test | 2 ++ 5 files changed, 9 insertions(+) diff --git a/mysql-test/suite/galera/t/MW-388.test b/mysql-test/suite/galera/t/MW-388.test index fafdde092bf..2d3fe5b5d93 100644 --- a/mysql-test/suite/galera/t/MW-388.test +++ b/mysql-test/suite/galera/t/MW-388.test @@ -1,5 +1,7 @@ --source include/galera_cluster.inc +--source include/have_debug.inc --source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc --connection node_1 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(255)) Engine=InnoDB; diff --git a/mysql-test/suite/galera/t/MW-86-wait1.test b/mysql-test/suite/galera/t/MW-86-wait1.test index a7476b74e68..c5b78be64e5 100644 --- a/mysql-test/suite/galera/t/MW-86-wait1.test +++ b/mysql-test/suite/galera/t/MW-86-wait1.test @@ -7,6 +7,7 @@ --source include/have_binlog_format_row.inc --source include/have_debug.inc --source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc --connection node_2 # Make sure no signals have been leftover from previous tests to surprise us. diff --git a/mysql-test/suite/galera/t/MW-86-wait8.test b/mysql-test/suite/galera/t/MW-86-wait8.test index 551b0f67b7c..a648924b9d8 100644 --- a/mysql-test/suite/galera/t/MW-86-wait8.test +++ b/mysql-test/suite/galera/t/MW-86-wait8.test @@ -3,7 +3,9 @@ # --source include/galera_cluster.inc --source include/have_binlog_format_row.inc +--source include/have_debug.inc --source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc --connection node_2 # Make sure no signals have been leftover from previous tests to surprise us. diff --git a/mysql-test/suite/galera/t/galera_query_cache_sync_wait.test b/mysql-test/suite/galera/t/galera_query_cache_sync_wait.test index e13e7f1f748..6d1e21fd94d 100644 --- a/mysql-test/suite/galera/t/galera_query_cache_sync_wait.test +++ b/mysql-test/suite/galera/t/galera_query_cache_sync_wait.test @@ -1,6 +1,8 @@ --source include/galera_cluster.inc +--source include/have_debug.inc --source include/have_debug_sync.inc --source include/have_query_cache.inc +--source include/galera_have_debug_sync.inc CREATE TABLE t1 (id INT PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); diff --git a/mysql-test/suite/galera/t/galera_var_retry_autocommit.test b/mysql-test/suite/galera/t/galera_var_retry_autocommit.test index 142f02546b4..d12ae7936cb 100644 --- a/mysql-test/suite/galera/t/galera_var_retry_autocommit.test +++ b/mysql-test/suite/galera/t/galera_var_retry_autocommit.test @@ -4,7 +4,9 @@ --source include/galera_cluster.inc --source include/have_innodb.inc +--source include/have_debug.inc --source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc --connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 From e3104c4a8c6b97bbc295a2ec7802c30c51fb7801 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Wed, 17 Jun 2020 08:07:01 +0300 Subject: [PATCH 36/73] MDEV-22179 rr support for mtr * --rr-dir to change store dir * --rr-arg doesn't enable --rr (good for scripts) * Bootstrap is saved to rr.boot --- mysql-test/mysql-test-run.pl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 93cf00498b4..ea9f0e9ba74 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -261,7 +261,9 @@ our %gprof_dirs; our $glob_debugger= 0; our $opt_gdb; -our $opt_rr; +my $opt_rr; +my $opt_rr_dir; +my @rr_record_args; our $opt_client_gdb; my $opt_boot_gdb; my $opt_boot_rr; @@ -335,7 +337,6 @@ my $opt_callgrind; my %mysqld_logs; my $opt_debug_sync_timeout= 300; # Default timeout for WAIT_FOR actions. my $warn_seconds = 60; -my @rr_record_args; sub testcase_timeout ($) { my ($tinfo)= @_; @@ -1321,6 +1322,7 @@ sub command_line_setup { 'gdb=s' => \$opt_gdb, 'rr' => \$opt_rr, 'rr-arg=s' => \@rr_record_args, + 'rr-dir=s' => \$opt_rr_dir, 'client-gdb' => \$opt_client_gdb, 'manual-gdb' => \$opt_manual_gdb, 'manual-lldb' => \$opt_manual_lldb, @@ -1785,11 +1787,6 @@ sub command_line_setup { mtr_error("Coverage test needs the source - please use source dist"); } - if ( @rr_record_args ) - { - $opt_rr= 1; - } - # -------------------------------------------------------------------------- # Check debug related options # -------------------------------------------------------------------------- @@ -3412,7 +3409,7 @@ sub mysql_install_db { if ($opt_boot_rr) { $args= ["record", @rr_record_args, $exe_mysqld_bootstrap, @$args]; $exe_mysqld_bootstrap= "rr"; - my $rr_dir= "$opt_vardir/rr.bootstrap"; + my $rr_dir= $opt_rr_dir ? $opt_rr_dir : "$opt_vardir/rr.boot"; $ENV{'_RR_TRACE_DIR'}= $rr_dir; mkpath($rr_dir); } @@ -5390,7 +5387,7 @@ sub mysqld_start ($$) { { $args= ["record", @rr_record_args, "$exe", @$args]; $exe= "rr"; - my $rr_dir= "$opt_vardir/rr". $mysqld->after('mysqld'); + my $rr_dir= $opt_rr_dir ? $opt_rr_dir : "$opt_vardir/rr". $mysqld->after('mysqld'); $ENV{'_RR_TRACE_DIR'}= $rr_dir; mkpath($rr_dir); } @@ -6578,11 +6575,14 @@ Options for strace stracer= Specify name and path to the trace program to use. Default is "strace". Example: $0 --stracer=ktrace. -Options for rr(Record and Replay) +Options for rr (Record and Replay) rr Run the "mysqld" executables using rr. Default run option is "rr record mysqld mysqld_options" boot-rr Start bootstrap server in rr rr-arg=ARG Option to give rr record, can be specified more then once + rr-dir=DIR The directory where rr recordings are stored. Defaults + to 'vardir'/rr.0 (rr.boot for bootstrap instance and + rr.1, ..., rr.N for slave instances). Misc options user=USER User for connecting to mysqld(default: $opt_user) From 97f7d4a9b4da77cb79699a0ea873e4a0e628e8a3 Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Mon, 15 Jun 2020 13:40:50 +0300 Subject: [PATCH 37/73] MDEV-22726: Add check that one can't change general or slow log to a transactional engine --- mysql-test/r/log_tables.result | 6 ++++++ mysql-test/t/log_tables.test | 8 ++++++++ sql/share/errmsg-utf8.txt | 2 ++ sql/sql_table.cc | 7 +++++++ 4 files changed, 23 insertions(+) diff --git a/mysql-test/r/log_tables.result b/mysql-test/r/log_tables.result index aed5dbfba3d..6b5a2ad1577 100644 --- a/mysql-test/r/log_tables.result +++ b/mysql-test/r/log_tables.result @@ -259,6 +259,12 @@ Warning 1286 Unknown storage engine 'NonExistentEngine' alter table mysql.slow_log engine=memory; ERROR HY000: Storage engine MEMORY cannot be used for log tables set storage_engine= @save_storage_engine; +ALTER TABLE mysql.general_log ENGINE=Aria; +ERROR HY000: Only non-transactional Aria table can be used for logging +ALTER TABLE mysql.general_log ENGINE=Aria transactional = 0; +ALTER TABLE mysql.slow_log ENGINE=Aria; +ERROR HY000: Only non-transactional Aria table can be used for logging +ALTER TABLE mysql.slow_log ENGINE=Aria transactional = 0; drop table mysql.slow_log; drop table mysql.general_log; drop table mysql.general_log; diff --git a/mysql-test/t/log_tables.test b/mysql-test/t/log_tables.test index 4ea8a7639ef..1146611ac6c 100644 --- a/mysql-test/t/log_tables.test +++ b/mysql-test/t/log_tables.test @@ -269,6 +269,14 @@ alter table mysql.slow_log engine=memory; #alter table mysql.slow_log engine=blackhole; set storage_engine= @save_storage_engine; +# Make sure only non-transactional Aria table can be used for logging +--error ER_TRANSACTIONAL_ARIA_LOG_ENGINE +ALTER TABLE mysql.general_log ENGINE=Aria; +ALTER TABLE mysql.general_log ENGINE=Aria transactional = 0; +--error ER_TRANSACTIONAL_ARIA_LOG_ENGINE +ALTER TABLE mysql.slow_log ENGINE=Aria; +ALTER TABLE mysql.slow_log ENGINE=Aria transactional = 0; + drop table mysql.slow_log; drop table mysql.general_log; diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index ff495f69a4f..8db992c7501 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -7731,3 +7731,5 @@ ER_GEOJSON_EMPTY_COORDINATES ER_MYROCKS_CANT_NOPAD_COLLATION eng "MyRocks doesn't currently support collations with \"No pad\" attribute." +ER_TRANSACTIONAL_ARIA_LOG_ENGINE + eng "Only non-transactional Aria table can be used for logging" diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 7cce3bcc323..b3a600eec36 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -8925,6 +8925,13 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, DBUG_RETURN(true); } + if (create_info->db_type == maria_hton && + create_info->transactional != HA_CHOICE_NO) + { + my_error(ER_TRANSACTIONAL_ARIA_LOG_ENGINE, MYF(0)); + DBUG_RETURN(true); + } + #ifdef WITH_PARTITION_STORAGE_ENGINE if (alter_info->flags & Alter_info::ALTER_PARTITION) { From b4abe7c91f9b06d50f02a15d9a873a81d5b0b405 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 23 Jun 2020 23:28:37 +0300 Subject: [PATCH 38/73] MDEV-22993: Crash on EXPLAIN with PUSHED DOWN SELECT and subquery - select_describe() should not attempt to produce query plans for subqueries if the query is handled by a Select Handler. - JOIN::save_explain_data_intern should not add links to Explain_select for children selects if: 1. The whole query is handled by the Select Handler, or 2. this select (and so its children) is handled by Derived Handler. --- .../federatedx_create_handlers.result | 93 +++++++++++++++---- .../federated/federatedx_create_handlers.test | 34 +++++++ sql/sql_select.cc | 54 +++++++---- 3 files changed, 143 insertions(+), 38 deletions(-) diff --git a/mysql-test/suite/federated/federatedx_create_handlers.result b/mysql-test/suite/federated/federatedx_create_handlers.result index 65a9d52803f..63e56bff425 100644 --- a/mysql-test/suite/federated/federatedx_create_handlers.result +++ b/mysql-test/suite/federated/federatedx_create_handlers.result @@ -209,7 +209,6 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t3 ALL NULL NULL NULL NULL 7 1 PRIMARY ref key0 key0 18 federated.t3.name 2 2 PUSHED DERIVED NULL NULL NULL NULL NULL NULL NULL NULL -3 MATERIALIZED t2 ALL NULL NULL NULL NULL 7 Using temporary ANALYZE FORMAT=JSON SELECT * FROM federated.t3, (SELECT t1.name FROM federated.t1 @@ -251,25 +250,7 @@ ANALYZE "select_id": 2, "table": { "message": "Pushed derived" - }, - "subqueries": [ - { - "query_block": { - "select_id": 3, - "temporary_table": { - "table": { - "table_name": "t2", - "access_type": "ALL", - "r_loops": 0, - "rows": 7, - "r_rows": null, - "filtered": 100, - "r_filtered": null - } - } - } - } - ] + } } } } @@ -319,6 +300,78 @@ select @var; @var xxx select name into outfile 'tmp.txt' from federated.t1; +# +# MDEV-22993: Crash on EXPLAIN with PUSHED DOWN SELECT and subquery +# +explain +select * from federated.t1 +where name in (select name from federated.t2); +id select_type table type possible_keys key key_len ref rows Extra +1 PUSHED SELECT NULL NULL NULL NULL NULL NULL NULL NULL +explain format=json +select * from federated.t1 +where name in (select name from federated.t2); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "message": "Pushed select" + } + } +} +# +# MDEV-22993, testcase #2: EXPLAIN output doesn't make sense when +# derived table pushdown is used. +# +create table t5 (a int) engine=myisam; +insert into t5 values (1),(2); +# Must not show lines with id=3 +explain +select * from t5, +(select id from federated.t1 +where name in (select name from federated.t2) or name like 'foo%') as TQ; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t5 ALL NULL NULL NULL NULL 2 +1 PRIMARY ALL NULL NULL NULL NULL 5 Using join buffer (flat, BNL join) +2 PUSHED DERIVED NULL NULL NULL NULL NULL NULL NULL NULL +# Must not show elements with select_id=3 +explain format=json +select * from t5, +(select id from federated.t1 +where name in (select name from federated.t2) or name like 'foo%') as TQ; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t5", + "access_type": "ALL", + "rows": 2, + "filtered": 100 + }, + "block-nl-join": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 5, + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "65", + "join_type": "BNL", + "materialized": { + "query_block": { + "select_id": 2, + "table": { + "message": "Pushed derived" + } + } + } + } + } +} +drop table t5; DROP TABLE federated.t1, federated.t2, federated.t3, federated.t4; connection slave; DROP TABLE federated.t1, federated.t2; diff --git a/mysql-test/suite/federated/federatedx_create_handlers.test b/mysql-test/suite/federated/federatedx_create_handlers.test index 42a03e60d67..d765588b79b 100644 --- a/mysql-test/suite/federated/federatedx_create_handlers.test +++ b/mysql-test/suite/federated/federatedx_create_handlers.test @@ -167,6 +167,40 @@ select name into outfile 'tmp.txt' from federated.t1; let $path=`select concat(@@datadir, 'test/tmp.txt')`; remove_file $path; +--echo # +--echo # MDEV-22993: Crash on EXPLAIN with PUSHED DOWN SELECT and subquery +--echo # + +explain +select * from federated.t1 +where name in (select name from federated.t2); + +explain format=json +select * from federated.t1 +where name in (select name from federated.t2); + +--echo # +--echo # MDEV-22993, testcase #2: EXPLAIN output doesn't make sense when +--echo # derived table pushdown is used. +--echo # + +create table t5 (a int) engine=myisam; +insert into t5 values (1),(2); + +--echo # Must not show lines with id=3 +explain +select * from t5, +(select id from federated.t1 + where name in (select name from federated.t2) or name like 'foo%') as TQ; + +--echo # Must not show elements with select_id=3 +explain format=json +select * from t5, +(select id from federated.t1 + where name in (select name from federated.t2) or name like 'foo%') as TQ; + +drop table t5; + DROP TABLE federated.t1, federated.t2, federated.t3, federated.t4; connection slave; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 040c9acd2ae..0ca5ab23288 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -26840,24 +26840,33 @@ int JOIN::save_explain_data_intern(Explain_query *output, output->add_node(xpl_sel); } - for (SELECT_LEX_UNIT *tmp_unit= join->select_lex->first_inner_unit(); - tmp_unit; - tmp_unit= tmp_unit->next_unit()) + /* + Don't try to add query plans for child selects if this select was pushed + down into a Smart Storage Engine: + - the entire statement was pushed down ("PUSHED SELECT"), or + - this derived table was pushed down ("PUSHED DERIVED") + */ + if (!select_lex->pushdown_select && select_lex->type != pushed_derived_text) { - /* - Display subqueries only if - (1) they are not parts of ON clauses that were eliminated by table - elimination. - (2) they are not merged derived tables - (3) they are not hanging CTEs (they are needed for execution) - */ - if (!(tmp_unit->item && tmp_unit->item->eliminated) && // (1) - (!tmp_unit->derived || - tmp_unit->derived->is_materialized_derived()) && // (2) - !(tmp_unit->with_element && - (!tmp_unit->derived || !tmp_unit->derived->derived_result))) // (3) - { - explain->add_child(tmp_unit->first_select()->select_number); + for (SELECT_LEX_UNIT *tmp_unit= join->select_lex->first_inner_unit(); + tmp_unit; + tmp_unit= tmp_unit->next_unit()) + { + /* + Display subqueries only if + (1) they are not parts of ON clauses that were eliminated by table + elimination. + (2) they are not merged derived tables + (3) they are not hanging CTEs (they are needed for execution) + */ + if (!(tmp_unit->item && tmp_unit->item->eliminated) && // (1) + (!tmp_unit->derived || + tmp_unit->derived->is_materialized_derived()) && // (2) + !(tmp_unit->with_element && + (!tmp_unit->derived || !tmp_unit->derived->derived_result))) // (3) + { + explain->add_child(tmp_unit->first_select()->select_number); + } } } @@ -26890,7 +26899,16 @@ static void select_describe(JOIN *join, bool need_tmp_table, bool need_order, THD *thd=join->thd; select_result *result=join->result; DBUG_ENTER("select_describe"); - + + if (join->select_lex->pushdown_select) + { + /* + The whole statement was pushed down to a Smart Storage Engine. Do not + attempt to produce a query plan locally. + */ + DBUG_VOID_RETURN; + } + /* Update the QPF with latest values of using_temporary, using_filesort */ for (SELECT_LEX_UNIT *unit= join->select_lex->first_inner_unit(); unit; From 33de71c2f8ac8de49baef8ffa87cec6431684986 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Tue, 19 May 2020 15:38:34 +0300 Subject: [PATCH 39/73] MDEV-22632 wsrep XID checkpointing can happen out of order for certification failure When a transaction fails in certification phase, it has connsumed one GTID, but as transaction must rollback, it will not go for commit ordering, and because of this also the wsrep XID checkpointing can happen out of order. This PR will make the thread, which has failed for certiication failure to wait for its commit order turn for checkpointing wsrep IXD in innodb rollback segment. There is a specific test for wsrep XID checkpointing ordering in mtr test: mysql-wsrep-bugs-607, which is added in this PR. Test galera_slave_replay depends also on this fix, as the second test phase may also assert for bad wsrep XID checkpointing order. galera_slave_replay.test had also other problems, which caused the test to fail immediately, thse are now fixes in this PR as well. --- .../suite/galera/r/galera_slave_replay.result | 8 +-- .../galera/r/mysql-wsrep-bugs-607.result | 31 ++++++++ .../suite/galera/t/galera_slave_replay.test | 17 +++-- .../suite/galera/t/mysql-wsrep-bugs-607.test | 70 +++++++++++++++++++ sql/wsrep_server_service.cc | 14 +++- 5 files changed, 127 insertions(+), 13 deletions(-) create mode 100644 mysql-test/suite/galera/r/mysql-wsrep-bugs-607.result create mode 100644 mysql-test/suite/galera/t/mysql-wsrep-bugs-607.test diff --git a/mysql-test/suite/galera/r/galera_slave_replay.result b/mysql-test/suite/galera/r/galera_slave_replay.result index e8dd6ae87b1..e2cbe2a220f 100644 --- a/mysql-test/suite/galera/r/galera_slave_replay.result +++ b/mysql-test/suite/galera/r/galera_slave_replay.result @@ -4,9 +4,7 @@ connection node_2; connection node_1; connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; connection node_3; -SET GLOBAL wsrep_on=OFF; RESET MASTER; -SET GLOBAL wsrep_on=ON; connection node_2a; START SLAVE; connection node_3; @@ -74,8 +72,10 @@ UPDATE t1 SET f2 = 'd' WHERE f1 = 3; connection node_2a; SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync'; SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; -connection node_3; +connection node_1; UPDATE test.t1 SET f2 = 'e' WHERE f1 = 3; +connection node_2a; +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached"; connection node_3; COMMIT; connection node_2a; @@ -96,6 +96,4 @@ RESET SLAVE; DROP TABLE t1; connection node_3; DROP TABLE t1; -SET GLOBAL wsrep_on=OFF; RESET MASTER; -SET GLOBAL wsrep_on=ON; diff --git a/mysql-test/suite/galera/r/mysql-wsrep-bugs-607.result b/mysql-test/suite/galera/r/mysql-wsrep-bugs-607.result new file mode 100644 index 00000000000..9d9614ffefe --- /dev/null +++ b/mysql-test/suite/galera/r/mysql-wsrep-bugs-607.result @@ -0,0 +1,31 @@ +connection node_2; +connection node_1; +# +# test the order of wsrep XID storage after certifiation failure +# +connection node_1; +set session wsrep_sync_wait=0; +create table t1 (i int primary key, j int); +insert into t1 values (4, 0); +connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connection node_2b; +set session wsrep_sync_wait=0; +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; +connection node_1; +UPDATE test.t1 set j=1 where i=4; +connection node_2b; +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached"; +connection node_2; +set session wsrep_sync_wait=0; +set session wsrep_retry_autocommit=0; +UPDATE test.t1 SET j=2 WHERE i=4; +connection node_2b; +SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; +SET GLOBAL debug_dbug = ""; +SET DEBUG_SYNC = "RESET"; +connection node_2; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +select * from t1; +i j +4 1 +DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_slave_replay.test b/mysql-test/suite/galera/t/galera_slave_replay.test index 37c4cbd5b43..a908fb95c13 100644 --- a/mysql-test/suite/galera/t/galera_slave_replay.test +++ b/mysql-test/suite/galera/t/galera_slave_replay.test @@ -21,9 +21,7 @@ # --connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 --connection node_3 -SET GLOBAL wsrep_on=OFF; RESET MASTER; -SET GLOBAL wsrep_on=ON; --connection node_2a # @@ -163,18 +161,25 @@ UPDATE t1 SET f2 = 'd' WHERE f1 = 3; # block applier SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; -# Inject a conflicting update from node 3 ---connection node_3 +# Inject a conflicting update from node 1 +--connection node_1 UPDATE test.t1 SET f2 = 'e' WHERE f1 = 3; +--connection node_2a +# wait until applier has reached the sync point +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached"; +--let $expected_cert_failures = `SELECT VARIABLE_VALUE+1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_cert_failures'` + # send the update from master --connection node_3 --error 0 COMMIT; --connection node_2a +--let $wait_condition = SELECT VARIABLE_VALUE = $expected_cert_failures FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_cert_failures' +--source include/wait_condition.inc -# release the applier +# release the applier from node 1 SET GLOBAL debug_dbug = ""; SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; @@ -198,6 +203,4 @@ DROP TABLE t1; --connection node_3 DROP TABLE t1; -SET GLOBAL wsrep_on=OFF; RESET MASTER; -SET GLOBAL wsrep_on=ON; diff --git a/mysql-test/suite/galera/t/mysql-wsrep-bugs-607.test b/mysql-test/suite/galera/t/mysql-wsrep-bugs-607.test new file mode 100644 index 00000000000..c24ec7911e2 --- /dev/null +++ b/mysql-test/suite/galera/t/mysql-wsrep-bugs-607.test @@ -0,0 +1,70 @@ +--source include/galera_cluster.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc +--source include/have_log_bin.inc + +# +# Test case to stress the order of wsrep XID checkpointing. +# +# In buggy version, the transaction which failed certification can +# rush to record wsrep XID checkpoint before the offending applier, +# causing assert in innodb sys header update routine +# + +--echo # +--echo # test the order of wsrep XID storage after certifiation failure +--echo # + +--connection node_1 +set session wsrep_sync_wait=0; + +create table t1 (i int primary key, j int); + +insert into t1 values (4, 0); + +--connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2 +--connection node_2b +set session wsrep_sync_wait=0; +# wait for the last insert to be replicated from node 1 +--let $wait_condition = SELECT COUNT(*) = 1 FROM test.t1; +--source include/wait_condition.inc + +# block applier before applying +SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb"; + +# send update from node 1, it will pause in the sync point +--connection node_1 +UPDATE test.t1 set j=1 where i=4; + +--connection node_2b +# wait until applier has reached the sync point +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached"; + +# look number of cert failures so far, and expect one more to happen +--let $expected_cert_failures = `SELECT VARIABLE_VALUE+1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_cert_failures'` + +# Inject a conflicting update in node 2, it should fail in certification +--connection node_2 +set session wsrep_sync_wait=0; +set session wsrep_retry_autocommit=0; +--send UPDATE test.t1 SET j=2 WHERE i=4 + +--connection node_2b +# wait until the update has hit certification failure + +--let $wait_condition = SELECT VARIABLE_VALUE = $expected_cert_failures FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_cert_failures' +--source include/wait_condition.inc + +# release the applier +SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb"; + +SET GLOBAL debug_dbug = ""; +SET DEBUG_SYNC = "RESET"; + +--connection node_2 +--error ER_LOCK_DEADLOCK +--reap +select * from t1; + +DROP TABLE t1; diff --git a/sql/wsrep_server_service.cc b/sql/wsrep_server_service.cc index e6ccaca13b1..57b9c7fd626 100644 --- a/sql/wsrep_server_service.cc +++ b/sql/wsrep_server_service.cc @@ -303,9 +303,21 @@ wsrep::gtid Wsrep_server_service::get_position(wsrep::client_service&) return wsrep_get_SE_checkpoint(); } -void Wsrep_server_service::set_position(wsrep::client_service&, +void Wsrep_server_service::set_position(wsrep::client_service& c WSREP_UNUSED, const wsrep::gtid& gtid) { + Wsrep_client_service& cs WSREP_UNUSED (static_cast(c)); + DBUG_ASSERT(cs.m_client_state.transaction().state() + == wsrep::transaction::s_aborted); + // Wait until all prior committers have finished. + wsrep::gtid wait_for(gtid.id(), + wsrep::seqno(gtid.seqno().get() - 1)); + if (auto err = Wsrep_server_state::instance().provider() + .wait_for_gtid(wait_for, std::numeric_limits::max())) + { + WSREP_WARN("Wait for gtid returned error %d while waiting for " + "prior transactions to commit before setting position", err); + } wsrep_set_SE_checkpoint(gtid); } From bffa8264aa4d512d15723e285a7dd6792503a396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 24 Jun 2020 13:19:56 +0300 Subject: [PATCH 40/73] Stabilize glera_var_cluster_conf_id test case. --- mysql-test/suite/galera/t/galera_var_cluster_conf_id.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_var_cluster_conf_id.test b/mysql-test/suite/galera/t/galera_var_cluster_conf_id.test index c569634823f..dd4a630035d 100644 --- a/mysql-test/suite/galera/t/galera_var_cluster_conf_id.test +++ b/mysql-test/suite/galera/t/galera_var_cluster_conf_id.test @@ -1,9 +1,9 @@ --source include/galera_cluster.inc --connection node_1 ---replace_regex /18446744073709551/ERROR/ /[0-9]/#/ +--replace_regex /18446744073709551/ERROR/ /[0-9+]/#/ show status like 'wsrep_cluster_conf_id'; --connection node_2 ---replace_regex /18446744073709551/ERROR/ /[0-9]/#/ +--replace_regex /18446744073709551/ERROR/ /[0-9+]/#/ show status like 'wsrep_cluster_conf_id'; From dc68846ec5ffdd6f08d93dc3bda123ff9cef04fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 24 Jun 2020 13:41:52 +0300 Subject: [PATCH 41/73] Disable sporadically failing galera_toi_truncate test case --- mysql-test/suite/galera/disabled.def | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 81970cd22aa..302180b16bd 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -37,6 +37,7 @@ galera_split_brain : MDEV-18280 Galera test failure on galera_split_brain and ga galera_ssl_upgrade : MDEV-19950 Galera test failure on galera_ssl_upgrade galera_sst_mariabackup_encrypt_with_key : MDEV-21484 galera_sst_mariabackup_encrypt_with_key galera_toi_ddl_nonconflicting : MDEV-21518 galera.galera_toi_ddl_nonconflicting +galera_toi_truncate : MDEV-22996 Hang on galera_toi_truncate test case galera_var_innodb_disallow_writes : MDEV-20928 galera.galera_var_innodb_disallow_writes galera_var_node_address : MDEV-20485 Galera test failure galera_var_notify_cmd : MDEV-21905 Galera test galera_var_notify_cmd causes hang From f1838434b85db2d640ee21d0cbc2a4df1dc550e1 Mon Sep 17 00:00:00 2001 From: Sujatha Date: Fri, 29 May 2020 11:36:28 +0530 Subject: [PATCH 42/73] MDEV-22706: Assertion `!current' failed in PROFILING::start_new_query Analysis: ======== When "Profiling" is enabled, server collects the resource usage of each statement that gets executed in current session. Profiling doesn't support nested statements. In order to ensure this behavior when profiling is enabled for a statement, there should not be any other active query which is being profiled. This active query information is stored in 'current' variable. When a nested query arrives it finds 'current' being not NULL and server aborts. When 'init_connect' and 'init_slave' system variables are set they contain a set of statements to be executed. "execute_init_command" is the function call which invokes "dispatch_command" for each statement provided in 'init_connect', 'init_slave' system variables. "execute_init_command" invokes "start_new_query" and it passes the statement list to "dispatch_command". This "dispatch_command" intern invokes "start_new_query" which leads to nesting of queries. Hence '!current' assert is triggered. Fix: === Remove profiling from "execute_init_command" as it will be done within "dispatch_command" execution. --- mysql-test/r/nested_profiling.result | 16 +++++++++++ mysql-test/t/nested_profiling.test | 42 ++++++++++++++++++++++++++++ sql/sql_parse.cc | 8 ------ 3 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 mysql-test/r/nested_profiling.result create mode 100644 mysql-test/t/nested_profiling.test diff --git a/mysql-test/r/nested_profiling.result b/mysql-test/r/nested_profiling.result new file mode 100644 index 00000000000..86841383046 --- /dev/null +++ b/mysql-test/r/nested_profiling.result @@ -0,0 +1,16 @@ +SET @saved_profiling=@@GLOBAL.profiling; +SET @saved_init_connect=@@GLOBAL.init_connect; +SET GLOBAL init_connect="set @a=2;set @b=3"; +SET GLOBAL profiling=on; +create user mysqltest1@localhost; +SELECT @a, @b; +@a @b +2 3 +SHOW PROFILES; +Query_ID Duration Query +1 # set @a=2;set @b=3 +2 # set @b=3 +3 # SELECT @a, @b +DROP USER mysqltest1@localhost; +SET GLOBAL profiling=@saved_profiling; +SET GLOBAL init_connect=@saved_init_connect; diff --git a/mysql-test/t/nested_profiling.test b/mysql-test/t/nested_profiling.test new file mode 100644 index 00000000000..ba89aefc647 --- /dev/null +++ b/mysql-test/t/nested_profiling.test @@ -0,0 +1,42 @@ +# ==== Purpose ==== +# +# Test verifies that "init_connect" and "init_slave" system variables work +# fine when "profiling=on". +# +# ==== Implementation ==== +# +# Steps: +# 0 - Create regular user without super privilege so that "init_connect" +# variable is effective. +# 1 - Enable profiling. +# 2 - Start a new connection which will try to execute the statements +# specified in "init_connect". No assert should be reported. +# 3 - Execute SHOW PROFILES to verify that statements specified in +# "init_connect" are displayed as part of profiling. +# +# ==== References ==== +# +# MDEV-22706: Assertion `!current' failed in PROFILING::start_new_query +# +--source include/not_embedded.inc +--source include/have_profiling.inc + +SET @saved_profiling=@@GLOBAL.profiling; +SET @saved_init_connect=@@GLOBAL.init_connect; +SET GLOBAL init_connect="set @a=2;set @b=3"; +SET GLOBAL profiling=on; + +create user mysqltest1@localhost; +connect (con1,localhost,mysqltest1,,); +connection con1; +SELECT @a, @b; +--replace_column 2 # +SHOW PROFILES; + +#========== Clean up =========== +connection default; +disconnect con1; +DROP USER mysqltest1@localhost; + +SET GLOBAL profiling=@saved_profiling; +SET GLOBAL init_connect=@saved_init_connect; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 656da3b6a79..14ac657862f 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -669,11 +669,6 @@ void execute_init_command(THD *thd, LEX_STRING *init_command, char *buf= thd->strmake(init_command->str, len); mysql_rwlock_unlock(var_lock); -#if defined(ENABLED_PROFILING) - thd->profiling.start_new_query(); - thd->profiling.set_query_source(buf, len); -#endif - THD_STAGE_INFO(thd, stage_execution_of_init_command); save_client_capabilities= thd->client_capabilities; thd->client_capabilities|= CLIENT_MULTI_QUERIES; @@ -688,9 +683,6 @@ void execute_init_command(THD *thd, LEX_STRING *init_command, thd->client_capabilities= save_client_capabilities; thd->net.vio= save_vio; -#if defined(ENABLED_PROFILING) - thd->profiling.finish_current_query(); -#endif } From 7ee6a3ae5f805d84df6d5c78856b34fa9c2696a4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 25 Jun 2020 09:58:42 +0200 Subject: [PATCH 43/73] MDEV-22950 followup Deadlock in DbugParse, on Linux. In 10.1, DBUG recursive mutex was improperly implemented. CODE_STATE::locked counter was never updated. Copy the code around LockMutex/UnlockMutex from 10.2 --- dbug/dbug.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbug/dbug.c b/dbug/dbug.c index b0e1b0eaae6..007769c3c37 100644 --- a/dbug/dbug.c +++ b/dbug/dbug.c @@ -330,10 +330,13 @@ static void LockMutex(CODE_STATE *cs) { if (!cs->locked) pthread_mutex_lock(&THR_LOCK_dbug); + cs->locked++; } static void UnlockMutex(CODE_STATE *cs) { - if (!cs->locked) + --cs->locked; + assert(cs->locked >= 0); + if (cs->locked == 0) pthread_mutex_unlock(&THR_LOCK_dbug); } static void LockIfInitSettings(CODE_STATE *cs) From 3bc89395529b099ef744953263ddc10b1f0ea1bd Mon Sep 17 00:00:00 2001 From: Sujatha Date: Wed, 17 Jun 2020 10:48:28 +0530 Subject: [PATCH 44/73] MDEV-22806: MSAN reports use-of-uninitialized-value for rpl_parallel_conflicts.test Problem: ======== Relay_log_info::flush reports following MSAN issue. ==17820==WARNING: MemorySanitizer: use-of-uninitialized-value is reported #5 0x00005584f0981441 in my_write (Filedes=22, Buffer=0x72500003e818 "5\n./slave-relay-bin.000003\n21385\n master-bin.000001\n21643\n0\n", '\245' ..., Count=118, MyFlags=532) at /home/sujatha/bug_repo/test-10.5-msan/mysys/my_write.c:49 Analysis: ========= In parallel replication at the end of each statement execution the worker execution status is updated in 'relay-log.info' file. When two workers try to flush the status at the same time, since the write to cache is not serialized both workers write to the same address simultaneously and increment the length twice. Because of this the length of buffer is more than actual data. When flush code tries to read the buffer beyond valid data length MSAN reports uninitialized values error. Fix: === Serialize the relay log flush operation using "rli->data_lock". --- sql/rpl_rli.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 905462ecfbd..c196a65809a 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1445,8 +1445,14 @@ bool Relay_log_info::stmt_done(my_off_t event_master_log_pos, THD *thd, } DBUG_EXECUTE_IF("inject_crash_before_flush_rli", DBUG_SUICIDE();); if (mi->using_gtid == Master_info::USE_GTID_NO) + { + if (rgi->is_parallel_exec) + mysql_mutex_lock(&data_lock); if (flush_relay_log_info(this)) error= 1; + if (rgi->is_parallel_exec) + mysql_mutex_unlock(&data_lock); + } DBUG_EXECUTE_IF("inject_crash_after_flush_rli", DBUG_SUICIDE();); } DBUG_RETURN(error); From 3c238ac51c21dd0b7ba410012cf317298873c0c2 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sat, 27 Jun 2020 11:46:30 +0300 Subject: [PATCH 45/73] MDEV-20213 binlog_encryption.binlog_incident failed in buildbot, server crashed in Check::validate Check::validate(): Relax a debug assertion. TRX_SYS_SPACE fil_space_t can be created and became visible to this assertion before fil_system.sys_space becomes initialized --- storage/innobase/fil/fil0fil.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 60571fac45c..1c3ef644782 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -4642,7 +4642,8 @@ struct Check { ut_list_validate(space->chain, check); ut_a(space->size == check.size); ut_ad(space->id != TRX_SYS_SPACE - || space == fil_system.sys_space); + || space == fil_system.sys_space + || fil_system.sys_space == NULL); ut_ad(space->id != SRV_TMP_SPACE_ID || space == fil_system.temp_space); return(check.n_open); From 52c4abbff2ec4b97d6c69f238457bf759591ce15 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sat, 27 Jun 2020 12:15:17 +0300 Subject: [PATCH 46/73] MDEV-20213 binlog_encryption.binlog_incident failed in buildbot, server crashed in Check::validate follow up fil_system.sys_space is a shared variable between the thread which assigns a value to it, and the thread which does Check::validate() SysTablespace::open_or_create(): protect a shared variable with a mutex to avoid any data race surprises. --- storage/innobase/fsp/fsp0sysspace.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index dd4eb294dc4..6c185cbcf49 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -913,9 +913,13 @@ SysTablespace::open_or_create( } else { ut_ad(!fil_system.sys_space); ut_ad(space_id() == TRX_SYS_SPACE); - space = fil_system.sys_space = fil_space_create( + space = fil_space_create( name(), TRX_SYS_SPACE, flags(), FIL_TYPE_TABLESPACE, NULL); + + mutex_enter(&fil_system.mutex); + fil_system.sys_space = space; + mutex_exit(&fil_system.mutex); if (!space) { return DB_ERROR; } From e4cff9a85565d0183ab53601c1cef462f5f45fe5 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sat, 27 Jun 2020 12:30:17 +0300 Subject: [PATCH 47/73] MDEV-19298 Assertion `space->id != 0xFFFFFFFEU || space == fil_system.temp_space' failed in Check::validate upon crash upgrade from 10.2.22 This issue is pretty much the same as MDEV-20213. The fix is similar to: 3c238ac51c21dd0b7ba410012cf317298873c0c2 52c4abbff2ec4b97d6c69f238457bf759591ce15 Check::validate(): fix a debug assertion SysTablespace::open_or_create(): protect assigning to a shared variable with a mutex --- storage/innobase/fil/fil0fil.cc | 19 ++++++++++++++----- storage/innobase/fsp/fsp0sysspace.cc | 6 +++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 1c3ef644782..fc91d10cab6 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -4641,11 +4641,20 @@ struct Check { Check check; ut_list_validate(space->chain, check); ut_a(space->size == check.size); - ut_ad(space->id != TRX_SYS_SPACE - || space == fil_system.sys_space - || fil_system.sys_space == NULL); - ut_ad(space->id != SRV_TMP_SPACE_ID - || space == fil_system.temp_space); + + switch (space->id) { + case TRX_SYS_SPACE: + ut_ad(fil_system.sys_space == NULL + || fil_system.sys_space == space); + break; + case SRV_TMP_SPACE_ID: + ut_ad(fil_system.temp_space == NULL + || fil_system.temp_space == space); + break; + default: + break; + } + return(check.n_open); } }; diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index 6c185cbcf49..451187a35d9 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -904,9 +904,13 @@ SysTablespace::open_or_create( } else if (is_temp) { ut_ad(!fil_system.temp_space); ut_ad(space_id() == SRV_TMP_SPACE_ID); - space = fil_system.temp_space = fil_space_create( + space = fil_space_create( name(), SRV_TMP_SPACE_ID, flags(), FIL_TYPE_TEMPORARY, NULL); + + mutex_enter(&fil_system.mutex); + fil_system.temp_space = space; + mutex_exit(&fil_system.mutex); if (!space) { return DB_ERROR; } From c032c2ef6642e5ab28a62e96e2ba7f5f882eca61 Mon Sep 17 00:00:00 2001 From: Kentoku SHIBA Date: Fri, 12 Apr 2019 17:00:04 +0900 Subject: [PATCH 48/73] MDEV-18993 The keep-alive connection (set spider_conn_recycle_mode = 1) in spider would cause cash in MariaDB (#1269) Fix the following valgrind error. ==94390== Thread 29: ==94390== Invalid read of size 8 ==94390== at 0x78389D: thd_increment_bytes_sent (sql_class.cc:4265) ==94390== by 0xC8EC46: net_real_write (net_serv.cc:730) ==94390== by 0xC8E0C8: net_flush (net_serv.cc:383) ==94390== by 0xC8E4D0: net_write_command (net_serv.cc:521) ==94390== by 0xADCE61: cli_advanced_command (client.c:468) ==94390== by 0xAE3CAF: mysql_close_slow_part (client.c:3671) ==94390== by 0xAE3D28: mysql_close (client.c:3683) ==94390== by 0x149E69A8: spider_db_mbase::disconnect() (spd_db_mysql.cc:2217) ==94390== by 0x1491EA26: spider_db_disconnect(st_spider_conn*) (spd_db_conn.cc:297) ==94390== by 0x14948EBE: spider_free_conn_alloc(st_spider_conn*) (spd_conn.cc:196) ==94390== by 0x1494B26A: spider_free_conn(st_spider_conn*) (spd_conn.cc:1251) ==94390== by 0x1494941F: spider_free_conn_from_trx(st_spider_transaction*, st_spider_conn*, bool, bool, int*) (spd_conn.cc:315) ==94390== Address 0x1f0e0990 is 4,832 bytes inside a block of size 25,728 free'd ==94390== at 0x4C2ACBD: free (vg_replace_malloc.c:530) ==94390== by 0x13F5545: my_free (my_malloc.c:222) ==94390== by 0x6C75B7: ilink::operator delete(void*, unsigned long) (sql_list.h:618) ==94390== by 0x77B9F6: THD::~THD() (sql_class.cc:1724) ==94390== by 0x1494FCE0: spider_bg_conn_action(void*) (spd_conn.cc:2580) ==94390== by 0x4E3DDD4: start_thread (in /usr/lib64/libpthread-2.17.so) ==94390== by 0x5FBFEAC: clone (in /usr/lib64/libc-2.17.so) ==94390== Block was alloc'd at ==94390== at 0x4C29BC3: malloc (vg_replace_malloc.c:299) ==94390== by 0x13F4DFA: my_malloc (my_malloc.c:101) ==94390== by 0x1491CF06: ilink::operator new(unsigned long) (sql_list.h:614) ==94390== by 0x1494F7FD: spider_bg_conn_action(void*) (spd_conn.cc:2501) ==94390== by 0x4E3DDD4: start_thread (in /usr/lib64/libpthread-2.17.so) ==94390== by 0x5FBFEAC: clone (in /usr/lib64/libc-2.17.so) ==94390== Invalid write of size 8 ==94390== at 0x7838AF: thd_increment_bytes_sent (sql_class.cc:4265) ==94390== by 0xC8EC46: net_real_write (net_serv.cc:730) ==94390== by 0xC8E0C8: net_flush (net_serv.cc:383) ==94390== by 0xC8E4D0: net_write_command (net_serv.cc:521) ==94390== by 0xADCE61: cli_advanced_command (client.c:468) ==94390== by 0xAE3CAF: mysql_close_slow_part (client.c:3671) ==94390== by 0xAE3D28: mysql_close (client.c:3683) ==94390== by 0x149E69A8: spider_db_mbase::disconnect() (spd_db_mysql.cc:2217) ==94390== by 0x1491EA26: spider_db_disconnect(st_spider_conn*) (spd_db_conn.cc:297) ==94390== by 0x14948EBE: spider_free_conn_alloc(st_spider_conn*) (spd_conn.cc:196) ==94390== by 0x1494B26A: spider_free_conn(st_spider_conn*) (spd_conn.cc:1251) ==94390== by 0x1494941F: spider_free_conn_from_trx(st_spider_transaction*, st_spider_conn*, bool, bool, int*) (spd_conn.cc:315) ==94390== Address 0x1f0e0990 is 4,832 bytes inside a block of size 25,728 free'd ==94390== at 0x4C2ACBD: free (vg_replace_malloc.c:530) ==94390== by 0x13F5545: my_free (my_malloc.c:222) ==94390== by 0x6C75B7: ilink::operator delete(void*, unsigned long) (sql_list.h:618) ==94390== by 0x77B9F6: THD::~THD() (sql_class.cc:1724) ==94390== by 0x1494FCE0: spider_bg_conn_action(void*) (spd_conn.cc:2580) ==94390== by 0x4E3DDD4: start_thread (in /usr/lib64/libpthread-2.17.so) ==94390== by 0x5FBFEAC: clone (in /usr/lib64/libc-2.17.so) ==94390== Block was alloc'd at ==94390== at 0x4C29BC3: malloc (vg_replace_malloc.c:299) ==94390== by 0x13F4DFA: my_malloc (my_malloc.c:101) ==94390== by 0x1491CF06: ilink::operator new(unsigned long) (sql_list.h:614) ==94390== by 0x1494F7FD: spider_bg_conn_action(void*) (spd_conn.cc:2501) ==94390== by 0x4E3DDD4: start_thread (in /usr/lib64/libpthread-2.17.so) ==94390== by 0x5FBFEAC: clone (in /usr/lib64/libc-2.17.so) --- storage/spider/spd_db_mysql.cc | 3 +++ storage/spider/spd_environ.h | 1 + 2 files changed, 4 insertions(+) diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index 0329509c748..6ee4be4c7cc 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -1948,6 +1948,9 @@ int spider_db_mbase::connect( connect_retry_count--; my_sleep((ulong) connect_retry_interval); } else { +#ifdef SPIDER_NET_HAS_THD + db_conn->net.thd = NULL; +#endif if (connect_mutex) pthread_mutex_unlock(&spider_open_conn_mutex); break; diff --git a/storage/spider/spd_environ.h b/storage/spider/spd_environ.h index 5e66a912582..ded2927482b 100644 --- a/storage/spider/spd_environ.h +++ b/storage/spider/spd_environ.h @@ -25,6 +25,7 @@ #if defined(MARIADB_BASE_VERSION) && MYSQL_VERSION_ID >= 100100 #define SPIDER_SUPPORT_CREATE_OR_REPLACE_TABLE +#define SPIDER_NET_HAS_THD #endif #if defined(MARIADB_BASE_VERSION) && MYSQL_VERSION_ID >= 100211 From 37cb7a0071febdba7a5ae61c2cd9e87def37454e Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 27 Jun 2020 12:55:55 +0530 Subject: [PATCH 49/73] MDEV-17606: Query returns wrong results (while using CHARACTER SET utf8) The issue here was that the left expr and right expr of the ANY subquery had different character sets, so we were converting the left expr to utf8 character set. So when this conversion was happening we were actually converting the item inside the cache, it looked like (convert(t1.l1 using utf8)), which is incorrect. To fix this problem we are going to store the reference of the left expr and convert that to utf8 character set, it would look like convert((`test`.`t1`.`l1`) using utf8) --- mysql-test/r/subselect4.result | 18 ++++++++++++++++++ mysql-test/t/subselect4.test | 13 +++++++++++++ sql/item_subselect.cc | 2 +- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 863105b24b6..22d4938fb78 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2566,3 +2566,21 @@ SELECT sum(a), t2.a, t2.b FROM t2 HAVING t2.a IN (SELECT t2.b FROM t1); sum(a) a b 6 1 1 DROP TABLE t1,t2; +# +# MDEV-17606: Query returns wrong results (while using CHARACTER SET utf8) +# +CREATE TABLE t1(l1 varchar(10), i2 int); +INSERT INTO t1 VALUES ('e',2),('o',6),('x',4); +CREATE TABLE t2 (v1 varchar(10) CHARACTER SET utf8, KEY v1 (v1(3))); +INSERT INTO t2 VALUES ('k'),('rid'),('f'),('x'); +EXPLAIN EXTENDED SELECT * FROM t1 where ( t1.l1 < ANY (SELECT MAX(t2.v1) FROM t2)); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 Using where +2 SUBQUERY t2 ALL NULL NULL NULL NULL 4 100.00 +Warnings: +Note 1003 select `test`.`t1`.`l1` AS `l1`,`test`.`t1`.`i2` AS `i2` from `test`.`t1` where ((`test`.`t1`.`l1`,((select max(`test`.`t2`.`v1`) from `test`.`t2`) > convert((`test`.`t1`.`l1`) using utf8)))) +SELECT * FROM t1 where ( t1.l1 < ANY (SELECT MAX(t2.v1) FROM t2)); +l1 i2 +e 2 +o 6 +DROP TABLE t1, t2; diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index f0b1d16be7b..b7a9c95abe7 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -2099,3 +2099,16 @@ SET @@sql_select_limit= @save_sql_select_limit; eval EXPLAIN EXTENDED $query; eval $query; DROP TABLE t1,t2; + +--echo # +--echo # MDEV-17606: Query returns wrong results (while using CHARACTER SET utf8) +--echo # + +CREATE TABLE t1(l1 varchar(10), i2 int); +INSERT INTO t1 VALUES ('e',2),('o',6),('x',4); +CREATE TABLE t2 (v1 varchar(10) CHARACTER SET utf8, KEY v1 (v1(3))); +INSERT INTO t2 VALUES ('k'),('rid'),('f'),('x'); + +EXPLAIN EXTENDED SELECT * FROM t1 where ( t1.l1 < ANY (SELECT MAX(t2.v1) FROM t2)); +SELECT * FROM t1 where ( t1.l1 < ANY (SELECT MAX(t2.v1) FROM t2)); +DROP TABLE t1, t2; diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 16ef8a192c5..ebe8e23add5 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -2015,7 +2015,7 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join) The swap is needed for expressions of type 'f1 < ALL ( SELECT ....)' where we want to evaluate the sub query even if f1 would be null. */ - subs= func->create_swap(thd, *(optimizer->get_cache()), subs); + subs= func->create_swap(thd, expr, subs); thd->change_item_tree(place, subs); if (subs->fix_fields(thd, &subs)) DBUG_RETURN(true); From 9fff91b59bc9ce22c77164f0129a765e3be3e9f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Sat, 27 Jun 2020 17:28:51 +0300 Subject: [PATCH 50/73] Fix cross-compilation for systemd files Upstreamed from https://salsa.debian.org/mariadb-team/mariadb-10.4/-/blob/master/debian/patches/930314-cross-build.patch which has been running in Debian successfully for a year now. Original bug report: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=930314 --- cmake/systemd.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/systemd.cmake b/cmake/systemd.cmake index e353004e7d2..f48cca43142 100644 --- a/cmake/systemd.cmake +++ b/cmake/systemd.cmake @@ -17,6 +17,9 @@ MACRO(CHECK_SYSTEMD) IF(UNIX) INCLUDE(FindPkgConfig) # http://www.cmake.org/cmake/help/v3.0/module/FindPkgConfig.html + INCLUDE(CheckFunctionExists) + INCLUDE(CheckIncludeFiles) + INCLUDE(CheckLibraryExists) SET(WITH_SYSTEMD "auto" CACHE STRING "Enable systemd scripts and notification support") IF(WITH_SYSTEMD STREQUAL "yes" OR WITH_SYSTEMD STREQUAL "auto") IF(PKG_CONFIG_FOUND) From d4d42a6ad07d4b6a3ea3f4f5aee29da0fd887675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Sat, 27 Jun 2020 17:36:53 +0300 Subject: [PATCH 51/73] Revert "Fix cross-compilation for systemd files" This reverts commit 9fff91b59bc9ce22c77164f0129a765e3be3e9f3 which was pushed on master by accident. --- cmake/systemd.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmake/systemd.cmake b/cmake/systemd.cmake index f48cca43142..e353004e7d2 100644 --- a/cmake/systemd.cmake +++ b/cmake/systemd.cmake @@ -17,9 +17,6 @@ MACRO(CHECK_SYSTEMD) IF(UNIX) INCLUDE(FindPkgConfig) # http://www.cmake.org/cmake/help/v3.0/module/FindPkgConfig.html - INCLUDE(CheckFunctionExists) - INCLUDE(CheckIncludeFiles) - INCLUDE(CheckLibraryExists) SET(WITH_SYSTEMD "auto" CACHE STRING "Enable systemd scripts and notification support") IF(WITH_SYSTEMD STREQUAL "yes" OR WITH_SYSTEMD STREQUAL "auto") IF(PKG_CONFIG_FOUND) From ca55e09e9a9a5e30c4137384d96a3bc07493fb01 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 23 Jun 2020 15:37:41 +1000 Subject: [PATCH 52/73] signal handler: use mariadb kb URL rather than MySQL one --- sql/signal_handler.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index d9b3fece854..05e1d2176f9 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -229,7 +229,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) } my_safe_printf_stderr("%s", "The manual page at " - "http://dev.mysql.com/doc/mysql/en/crashing.html contains\n" + "https://mariadb.com/kb/en/how-to-produce-a-full-stack-trace-for-mysqld/ contains\n" "information that should help you find out what is causing the crash.\n"); #endif /* HAVE_STACKTRACE */ From 17109001d6db712187ff2ccb43f6c18b01458318 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Tue, 30 Jun 2020 12:00:54 +0300 Subject: [PATCH 53/73] speed up fil_validate() in debug builds This function is very common in a debug build. I can even see it in profiler. This patch reduces execution time of fil_validate() from 8948ns 8367ns 8650ns 8906ns 8448ns to 260ns 232ns 403ns 275ns 169ns in my environment. The trick is a faster fil_space_t iteration. Hash table is typically initialized with a size of 50,000. And looping through it is slow. Slower, than iterating an exact amount of fil_space_t which is typically less than ten. Only debug builds are affected. --- storage/innobase/fil/fil0fil.cc | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 69790c44617..42f7a7fc3c9 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -5314,24 +5314,15 @@ bool fil_validate(void) /*==============*/ { - fil_space_t* space; fil_node_t* fil_node; ulint n_open = 0; mutex_enter(&fil_system->mutex); - /* Look for spaces in the hash table */ - - for (ulint i = 0; i < hash_get_n_cells(fil_system->spaces); i++) { - - for (space = static_cast( - HASH_GET_FIRST(fil_system->spaces, i)); - space != 0; - space = static_cast( - HASH_GET_NEXT(hash, space))) { - - n_open += Check::validate(space); - } + for (fil_space_t *space = UT_LIST_GET_FIRST(fil_system->space_list); + space != NULL; + space = UT_LIST_GET_NEXT(space_list, space)) { + n_open += Check::validate(space); } ut_a(fil_system->n_open == n_open); From 17ce75b9367bdef0e90572d2b94100609df783a5 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 30 Jun 2020 12:45:37 +0200 Subject: [PATCH 54/73] MDEV-23052 mysql_install_db.exe can run on existing non-empty directory, and remove it on error Disable existing non-empty datadir for mysql_install_db.exe --- sql/mysql_install_db.cc | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index b841c64c331..47ce6171357 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -426,8 +426,8 @@ static int register_service() static void clean_directory(const char *dir) { - char dir2[MAX_PATH+2]; - *(strmake_buf(dir2, dir)+1)= 0; + char dir2[MAX_PATH + 4]= {}; + snprintf(dir2, MAX_PATH+2, "%s\\*", dir); SHFILEOPSTRUCT fileop; fileop.hwnd= NULL; /* no status display */ @@ -556,7 +556,7 @@ static int create_db_instance() DWORD cwd_len= MAX_PATH; char cmdline[3*MAX_PATH]; FILE *in; - bool cleanup_datadir= true; + bool created_datadir= false; DWORD last_error; verbose("Running bootstrap"); @@ -565,7 +565,11 @@ static int create_db_instance() /* Create datadir and datadir/mysql, if they do not already exist. */ - if (!CreateDirectory(opt_datadir, NULL) && (GetLastError() != ERROR_ALREADY_EXISTS)) + if (CreateDirectory(opt_datadir, NULL)) + { + created_datadir= true; + } + else if (GetLastError() != ERROR_ALREADY_EXISTS) { last_error = GetLastError(); switch(last_error) @@ -602,9 +606,11 @@ static int create_db_instance() } } - if (PathIsDirectoryEmpty(opt_datadir)) + if (!PathIsDirectoryEmpty(opt_datadir)) { - cleanup_datadir= false; + fprintf(stderr,"ERROR : Data directory %s is not empty." + " Only new or empty existing directories are accepted for --datadir\n",opt_datadir); + exit(1); } if (!CreateDirectory("mysql",NULL)) @@ -732,10 +738,12 @@ static int create_db_instance() } end: - if (ret && cleanup_datadir) + if (ret) { SetCurrentDirectory(cwd); clean_directory(opt_datadir); + if (created_datadir) + RemoveDirectory(opt_datadir); } return ret; } From 59c999fc98dedaab9e192a7d25b626ac314e4a84 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 30 Jun 2020 12:47:05 +0200 Subject: [PATCH 55/73] MDEV-23052 - add mysql_install_db.exe test with existing directory. --- mysql-test/main/mysql_install_db_win.result | 2 ++ mysql-test/main/mysql_install_db_win.test | 24 +++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/mysql-test/main/mysql_install_db_win.result b/mysql-test/main/mysql_install_db_win.result index 950ff868035..31336847a38 100644 --- a/mysql-test/main/mysql_install_db_win.result +++ b/mysql-test/main/mysql_install_db_win.result @@ -11,5 +11,7 @@ SELECT @@datadir; @@datadir DATADIR/ # Kill the server +1 +FOUND 1 /is not empty. Only new or empty existing directories are accepted for --datadir/ in install.log connection default; # restart diff --git a/mysql-test/main/mysql_install_db_win.test b/mysql-test/main/mysql_install_db_win.test index 7bf62903219..3f0256e860e 100644 --- a/mysql-test/main/mysql_install_db_win.test +++ b/mysql-test/main/mysql_install_db_win.test @@ -18,6 +18,30 @@ SELECT @@datadir; # restart in the original datadir again --source include/kill_mysqld.inc rmdir $ddir; + +# MDEV-23052 +# 1. mysql_install_db works on existing, empty directory +mkdir $ddir; +exec $MYSQL_INSTALL_DB_EXE --datadir=$ddir --password=foo -R > /dev/null; +rmdir $ddir; + +# 2. mysql_install_db rejects existing, non-empty directory, and does not +# remove it. +mkdir $ddir; +write_file $ddir/1; +EOF + +error 1; +exec $MYSQL_INSTALL_DB_EXE --datadir=$ddir --password=foo -R > $MYSQLTEST_VARDIR/tmp/install.log 2>&1; +list_files $ddir; +let $log=$MYSQLTEST_VARDIR/tmp/install.log; +let SEARCH_FILE=$log; +let SEARCH_PATTERN=is not empty. Only new or empty existing directories are accepted for --datadir; +--source include/search_pattern_in_file.inc +remove_file $log; + +rmdir $ddir; + let $restart_parameters=; connection default; --source include/start_mysqld.inc From 8e8f9671cbef43ef347e2c942de074278847241f Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 30 Jun 2020 14:03:22 +0200 Subject: [PATCH 56/73] MDEV-21773: added missing include file to mtr tests --- mysql-test/suite/galera/t/galera_as_slave_gtid.inc | 1 + mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.cnf | 2 +- mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.test | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid.inc b/mysql-test/suite/galera/t/galera_as_slave_gtid.inc index 461e7833a94..ca6cce52497 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_gtid.inc +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid.inc @@ -8,6 +8,7 @@ # --source include/have_innodb.inc +--source include/have_log_bin.inc --source include/galera_cluster.inc # As node #3 is not a Galera node, and galera_cluster.inc does not open connetion to it diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.cnf b/mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.cnf index adcba9b6069..efabe4161aa 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.cnf +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.cnf @@ -5,4 +5,4 @@ log-bin=mysqld-bin log-slave-updates binlog-format=ROW -gtid_pos_auto_engines=InnoDB \ No newline at end of file +gtid_pos_auto_engines=InnoDB diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.test b/mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.test index 990dd35f40e..5ef98573660 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.test +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid_auto_engine.test @@ -11,4 +11,4 @@ --connection node_2 DROP TABLE mysql.gtid_slave_pos_InnoDB; -CALL mtr.add_suppression("The automatically created table"); \ No newline at end of file +CALL mtr.add_suppression("The automatically created table"); From 4a2e7b5368b2c78c5965b74727727053c67b406d Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Tue, 30 Jun 2020 18:16:01 +0530 Subject: [PATCH 57/73] MDEV-22984: Throw an error when arguments to window functions are window functions Window function is not allowed as arguments to window functions according to the standard. --- mysql-test/r/win.result | 10 ++++++++++ mysql-test/t/win.test | 13 +++++++++++++ sql/item_windowfunc.cc | 1 + 3 files changed, 24 insertions(+) diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result index c73f9f8ce6b..9c85315b7c1 100644 --- a/mysql-test/r/win.result +++ b/mysql-test/r/win.result @@ -3788,5 +3788,15 @@ row_number() OVER() 3 DROP TABLE t1; # +# MDEV-22984: Throw an error when arguments to window functions are window functions +# +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 VALUES (1,1), (2,2), (3,3); +SELECT NTILE(MAX(a) OVER (PARTITION BY a)) OVER (PARTITION BY a ORDER BY b) FROM t1; +ERROR HY000: Window functions can not be used as arguments to group functions. +SELECT FIRST_VALUE(MAX(a) OVER (PARTITION BY a)) OVER (ORDER BY a) AS x FROM t1 GROUP BY a; +ERROR HY000: Window functions can not be used as arguments to group functions. +DROP TABLE t1; +# # End of 10.2 tests # diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test index 37c107633d9..d19ff2c624d 100644 --- a/mysql-test/t/win.test +++ b/mysql-test/t/win.test @@ -2459,6 +2459,19 @@ ANALYZE FORMAT=JSON SELECT row_number() OVER() FROM t1; SELECT row_number() OVER() FROM t1; DROP TABLE t1; +--echo # +--echo # MDEV-22984: Throw an error when arguments to window functions are window functions +--echo # + +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 VALUES (1,1), (2,2), (3,3); + +--error ER_SUM_FUNC_WITH_WINDOW_FUNC_AS_ARG +SELECT NTILE(MAX(a) OVER (PARTITION BY a)) OVER (PARTITION BY a ORDER BY b) FROM t1; +--error ER_SUM_FUNC_WITH_WINDOW_FUNC_AS_ARG +SELECT FIRST_VALUE(MAX(a) OVER (PARTITION BY a)) OVER (ORDER BY a) AS x FROM t1 GROUP BY a; +DROP TABLE t1; + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/item_windowfunc.cc b/sql/item_windowfunc.cc index 7cb07af440b..a3edacd880e 100644 --- a/sql/item_windowfunc.cc +++ b/sql/item_windowfunc.cc @@ -237,6 +237,7 @@ bool Item_sum_hybrid_simple::fix_fields(THD *thd, Item **ref) if ((!item->fixed && item->fix_fields(thd, args)) || (item= args[i])->check_cols(1)) return TRUE; + with_window_func|= item->with_window_func; } Type_std_attributes::set(args[0]); for (uint i= 0; i < arg_count && !with_subselect; i++) From cc0dca366357651ddb549e31a12b1ecd39c7380e Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Tue, 30 Jun 2020 18:11:35 +0530 Subject: [PATCH 58/73] MDEV-22910: SIGSEGV in Opt_trace_context::is_started & SIGSEGV in Json_writer::add_table_name (on optimized builds) Make sure to initialize members of TABLE::reginfo when TABLE::init is called. In this case the problem was that table->reginfo.join_tab was set for the SELECT query and then was reused by the UPDATE query. This case occurred only when the SELECT query had a degenerate join. --- mysql-test/main/opt_trace.result | 31 +++++++++++++++++++++++++++++++ mysql-test/main/opt_trace.test | 11 +++++++++++ sql/opt_range.cc | 6 ++---- sql/table.cc | 2 ++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result index 6da22802cca..ee1273decf6 100644 --- a/mysql-test/main/opt_trace.result +++ b/mysql-test/main/opt_trace.result @@ -8559,5 +8559,36 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) ] ] drop table t1; +# +# MDEV-22910:SIGSEGV in Opt_trace_context::is_started & SIGSEGV in Json_writer::add_table_name +# (on optimized builds) +# +CREATE TABLE t1( a INT, b INT, PRIMARY KEY( a ) ); +SELECT sum(b), row_number() OVER (order by b) FROM t1 WHERE a = 101; +sum(b) row_number() OVER (order by b) +NULL 1 +UPDATE t1 SET b=10 WHERE a=1; +SELECT JSON_DETAILED(JSON_EXTRACT(trace, '$**.range_scan_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; +JSON_DETAILED(JSON_EXTRACT(trace, '$**.range_scan_alternatives')) +[ + + [ + + { + "index": "PRIMARY", + "ranges": + [ + "(1) <= (a) <= (1)" + ], + "rowid_ordered": true, + "using_mrr": false, + "index_only": false, + "rows": 0, + "cost": 1.125, + "chosen": true + } + ] +] +DROP TABLE t1; set optimizer_trace='enabled=off'; # End of 10.4 tests diff --git a/mysql-test/main/opt_trace.test b/mysql-test/main/opt_trace.test index 8633ed5b020..9040b5a54d0 100644 --- a/mysql-test/main/opt_trace.test +++ b/mysql-test/main/opt_trace.test @@ -611,5 +611,16 @@ EXPLAIN EXTENDED SELECT * from t1 WHERE b >= 10 and b < 25; select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; drop table t1; +--echo # +--echo # MDEV-22910:SIGSEGV in Opt_trace_context::is_started & SIGSEGV in Json_writer::add_table_name +--echo # (on optimized builds) +--echo # + +CREATE TABLE t1( a INT, b INT, PRIMARY KEY( a ) ); +SELECT sum(b), row_number() OVER (order by b) FROM t1 WHERE a = 101; +UPDATE t1 SET b=10 WHERE a=1; +SELECT JSON_DETAILED(JSON_EXTRACT(trace, '$**.range_scan_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; +DROP TABLE t1; + set optimizer_trace='enabled=off'; --echo # End of 10.4 tests diff --git a/sql/opt_range.cc b/sql/opt_range.cc index ecbf5d6afcb..86935f3ef27 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -2690,10 +2690,8 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, DBUG_PRINT("info",("Time to scan table: %g", read_time)); Json_writer_object table_records(thd); - if (head->reginfo.join_tab) - table_records.add_table_name(head->reginfo.join_tab); - else - table_records.add_table_name(head); + table_records.add_table_name(head); + Json_writer_object trace_range(thd, "range_analysis"); { Json_writer_object table_rec(thd, "table_scan"); diff --git a/sql/table.cc b/sql/table.cc index 648b1794582..7f2f008cba0 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -5160,6 +5160,8 @@ void TABLE::init(THD *thd, TABLE_LIST *tl) fulltext_searched= 0; file->ft_handler= 0; reginfo.impossible_range= 0; + reginfo.join_tab= NULL; + reginfo.not_exists_optimize= FALSE; created= TRUE; cond_selectivity= 1.0; cond_selectivity_sampling_explain= NULL; From 1ea266f3ef36e779d23697b18cb94e1b0f8e65ef Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Mon, 29 Jun 2020 15:39:01 +0300 Subject: [PATCH 59/73] MDEV-23003 INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION requires SUPER instead PROCESS privilege Fix a typo in a source code. Now real required privileges corresponds to a ones mentions in documentation. Documentation states that this table requires PROCESS privilege: https://mariadb.com/kb/en/information-schema-innodb_tablespaces_encryption-table/ --- storage/innobase/handler/i_s.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 2729a755570..21eb9da481e 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -8288,7 +8288,7 @@ i_s_tablespaces_encryption_fill_table( RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); /* deny access to user without PROCESS_ACL privilege */ - if (check_global_access(thd, SUPER_ACL)) { + if (check_global_access(thd, PROCESS_ACL)) { DBUG_RETURN(0); } From fbfb5b5f68a428ec819bc09d14b30cebf660b37b Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Wed, 1 Jul 2020 11:39:22 +0530 Subject: [PATCH 60/73] MDEV-22852: SIGSEGV in sortlength (optimized builds) The issue here is for a DEPENDENT subquery that has an aggregate function in the ORDER BY clause, is wrapped inside an Item_aggregate_ref. For computation of ORDER BY we need to refer to the temp table field corresponding to this item. But in the function make_sortorder, we were explicitly casting Item_aggrgate_ref to Item_sum, which leads to us not getting the temp table field corresponding to the item. --- mysql-test/r/subselect4.result | 12 ++++++++++++ mysql-test/t/subselect4.test | 12 ++++++++++++ sql/filesort.cc | 9 ++++++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 51f199bd1e9..a718895dc34 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2566,3 +2566,15 @@ SELECT sum(a), t2.a, t2.b FROM t2 HAVING t2.a IN (SELECT t2.b FROM t1); sum(a) a b 6 1 1 DROP TABLE t1,t2; +# +# MDEV-22852: SIGSEGV in sortlength (optimized builds) +# +SET @save_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='subquery_cache=off'; +CREATE TABLE t1 (a INT,b INT); +INSERT INTO t1 VALUES (0,0),(0,0); +SELECT (SELECT DISTINCT t1i.b FROM t1 t1i GROUP BY t1i.a ORDER BY MAX(t1o.b)) FROM t1 AS t1o; +(SELECT DISTINCT t1i.b FROM t1 t1i GROUP BY t1i.a ORDER BY MAX(t1o.b)) +0 +SET @@optimizer_switch= @save_optimizer_switch; +DROP TABLE t1; diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index 2b3f610d06f..057a4502684 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -2101,3 +2101,15 @@ SET @@sql_select_limit= @save_sql_select_limit; eval EXPLAIN EXTENDED $query; eval $query; DROP TABLE t1,t2; + +--echo # +--echo # MDEV-22852: SIGSEGV in sortlength (optimized builds) +--echo # + +SET @save_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='subquery_cache=off'; +CREATE TABLE t1 (a INT,b INT); +INSERT INTO t1 VALUES (0,0),(0,0); +SELECT (SELECT DISTINCT t1i.b FROM t1 t1i GROUP BY t1i.a ORDER BY MAX(t1o.b)) FROM t1 AS t1o; +SET @@optimizer_switch= @save_optimizer_switch; +DROP TABLE t1; diff --git a/sql/filesort.cc b/sql/filesort.cc index 1df2f93676d..f8952f6d711 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -483,7 +483,14 @@ uint Filesort::make_sortorder(THD *thd, JOIN *join, table_map first_table_bit) if (item->type() == Item::FIELD_ITEM) pos->field= ((Item_field*) item)->field; else if (item->type() == Item::SUM_FUNC_ITEM && !item->const_item()) - pos->field= ((Item_sum*) item)->get_tmp_table_field(); + { + // Aggregate, or Item_aggregate_ref + DBUG_ASSERT(first->type() == Item::SUM_FUNC_ITEM || + (first->type() == Item::REF_ITEM && + static_cast(first)->ref_type() == + Item_ref::AGGREGATE_REF)); + pos->field= first->get_tmp_table_field(); + } else if (item->type() == Item::COPY_STR_ITEM) { // Blob patch pos->item= ((Item_copy*) item)->get_item(); From 2ed415765aca144ea0177633cb5ac56a0c65be44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Sun, 7 Jun 2020 09:58:15 +0300 Subject: [PATCH 61/73] Fix RocksDB detection of ZSTD The library finder needs to have capitals in its name so that FIND_PACKAGE will load the correct finder and actually detect that libzstd is available. Without this change the CMake would just always silently skip ZSTD since it would never find it. Simplify Debian autopkgtest RocksDB part and make it more verbose so that future regressions like this are easier to debug. Also remove QUIET from the RocksDB FIND_PACKAGE call so that it is easier to read in build logs what libraries were detected. Also add missing underscores to error messages. --- cmake/{Findzstd.cmake => FindZSTD.cmake} | 0 storage/rocksdb/build_rocksdb.cmake | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename cmake/{Findzstd.cmake => FindZSTD.cmake} (100%) diff --git a/cmake/Findzstd.cmake b/cmake/FindZSTD.cmake similarity index 100% rename from cmake/Findzstd.cmake rename to cmake/FindZSTD.cmake diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index 310ca157550..f7e05330dd5 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -36,15 +36,15 @@ endif() # Optional compression libraries. foreach(compression_lib LZ4 BZip2 ZSTD snappy) - FIND_PACKAGE(${compression_lib} QUIET) + FIND_PACKAGE(${compression_lib}) SET(WITH_ROCKSDB_${compression_lib} AUTO CACHE STRING "Build RocksDB with ${compression_lib} compression. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") if(${WITH_ROCKSDB_${compression_lib}} STREQUAL "ON" AND NOT ${${compression_lib}_FOUND}) MESSAGE(FATAL_ERROR - "${compression_lib} library was not found, but WITH_ROCKSDB${compression_lib} option is ON.\ - Either set WITH_ROCKSDB${compression_lib} to OFF, or make sure ${compression_lib} is installed") + "${compression_lib} library was not found, but WITH_ROCKSDB_${compression_lib} option is ON.\ + Either set WITH_ROCKSDB_${compression_lib} to OFF, or make sure ${compression_lib} is installed") endif() endforeach() From 6d3747a294d75ab8153f82c5b2d61c3bf17b04de Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 28 Jun 2020 16:31:55 +0200 Subject: [PATCH 62/73] make rocksdb cmake checks less verbose on repeat * fix FindLZ4 to follow convention (LIBRARIES, not LIBRARY) * remove redundant checks from rocksdb/CMakeLists.txt * put all checks through the same macro that uniformly checks for a package, prints the message, adds definition --- cmake/FindLZ4.cmake | 6 +-- cmake/FindZSTD.cmake | 6 +-- storage/rocksdb/CMakeLists.txt | 10 ---- storage/rocksdb/build_rocksdb.cmake | 81 ++++++++++++++--------------- 4 files changed, 42 insertions(+), 61 deletions(-) diff --git a/cmake/FindLZ4.cmake b/cmake/FindLZ4.cmake index e97dd63e2b0..46c1fdc1b9f 100644 --- a/cmake/FindLZ4.cmake +++ b/cmake/FindLZ4.cmake @@ -1,9 +1,9 @@ find_path(LZ4_INCLUDE_DIR NAMES lz4.h) -find_library(LZ4_LIBRARY NAMES lz4) +find_library(LZ4_LIBRARIES NAMES lz4) include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS( LZ4 DEFAULT_MSG - LZ4_LIBRARY LZ4_INCLUDE_DIR) + LZ4_LIBRARIES LZ4_INCLUDE_DIR) -mark_as_advanced(LZ4_INCLUDE_DIR LZ4_LIBRARY) +mark_as_advanced(LZ4_INCLUDE_DIR LZ4_LIBRARIES) diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake index 8cd4c248145..ede59010742 100644 --- a/cmake/FindZSTD.cmake +++ b/cmake/FindZSTD.cmake @@ -5,10 +5,6 @@ # ZSTD_LIBRARIES - List of libraries when using zstd. # ZSTD_FOUND - True if zstd found. -if (DEFINED ZSTD_LIBRARIES) - return() -endif() - find_path(ZSTD_INCLUDE_DIR NAMES zstd.h HINTS ${ZSTD_ROOT_DIR}/include) @@ -18,7 +14,7 @@ find_library(ZSTD_LIBRARIES HINTS ${ZSTD_ROOT_DIR}/lib) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIR) +find_package_handle_standard_args(ZSTD DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIR) mark_as_advanced( ZSTD_LIBRARIES diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index 3a707d8c02b..7382df8ecf7 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -188,9 +188,6 @@ endif() TARGET_LINK_LIBRARIES(rocksdb rocksdb_aux_lib) - FIND_LIBRARY(LZ4_LIBRARY - NAMES liblz4${PIC_EXT}.a lz4 - HINTS ${WITH_LZ4}/lib) IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -273,13 +270,6 @@ IF(MSVC) ENDIF() ENDIF() -# Enable ZSTD if available. Upstream rocksdb cmake will use WITH_ZSTD and set -# defines within their code. -FIND_PACKAGE(zstd) -IF (ZSTD_FOUND) - SET(WITH_ZSTD ON) -ENDIF() - IF(GIT_EXECUTABLE AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/.git) EXECUTE_PROCESS( COMMAND ${GIT_EXECUTABLE} rev-parse HEAD diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index f7e05330dd5..78581182e7e 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -35,52 +35,47 @@ endif() # Optional compression libraries. -foreach(compression_lib LZ4 BZip2 ZSTD snappy) - FIND_PACKAGE(${compression_lib}) - - SET(WITH_ROCKSDB_${compression_lib} AUTO CACHE STRING - "Build RocksDB with ${compression_lib} compression. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") - - if(${WITH_ROCKSDB_${compression_lib}} STREQUAL "ON" AND NOT ${${compression_lib}_FOUND}) - MESSAGE(FATAL_ERROR - "${compression_lib} library was not found, but WITH_ROCKSDB_${compression_lib} option is ON.\ - Either set WITH_ROCKSDB_${compression_lib} to OFF, or make sure ${compression_lib} is installed") - endif() -endforeach() - -if(LZ4_FOUND AND (NOT WITH_ROCKSDB_LZ4 STREQUAL "OFF")) - add_definitions(-DLZ4) - include_directories(${LZ4_INCLUDE_DIR}) - list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARY}) -endif() - -if(BZIP2_FOUND AND (NOT WITH_ROCKSDB_BZip2 STREQUAL "OFF")) - add_definitions(-DBZIP2) - include_directories(${BZIP2_INCLUDE_DIR}) - list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES}) -endif() - -if(SNAPPY_FOUND AND (NOT WITH_ROCKSDB_snappy STREQUAL "OFF")) - add_definitions(-DSNAPPY) - include_directories(${snappy_INCLUDE_DIR}) - list(APPEND THIRDPARTY_LIBS ${snappy_LIBRARIES}) -endif() - include(CheckFunctionExists) -if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_ZSTD STREQUAL "OFF")) - SET(CMAKE_REQUIRED_LIBRARIES zstd) - CHECK_FUNCTION_EXISTS(ZDICT_trainFromBuffer ZSTD_VALID) - UNSET(CMAKE_REQUIRED_LIBRARIES) - if (WITH_ROCKSDB_ZSTD STREQUAL "ON" AND NOT ZSTD_VALID) +macro(check_lib package var) + STRING(TOUPPER ${package} PACKAGE_NAME) + SET(WITH_ROCKSDB_${package} AUTO CACHE STRING + "Build RocksDB with ${package} compression. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") + + IF (NOT ${WITH_ROCKSDB_${package}} STREQUAL "OFF") + FIND_PACKAGE(${package} QUIET) + IF (${${PACKAGE_NAME}_FOUND}) + IF(${ARGC} GREATER 2) + SET(CMAKE_REQUIRED_LIBRARIES ${${var}_LIBRARIES}) + CHECK_FUNCTION_EXISTS(${ARGV2} ${var}_VALID) + UNSET(CMAKE_REQUIRED_LIBRARIES) + ELSE() + SET(${var}_VALID TRUE) + ENDIF() + ENDIF() + ENDIF() + + IF(${${var}_VALID}) + MESSAGE_ONCE(rocksdb_${var} "Found ${package}: ${${var}_LIBRARIES}") + add_definitions(-D${PACKAGE_NAME}) + include_directories(${${var}_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS ${${var}_LIBRARIES}) + ELSEIF(${${PACKAGE_NAME}_FOUND}) + MESSAGE_ONCE(rocksdb_${var} "Found unusable ${package}: ${${var}_LIBRARIES} [${ARGV2}]") + ELSE() + MESSAGE_ONCE(rocksdb_${var} "Could NOT find ${package}") + ENDIF() + + IF (${WITH_ROCKSDB_${package}} STREQUAL "ON" AND NOT ${${PACKAGE_NAME}_FOUND}) MESSAGE(FATAL_ERROR - "WITH_ROCKSDB_ZSTD is ON and ZSTD library was found, but the version needs to be >= 1.1.3") + "${package} library was not found, but WITH_ROCKSDB_${package} option is ON.\ + Either set WITH_ROCKSDB_${package} to OFF, or make sure ${package} is installed") endif() - if (ZSTD_VALID) - add_definitions(-DZSTD) - include_directories(${ZSTD_INCLUDE_DIR}) - list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARIES}) - endif() -endif() +endmacro() + +check_lib(LZ4 LZ4) +check_lib(BZip2 BZIP2) +check_lib(snappy snappy) # rocksdb/cmake/modules/Findsnappy.cmake violates the convention +check_lib(ZSTD ZSTD ZDICT_trainFromBuffer) add_definitions(-DZLIB) list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARY}) From fe05c16c8d6d9ff90186578cdc3db460469e4fe1 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 30 Jun 2020 12:45:37 +0200 Subject: [PATCH 63/73] MDEV-23052 mysql_install_db.exe can run on existing non-empty directory, and remove it on error Disable existing non-empty datadir for mysql_install_db.exe --- sql/mysql_install_db.cc | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index c6912e41f6e..022ba18a3e3 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -399,8 +399,8 @@ static int register_service() static void clean_directory(const char *dir) { - char dir2[MAX_PATH+2]; - *(strmake_buf(dir2, dir)+1)= 0; + char dir2[MAX_PATH + 4]= {}; + snprintf(dir2, MAX_PATH+2, "%s\\*", dir); SHFILEOPSTRUCT fileop; fileop.hwnd= NULL; /* no status display */ @@ -551,7 +551,7 @@ static int create_db_instance() DWORD cwd_len= MAX_PATH; char cmdline[3*MAX_PATH]; FILE *in; - bool cleanup_datadir= true; + bool created_datadir= false; DWORD last_error; verbose("Running bootstrap"); @@ -560,7 +560,11 @@ static int create_db_instance() /* Create datadir and datadir/mysql, if they do not already exist. */ - if (!CreateDirectory(opt_datadir, NULL) && (GetLastError() != ERROR_ALREADY_EXISTS)) + if (CreateDirectory(opt_datadir, NULL)) + { + created_datadir= true; + } + else if (GetLastError() != ERROR_ALREADY_EXISTS) { last_error = GetLastError(); switch(last_error) @@ -597,9 +601,11 @@ static int create_db_instance() } } - if (PathIsDirectoryEmpty(opt_datadir)) + if (!PathIsDirectoryEmpty(opt_datadir)) { - cleanup_datadir= false; + fprintf(stderr,"ERROR : Data directory %s is not empty." + " Only new or empty existing directories are accepted for --datadir\n",opt_datadir); + exit(1); } if (!CreateDirectory("mysql",NULL)) @@ -735,10 +741,12 @@ static int create_db_instance() } end: - if (ret && cleanup_datadir) + if (ret) { SetCurrentDirectory(cwd); clean_directory(opt_datadir); + if (created_datadir) + RemoveDirectory(opt_datadir); } return ret; } From 9ed50ece339358fcd5ebd8089f3c780a75b5f520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 1 Jul 2020 17:18:47 +0300 Subject: [PATCH 64/73] MDEV-22779: Fix a memory leak in the unit test --- tests/mysql_client_test.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index 33655b80662..ec878ae830c 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -19889,6 +19889,8 @@ static void test_ps_params_in_ctes() rc= mysql_stmt_store_result(stmt); check_execute(stmt, rc); + mysql_stmt_close(stmt); + rc= mysql_query(mysql, "drop table t1"); myquery(rc); } From 5a097c5556dffc1aec73616f58cecf9345d96050 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 13 Mar 2020 14:59:02 +0100 Subject: [PATCH 65/73] MDEV-21222 mariabackup.incremental_backup failed with memory allocation failure mariabackup tries to allocate a buffer of page_size*page_size/4 size. for 64k page it means 1Gb, which doesn't work very well on 32-bit builders. Skip the 64k page test on 32bit. --- mysql-test/suite/mariabackup/incremental_backup.test | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mysql-test/suite/mariabackup/incremental_backup.test b/mysql-test/suite/mariabackup/incremental_backup.test index 3e877af1398..88e277fd95a 100644 --- a/mysql-test/suite/mariabackup/incremental_backup.test +++ b/mysql-test/suite/mariabackup/incremental_backup.test @@ -1,6 +1,11 @@ --source include/have_aria.inc --source include/innodb_page_size.inc +# see suite.pm "check for exact values, in case the default changes to be small everywhere" +if (`select @@max_binlog_stmt_cache_size = 4294963200 and @@innodb_page_size = 65536`) { + skip skipped on 32bit; # tries to allocate 1GB of memory + } + call mtr.add_suppression("InnoDB: New log files created"); let basedir=$MYSQLTEST_VARDIR/tmp/backup; From c36834c8324974f26770d64192898f4f45d9f772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 1 Jul 2020 17:23:00 +0300 Subject: [PATCH 66/73] MDEV-20377: Make WITH_MSAN more usable MemorySanitizer (clang -fsanitize=memory) requires that all code be compiled with instrumentation enabled. The only exception is the C runtime library. Failure to use instrumented libraries will cause bogus messages about memory being uninitialized. In WITH_MSAN builds, we must avoid calling getservbyname(), because even though it is a standard library function, it is not instrumented, not even in clang 10. Note: Before MariaDB Server 10.5, ./mtr will typically fail due to the old PCRE library, which was updated in MDEV-14024. The following cmake options were tested on 10.5 in commit 94d0bb4dbeb28a94d1f87fdd55f4297ff3df0157: cmake \ -DCMAKE_C_FLAGS='-march=native -O2' \ -DCMAKE_CXX_FLAGS='-stdlib=libc++ -march=native -O2' \ -DWITH_EMBEDDED_SERVER=OFF -DWITH_UNIT_TESTS=OFF -DCMAKE_BUILD_TYPE=Debug \ -DWITH_INNODB_{BZIP2,LZ4,LZMA,LZO,SNAPPY}=OFF \ -DPLUGIN_{ARCHIVE,TOKUDB,MROONGA,OQGRAPH,ROCKSDB,CONNECT,SPIDER}=NO \ -DWITH_SAFEMALLOC=OFF \ -DWITH_{ZLIB,SSL,PCRE}=bundled \ -DHAVE_LIBAIO_H=0 \ -DWITH_MSAN=ON MEM_MAKE_DEFINED(): An alias for VALGRIND_MAKE_MEM_DEFINED() and __msan_unpoison(). MEM_GET_VBITS(), MEM_SET_VBITS(): Aliases for VALGRIND_GET_VBITS(), VALGRIND_SET_VBITS(), __msan_copy_shadow(). InnoDB: Replace the UNIV_MEM_ macros with corresponding MEM_ macros. ut_crc32_8_hw(), ut_crc32_64_low_hw(): Use the compiler built-in functions instead of inline assembler when building WITH_MSAN. This will require at least -msse4.2 when building for IA-32 or AMD64. The inline assembler would not be instrumented, and would thus cause bogus failures. --- include/my_valgrind.h | 8 ++ libmysqld/libmysql.c | 2 + sql/mysqld.cc | 2 + storage/innobase/btr/btr0cur.cc | 26 ++--- storage/innobase/buf/buf0buddy.cc | 41 ++++---- storage/innobase/buf/buf0buf.cc | 23 +---- storage/innobase/buf/buf0dblwr.cc | 7 +- storage/innobase/buf/buf0flu.cc | 31 ++---- storage/innobase/buf/buf0lru.cc | 45 ++++----- storage/innobase/data/data0data.cc | 24 +---- storage/innobase/dict/dict0stats.cc | 59 ++++++------ storage/innobase/handler/ha_innodb.cc | 8 +- storage/innobase/include/buf0buf.h | 2 +- storage/innobase/include/buf0buf.ic | 2 +- storage/innobase/include/data0data.ic | 20 ++-- storage/innobase/include/dict0stats.ic | 43 +++++---- storage/innobase/include/mem0mem.ic | 6 +- storage/innobase/include/page0zip.ic | 4 +- storage/innobase/include/rem0rec.ic | 4 +- storage/innobase/include/srv0mon.h | 10 +- storage/innobase/include/univ.i | 57 ----------- storage/innobase/include/ut0pool.h | 9 +- storage/innobase/log/log0recv.cc | 8 +- storage/innobase/mem/mem0mem.cc | 5 +- storage/innobase/mtr/mtr0mtr.cc | 2 +- storage/innobase/os/os0proc.cc | 10 +- storage/innobase/page/page0cur.cc | 26 ++--- storage/innobase/page/page0zip.cc | 126 +++++++++++++------------ storage/innobase/row/row0ext.cc | 6 +- storage/innobase/row/row0ftsort.cc | 10 +- storage/innobase/row/row0ins.cc | 6 +- storage/innobase/row/row0log.cc | 40 +++++--- storage/innobase/row/row0merge.cc | 41 ++++---- storage/innobase/row/row0mysql.cc | 4 +- storage/innobase/row/row0sel.cc | 38 +++++--- storage/innobase/row/row0upd.cc | 4 +- storage/innobase/sync/sync0arr.cc | 6 +- storage/innobase/trx/trx0trx.cc | 10 +- storage/innobase/ut/ut0crc32.cc | 26 ++--- storage/innobase/ut/ut0ut.cc | 6 -- strings/my_vsnprintf.c | 8 +- 41 files changed, 368 insertions(+), 447 deletions(-) diff --git a/include/my_valgrind.h b/include/my_valgrind.h index dc0c9bcfad4..a4fc738fd59 100644 --- a/include/my_valgrind.h +++ b/include/my_valgrind.h @@ -38,6 +38,8 @@ # define MEM_NOACCESS(a,len) VALGRIND_MAKE_MEM_NOACCESS(a,len) # define MEM_CHECK_ADDRESSABLE(a,len) VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,len) # define MEM_CHECK_DEFINED(a,len) VALGRIND_CHECK_MEM_IS_DEFINED(a,len) +# define MEM_GET_VBITS(a,b,len) VALGRIND_GET_VBITS(a,b,len) +# define MEM_SET_VBITS(a,b,len) VALGRIND_SET_VBITS(a,b,len) # define REDZONE_SIZE 8 #elif defined(__SANITIZE_ADDRESS__) # include @@ -48,6 +50,8 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */ # define MEM_NOACCESS(a,len) ASAN_POISON_MEMORY_REGION(a,len) # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0) # define MEM_CHECK_DEFINED(a,len) ((void) 0) +# define MEM_GET_VBITS(a,b,len) ((void) 0) +# define MEM_SET_VBITS(a,b,len) ((void) 0) # define REDZONE_SIZE 8 #elif __has_feature(memory_sanitizer) # include @@ -57,6 +61,8 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */ # define MEM_NOACCESS(a,len) ((void) 0) # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0) # define MEM_CHECK_DEFINED(a,len) __msan_check_mem_is_initialized(a,len) +# define MEM_GET_VBITS(a,b,len) __msan_copy_shadow(b,a,len) +# define MEM_SET_VBITS(a,b,len) __msan_copy_shadow(a,b,len) # define REDZONE_SIZE 8 #else # define MEM_UNDEFINED(a,len) ((void) (a), (void) (len)) @@ -64,6 +70,8 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */ # define MEM_NOACCESS(a,len) ((void) 0) # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0) # define MEM_CHECK_DEFINED(a,len) ((void) 0) +# define MEM_GET_VBITS(a,b,len) ((void) 0) +# define MEM_SET_VBITS(a,b,len) ((void) 0) # define REDZONE_SIZE 0 #endif /* HAVE_VALGRIND_MEMCHECK_H */ diff --git a/libmysqld/libmysql.c b/libmysqld/libmysql.c index 13f7f074d80..970076393d0 100644 --- a/libmysqld/libmysql.c +++ b/libmysqld/libmysql.c @@ -154,8 +154,10 @@ int STDCALL mysql_server_init(int argc __attribute__((unused)), */ #if MYSQL_PORT_DEFAULT == 0 +# if !__has_feature(memory_sanitizer) // Work around MSAN deficiency if ((serv_ptr= getservbyname("mysql", "tcp"))) mysql_port= (uint) ntohs((ushort) serv_ptr->s_port); +# endif #endif if ((env= getenv("MYSQL_TCP_PORT"))) mysql_port=(uint) atoi(env); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index e40d5c38233..afe969b6e8d 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2415,9 +2415,11 @@ static void set_ports() */ #if MYSQL_PORT_DEFAULT == 0 +# if !__has_feature(memory_sanitizer) // Work around MSAN deficiency struct servent *serv_ptr; if ((serv_ptr= getservbyname("mysql", "tcp"))) SYSVAR_AUTOSIZE(mysqld_port, ntohs((u_short) serv_ptr->s_port)); +# endif #endif if ((env = getenv("MYSQL_TCP_PORT"))) { diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 5fc980b42d3..35e578e1a5c 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -961,10 +961,12 @@ btr_cur_search_to_nth_level_func( ut_ad(!(index->type & DICT_FTS)); ut_ad(index->page != FIL_NULL); - UNIV_MEM_INVALID(&cursor->up_match, sizeof cursor->up_match); - UNIV_MEM_INVALID(&cursor->up_bytes, sizeof cursor->up_bytes); - UNIV_MEM_INVALID(&cursor->low_match, sizeof cursor->low_match); - UNIV_MEM_INVALID(&cursor->low_bytes, sizeof cursor->low_bytes); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(&cursor->up_match, sizeof cursor->up_match); + MEM_UNDEFINED(&cursor->up_bytes, sizeof cursor->up_bytes); + MEM_UNDEFINED(&cursor->low_match, sizeof cursor->low_match); + MEM_UNDEFINED(&cursor->low_bytes, sizeof cursor->low_bytes); +#endif /* HAVE_valgrind_or_MSAN */ #ifdef UNIV_DEBUG cursor->up_match = ULINT_UNDEFINED; cursor->low_match = ULINT_UNDEFINED; @@ -3075,12 +3077,12 @@ btr_cur_optimistic_insert( const page_size_t& page_size = block->page.size; -#ifdef UNIV_DEBUG_VALGRIND +#ifdef HAVE_valgrind_or_MSAN if (page_size.is_compressed()) { - UNIV_MEM_ASSERT_RW(page, page_size.logical()); - UNIV_MEM_ASSERT_RW(block->page.zip.data, page_size.physical()); + MEM_CHECK_DEFINED(page, page_size.logical()); + MEM_CHECK_DEFINED(block->page.zip.data, page_size.physical()); } -#endif /* UNIV_DEBUG_VALGRIND */ +#endif /* HAVE_valgrind_or_MSAN */ leaf = page_is_leaf(page); @@ -6892,9 +6894,7 @@ btr_store_big_rec_extern_fields( BTR_EXTERN_FIELD_REF_SIZE)); #endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ extern_len = big_rec_vec->fields[i].len; - UNIV_MEM_ASSERT_RW(big_rec_vec->fields[i].data, - extern_len); - + MEM_CHECK_DEFINED(big_rec_vec->fields[i].data, extern_len); ut_a(extern_len > 0); prev_page_no = FIL_NULL; @@ -7561,7 +7561,7 @@ btr_copy_blob_prefix( mtr_commit(&mtr); if (page_no == FIL_NULL || copy_len != part_len) { - UNIV_MEM_ASSERT_RW(buf, copied_len); + MEM_CHECK_DEFINED(buf, copied_len); return(copied_len); } @@ -7717,7 +7717,7 @@ end_of_blob: func_exit: inflateEnd(&d_stream); mem_heap_free(heap); - UNIV_MEM_ASSERT_RW(buf, d_stream.total_out); + MEM_CHECK_DEFINED(buf, d_stream.total_out); return(d_stream.total_out); } diff --git a/storage/innobase/buf/buf0buddy.cc b/storage/innobase/buf/buf0buddy.cc index 0863facad52..a36fc30db9b 100644 --- a/storage/innobase/buf/buf0buddy.cc +++ b/storage/innobase/buf/buf0buddy.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2018, 2019, MariaDB Corporation. +Copyright (c) 2018, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -85,7 +85,6 @@ enum buf_buddy_state_t { are in use */ }; -#ifdef UNIV_DEBUG_VALGRIND /**********************************************************************//** Invalidate memory area that we won't access while page is free */ UNIV_INLINE @@ -95,15 +94,11 @@ buf_buddy_mem_invalid( buf_buddy_free_t* buf, /*!< in: block to check */ ulint i) /*!< in: index of zip_free[] */ { - const size_t size = BUF_BUDDY_LOW << i; - ut_ad(i <= BUF_BUDDY_SIZES); + ut_ad(i <= BUF_BUDDY_SIZES); - UNIV_MEM_ASSERT_W(buf, size); - UNIV_MEM_INVALID(buf, size); + MEM_CHECK_ADDRESSABLE(buf, BUF_BUDDY_LOW << i); + MEM_UNDEFINED(buf, BUF_BUDDY_LOW << i); } -#else /* UNIV_DEBUG_VALGRIND */ -# define buf_buddy_mem_invalid(buf, i) ut_ad((i) <= BUF_BUDDY_SIZES) -#endif /* UNIV_DEBUG_VALGRIND */ /**********************************************************************//** Check if a buddy is stamped free. @@ -361,11 +356,10 @@ buf_buddy_alloc_zip( if (buf) { /* Trash the page other than the BUF_BUDDY_STAMP_NONFREE. */ - UNIV_MEM_TRASH((void*) buf, ~i, BUF_BUDDY_STAMP_OFFSET); - UNIV_MEM_TRASH(BUF_BUDDY_STAMP_OFFSET + 4 - + buf->stamp.bytes, ~i, - (BUF_BUDDY_LOW << i) - - (BUF_BUDDY_STAMP_OFFSET + 4)); + MEM_UNDEFINED(buf, BUF_BUDDY_STAMP_OFFSET); + MEM_UNDEFINED(BUF_BUDDY_STAMP_OFFSET + 4 + buf->stamp.bytes, + (BUF_BUDDY_LOW << i) + - (BUF_BUDDY_STAMP_OFFSET + 4)); ut_ad(mach_read_from_4(buf->stamp.bytes + BUF_BUDDY_STAMP_OFFSET) == BUF_BUDDY_STAMP_NONFREE); @@ -402,8 +396,10 @@ buf_buddy_block_free( ut_d(bpage->in_zip_hash = FALSE); HASH_DELETE(buf_page_t, hash, buf_pool->zip_hash, fold, bpage); - ut_d(memset(buf, 0, UNIV_PAGE_SIZE)); - UNIV_MEM_INVALID(buf, UNIV_PAGE_SIZE); + ut_d(memset(buf, 0, srv_page_size)); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(buf, srv_page_size); +#endif /* HAVE_valgrind_or_MSAN */ block = (buf_block_t*) bpage; buf_page_mutex_enter(block); @@ -559,17 +555,16 @@ buf_buddy_relocate( ut_ad(!ut_align_offset(src, size)); ut_ad(!ut_align_offset(dst, size)); ut_ad(i >= buf_buddy_get_slot(UNIV_ZIP_SIZE_MIN)); - UNIV_MEM_ASSERT_W(dst, size); + MEM_CHECK_ADDRESSABLE(dst, size); space = mach_read_from_4((const byte*) src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); offset = mach_read_from_4((const byte*) src + FIL_PAGE_OFFSET); - /* Suppress Valgrind warnings about conditional jump - on uninitialized value. */ - UNIV_MEM_VALID(&space, sizeof space); - UNIV_MEM_VALID(&offset, sizeof offset); + /* Suppress Valgrind or MSAN warnings. */ + MEM_MAKE_DEFINED(&space, sizeof space); + MEM_MAKE_DEFINED(&offset, sizeof offset); ut_ad(space != BUF_BUDDY_STAMP_FREE); @@ -631,7 +626,7 @@ buf_buddy_relocate( /* The block must have been allocated, but it may contain uninitialized data. */ - UNIV_MEM_ASSERT_W(src, size); + MEM_CHECK_ADDRESSABLE(src, size); BPageMutex* block_mutex = buf_page_get_mutex(bpage); @@ -686,7 +681,7 @@ buf_buddy_free_low( buf_pool->buddy_stat[i].used--; recombine: - UNIV_MEM_ALLOC(buf, BUF_BUDDY_LOW << i); + MEM_UNDEFINED(buf, BUF_BUDDY_LOW << i); if (i == BUF_BUDDY_SIZES) { buf_buddy_block_free(buf_pool, buf); diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 802e67de1b0..8bd0fa0a885 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1487,8 +1487,6 @@ buf_block_init( buf_block_t* block, /*!< in: pointer to control block */ byte* frame) /*!< in: pointer to buffer frame */ { - UNIV_MEM_DESC(frame, UNIV_PAGE_SIZE); - /* This function should only be executed at database startup or by buf_pool_resize(). Either way, adaptive hash index must not exist. */ assert_block_ahi_empty_on_init(block); @@ -1635,7 +1633,7 @@ buf_chunk_init( for (i = chunk->size; i--; ) { buf_block_init(buf_pool, block, frame); - UNIV_MEM_INVALID(block->frame, UNIV_PAGE_SIZE); + MEM_UNDEFINED(block->frame, srv_page_size); /* Add the block to the free list */ UT_LIST_ADD_LAST(buf_pool->free, &block->page); @@ -2180,8 +2178,6 @@ buf_page_realloc( if (block->page.zip.data != NULL) { ut_ad(block->in_unzip_LRU_list); ut_d(new_block->in_unzip_LRU_list = TRUE); - UNIV_MEM_DESC(&new_block->page.zip.data, - page_zip_get_size(&new_block->page.zip)); buf_block_t* prev_block = UT_LIST_GET_PREV(unzip_LRU, block); UT_LIST_REMOVE(buf_pool->unzip_LRU, block); @@ -2215,7 +2211,7 @@ buf_page_realloc( buf_block_modify_clock_inc(block); memset(block->frame + FIL_PAGE_OFFSET, 0xff, 4); memset(block->frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 0xff, 4); - UNIV_MEM_INVALID(block->frame, UNIV_PAGE_SIZE); + MEM_UNDEFINED(block->frame, srv_page_size); buf_block_set_state(block, BUF_BLOCK_REMOVE_HASH); block->page.id = page_id_t(ULINT32_UNDEFINED, ULINT32_UNDEFINED); @@ -4620,9 +4616,6 @@ evict_from_pool: block->lock_hash_val = lock_rec_hash(page_id.space(), page_id.page_no()); - UNIV_MEM_DESC(&block->page.zip.data, - page_zip_get_size(&block->page.zip)); - if (buf_page_get_state(&block->page) == BUF_BLOCK_ZIP_PAGE) { #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG UT_LIST_REMOVE(buf_pool->zip_clean, &block->page); @@ -4644,7 +4637,7 @@ evict_from_pool: buf_block_set_io_fix(block, BUF_IO_READ); rw_lock_x_lock_inline(&block->lock, 0, file, line); - UNIV_MEM_INVALID(bpage, sizeof *bpage); + MEM_UNDEFINED(bpage, sizeof *bpage); rw_lock_x_unlock(hash_lock); buf_pool->n_pend_unzip++; @@ -5268,15 +5261,6 @@ buf_page_init( /* Set the state of the block */ buf_block_set_file_page(block, page_id); -#ifdef UNIV_DEBUG_VALGRIND - if (is_system_tablespace(page_id.space())) { - /* Silence valid Valgrind warnings about uninitialized - data being written to data files. There are some unused - bytes on some pages that InnoDB does not initialize. */ - UNIV_MEM_VALID(block->frame, UNIV_PAGE_SIZE); - } -#endif /* UNIV_DEBUG_VALGRIND */ - buf_block_init_low(block); block->lock_hash_val = lock_rec_hash(page_id.space(), @@ -5503,7 +5487,6 @@ buf_page_init_for_read( bpage->size.copy_from(page_size); mutex_enter(&buf_pool->zip_mutex); - UNIV_MEM_DESC(bpage->zip.data, bpage->size.physical()); buf_page_init_low(bpage); diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 42128f873eb..3ff44129cec 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -1147,7 +1147,7 @@ try_again: void * frame = buf_page_get_frame(bpage); if (bpage->size.is_compressed()) { - UNIV_MEM_ASSERT_RW(bpage->zip.data, bpage->size.physical()); + MEM_CHECK_DEFINED(bpage->zip.data, bpage->size.physical()); /* Copy the compressed page and clear the rest. */ memcpy(p, frame, bpage->size.physical()); @@ -1156,10 +1156,7 @@ try_again: univ_page_size.physical() - bpage->size.physical()); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); - - UNIV_MEM_ASSERT_RW(frame, - bpage->size.logical()); - + MEM_CHECK_DEFINED(frame, bpage->size.logical()); memcpy(p, frame, bpage->size.logical()); } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 77bbc530637..9b395bd3231 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. Copyright (c) 2013, 2014, Fusion-io This program is free software; you can redistribute it and/or modify it under @@ -449,18 +449,9 @@ buf_flush_insert_into_flush_list( incr_flush_list_size_in_bytes(block, buf_pool); -#ifdef UNIV_DEBUG_VALGRIND - void* p; - - if (block->page.size.is_compressed()) { - p = block->page.zip.data; - } else { - p = block->frame; - } - - UNIV_MEM_ASSERT_RW(p, block->page.size.physical()); -#endif /* UNIV_DEBUG_VALGRIND */ - + MEM_CHECK_DEFINED(block->page.size.is_compressed() + ? block->page.zip.data : block->frame, + block->page.size.physical()); #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG ut_a(buf_flush_validate_skip(buf_pool)); #endif /* UNIV_DEBUG || UNIV_BUF_DEBUG */ @@ -510,17 +501,9 @@ buf_flush_insert_sorted_into_flush_list( ut_d(block->page.in_flush_list = TRUE); block->page.oldest_modification = lsn; -#ifdef UNIV_DEBUG_VALGRIND - void* p; - - if (block->page.size.is_compressed()) { - p = block->page.zip.data; - } else { - p = block->frame; - } - - UNIV_MEM_ASSERT_RW(p, block->page.size.physical()); -#endif /* UNIV_DEBUG_VALGRIND */ + MEM_CHECK_DEFINED(block->page.size.is_compressed() + ? block->page.zip.data : block->frame, + block->page.size.physical()); prev_b = NULL; diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 317a17b142f..45dd2f2312e 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -809,7 +809,7 @@ buf_LRU_get_free_only( assert_block_ahi_empty(block); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - UNIV_MEM_ALLOC(block->frame, UNIV_PAGE_SIZE); + MEM_UNDEFINED(block->frame, srv_page_size); ut_ad(buf_pool_from_block(block) == buf_pool); @@ -1504,8 +1504,6 @@ func_exit: ut_ad(b->size.is_compressed()); - UNIV_MEM_DESC(b->zip.data, b->size.physical()); - /* The fields in_page_hash and in_LRU_list of the to-be-freed block descriptor should have been cleared in @@ -1609,17 +1607,20 @@ func_exit: The page was declared uninitialized by buf_LRU_block_remove_hashed(). We need to flag the contents of the page valid (which it still is) in - order to avoid bogus Valgrind warnings.*/ + order to avoid bogus Valgrind or MSAN warnings.*/ + buf_block_t* block = reinterpret_cast(bpage); - UNIV_MEM_VALID(((buf_block_t*) bpage)->frame, - UNIV_PAGE_SIZE); - btr_search_drop_page_hash_index((buf_block_t*) bpage); - UNIV_MEM_INVALID(((buf_block_t*) bpage)->frame, - UNIV_PAGE_SIZE); +#ifdef HAVE_valgrind_or_MSAN + MEM_MAKE_DEFINED(block->frame, srv_page_size); +#endif /* HAVE_valgrind_or_MSAN */ + btr_search_drop_page_hash_index(block); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(block->frame, srv_page_size); +#endif /* HAVE_valgrind_or_MSAN */ buf_pool_mutex_enter(buf_pool); - if (b != NULL) { + if (b) { mutex_enter(block_mutex); buf_page_unset_sticky(b); @@ -1627,7 +1628,7 @@ func_exit: mutex_exit(block_mutex); } - buf_LRU_block_free_hashed_page((buf_block_t*) bpage); + buf_LRU_block_free_hashed_page(block); return(true); } @@ -1660,15 +1661,10 @@ buf_LRU_block_free_non_file_page( buf_block_set_state(block, BUF_BLOCK_NOT_USED); - UNIV_MEM_ALLOC(block->frame, UNIV_PAGE_SIZE); -#ifdef UNIV_DEBUG - /* Wipe contents of page to reveal possible stale pointers to it */ - memset(block->frame, '\0', UNIV_PAGE_SIZE); -#else + MEM_UNDEFINED(block->frame, srv_page_size); /* Wipe page_no and space_id */ memset(block->frame + FIL_PAGE_OFFSET, 0xfe, 4); memset(block->frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 0xfe, 4); -#endif /* UNIV_DEBUG */ data = block->page.zip.data; if (data != NULL) { @@ -1704,7 +1700,7 @@ buf_LRU_block_free_non_file_page( ut_d(block->page.in_free_list = TRUE); } - UNIV_MEM_FREE(block->frame, UNIV_PAGE_SIZE); + MEM_NOACCESS(block->frame, srv_page_size); } /******************************************************************//** @@ -1751,9 +1747,9 @@ buf_LRU_block_remove_hashed( switch (buf_page_get_state(bpage)) { case BUF_BLOCK_FILE_PAGE: - UNIV_MEM_ASSERT_W(bpage, sizeof(buf_block_t)); - UNIV_MEM_ASSERT_W(((buf_block_t*) bpage)->frame, - UNIV_PAGE_SIZE); + MEM_CHECK_ADDRESSABLE(bpage, sizeof(buf_block_t)); + MEM_CHECK_ADDRESSABLE(((buf_block_t*) bpage)->frame, + srv_page_size); buf_block_modify_clock_inc((buf_block_t*) bpage); if (bpage->zip.data) { const page_t* page = ((buf_block_t*) bpage)->frame; @@ -1809,8 +1805,8 @@ buf_LRU_block_remove_hashed( case BUF_BLOCK_ZIP_PAGE: ut_a(bpage->oldest_modification == 0); if (bpage->size.is_compressed()) { - UNIV_MEM_ASSERT_W(bpage->zip.data, - bpage->size.physical()); + MEM_CHECK_ADDRESSABLE(bpage->zip.data, + bpage->size.physical()); } break; case BUF_BLOCK_POOL_WATCH: @@ -1864,8 +1860,7 @@ buf_LRU_block_remove_hashed( + FIL_PAGE_OFFSET, 0xff, 4); memset(((buf_block_t*) bpage)->frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 0xff, 4); - UNIV_MEM_INVALID(((buf_block_t*) bpage)->frame, - UNIV_PAGE_SIZE); + MEM_UNDEFINED(((buf_block_t*) bpage)->frame, srv_page_size); buf_page_set_state(bpage, BUF_BLOCK_REMOVE_HASH); /* Question: If we release bpage and hash mutex here diff --git a/storage/innobase/data/data0data.cc b/storage/innobase/data/data0data.cc index 17126e38e42..b4d41f47060 100644 --- a/storage/innobase/data/data0data.cc +++ b/storage/innobase/data/data0data.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2019, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -212,19 +212,7 @@ dtuple_validate( len = dfield_get_len(field); if (!dfield_is_null(field)) { - - const byte* data; - - data = static_cast(dfield_get_data(field)); -#ifndef UNIV_DEBUG_VALGRIND - ulint j; - - for (j = 0; j < len; j++) { - data++; - } -#endif /* !UNIV_DEBUG_VALGRIND */ - - UNIV_MEM_ASSERT_RW(data, len); + MEM_CHECK_DEFINED(dfield_get_data(field), len); } } @@ -683,14 +671,6 @@ skip_field: memcpy(data, dfield_get_data(dfield), local_prefix_len); /* Clear the extern field reference (BLOB pointer). */ memset(data + local_prefix_len, 0, BTR_EXTERN_FIELD_REF_SIZE); -#if 0 - /* The following would fail the Valgrind checks in - page_cur_insert_rec_low() and page_cur_insert_rec_zip(). - The BLOB pointers in the record will be initialized after - the record and the BLOBs have been written. */ - UNIV_MEM_ALLOC(data + local_prefix_len, - BTR_EXTERN_FIELD_REF_SIZE); -#endif dfield_set_data(dfield, data, local_len); dfield_set_ext(dfield); diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 77c1e9627eb..563729cd560 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -409,7 +409,7 @@ dict_stats_table_clone_create( t = (dict_table_t*) mem_heap_alloc(heap, sizeof(*t)); - UNIV_MEM_ASSERT_RW_ABORT(&table->id, sizeof(table->id)); + MEM_CHECK_DEFINED(&table->id, sizeof(table->id)); t->id = table->id; t->heap = heap; @@ -442,7 +442,7 @@ dict_stats_table_clone_create( idx = (dict_index_t*) mem_heap_alloc(heap, sizeof(*idx)); - UNIV_MEM_ASSERT_RW_ABORT(&index->id, sizeof(index->id)); + MEM_CHECK_DEFINED(&index->id, sizeof(index->id)); idx->id = index->id; idx->name = mem_heap_strdup(heap, index->name); @@ -589,23 +589,23 @@ dict_stats_assert_initialized_index( /*================================*/ const dict_index_t* index) /*!< in: index */ { - UNIV_MEM_ASSERT_RW_ABORT( + MEM_CHECK_DEFINED( index->stat_n_diff_key_vals, index->n_uniq * sizeof(index->stat_n_diff_key_vals[0])); - UNIV_MEM_ASSERT_RW_ABORT( + MEM_CHECK_DEFINED( index->stat_n_sample_sizes, index->n_uniq * sizeof(index->stat_n_sample_sizes[0])); - UNIV_MEM_ASSERT_RW_ABORT( + MEM_CHECK_DEFINED( index->stat_n_non_null_key_vals, index->n_uniq * sizeof(index->stat_n_non_null_key_vals[0])); - UNIV_MEM_ASSERT_RW_ABORT( + MEM_CHECK_DEFINED( &index->stat_index_size, sizeof(index->stat_index_size)); - UNIV_MEM_ASSERT_RW_ABORT( + MEM_CHECK_DEFINED( &index->stat_n_leaf_pages, sizeof(index->stat_n_leaf_pages)); } @@ -620,32 +620,32 @@ dict_stats_assert_initialized( { ut_a(table->stat_initialized); - UNIV_MEM_ASSERT_RW_ABORT(&table->stats_last_recalc, - sizeof(table->stats_last_recalc)); + MEM_CHECK_DEFINED(&table->stats_last_recalc, + sizeof table->stats_last_recalc); - UNIV_MEM_ASSERT_RW_ABORT(&table->stat_persistent, - sizeof(table->stat_persistent)); + MEM_CHECK_DEFINED(&table->stat_persistent, + sizeof table->stat_persistent); - UNIV_MEM_ASSERT_RW_ABORT(&table->stats_auto_recalc, - sizeof(table->stats_auto_recalc)); + MEM_CHECK_DEFINED(&table->stats_auto_recalc, + sizeof table->stats_auto_recalc); - UNIV_MEM_ASSERT_RW_ABORT(&table->stats_sample_pages, - sizeof(table->stats_sample_pages)); + MEM_CHECK_DEFINED(&table->stats_sample_pages, + sizeof table->stats_sample_pages); - UNIV_MEM_ASSERT_RW_ABORT(&table->stat_n_rows, - sizeof(table->stat_n_rows)); + MEM_CHECK_DEFINED(&table->stat_n_rows, + sizeof table->stat_n_rows); - UNIV_MEM_ASSERT_RW_ABORT(&table->stat_clustered_index_size, - sizeof(table->stat_clustered_index_size)); + MEM_CHECK_DEFINED(&table->stat_clustered_index_size, + sizeof table->stat_clustered_index_size); - UNIV_MEM_ASSERT_RW_ABORT(&table->stat_sum_of_other_index_sizes, - sizeof(table->stat_sum_of_other_index_sizes)); + MEM_CHECK_DEFINED(&table->stat_sum_of_other_index_sizes, + sizeof table->stat_sum_of_other_index_sizes); - UNIV_MEM_ASSERT_RW_ABORT(&table->stat_modified_counter, - sizeof(table->stat_modified_counter)); + MEM_CHECK_DEFINED(&table->stat_modified_counter, + sizeof table->stat_modified_counter); - UNIV_MEM_ASSERT_RW_ABORT(&table->stats_bg_flag, - sizeof(table->stats_bg_flag)); + MEM_CHECK_DEFINED(&table->stats_bg_flag, + sizeof table->stats_bg_flag); for (dict_index_t* index = dict_table_get_first_index(table); index != NULL; @@ -2312,20 +2312,19 @@ dict_stats_save_index_stat( pars_info_add_str_literal(pinfo, "database_name", db_utf8); pars_info_add_str_literal(pinfo, "table_name", table_utf8); pars_info_add_str_literal(pinfo, "index_name", index->name); - UNIV_MEM_ASSERT_RW_ABORT(&last_update, 4); + MEM_CHECK_DEFINED(&last_update, 4); pars_info_add_int4_literal(pinfo, "last_update", (lint)last_update); - UNIV_MEM_ASSERT_RW_ABORT(stat_name, strlen(stat_name)); + MEM_CHECK_DEFINED(stat_name, strlen(stat_name)); pars_info_add_str_literal(pinfo, "stat_name", stat_name); - UNIV_MEM_ASSERT_RW_ABORT(&stat_value, 8); + MEM_CHECK_DEFINED(&stat_value, 8); pars_info_add_ull_literal(pinfo, "stat_value", stat_value); if (sample_size != NULL) { - UNIV_MEM_ASSERT_RW_ABORT(sample_size, 8); + MEM_CHECK_DEFINED(sample_size, 8); pars_info_add_ull_literal(pinfo, "sample_size", *sample_size); } else { pars_info_add_literal(pinfo, "sample_size", NULL, UNIV_SQL_NULL, DATA_FIXBINARY, 0); } - UNIV_MEM_ASSERT_RW_ABORT(stat_description, strlen(stat_description)); pars_info_add_str_literal(pinfo, "stat_description", stat_description); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index bcb02704158..bb648d99777 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -7499,7 +7499,9 @@ build_template_field( ut_ad(clust_index->table == index->table); templ = prebuilt->mysql_template + prebuilt->n_template++; - UNIV_MEM_INVALID(templ, sizeof *templ); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(templ, sizeof *templ); +#endif /* HAVE_valgrind_or_MSAN */ templ->is_virtual = !field->stored_in_db(); if (!templ->is_virtual) { @@ -8619,7 +8621,9 @@ calc_row_difference( /* The field has changed */ ufield = uvect->fields + n_changed; - UNIV_MEM_INVALID(ufield, sizeof *ufield); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(ufield, sizeof *ufield); +#endif /* HAVE_valgrind_or_MSAN */ /* Let us use a dummy dfield to make the conversion from the MySQL column format to the InnoDB format */ diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index a9b8ef94bab..fd02279e2b0 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1737,7 +1737,7 @@ struct buf_block_t{ # define assert_block_ahi_empty(block) \ ut_a(my_atomic_addlint(&(block)->n_pointers, 0) == 0) # define assert_block_ahi_empty_on_init(block) do { \ - UNIV_MEM_VALID(&(block)->n_pointers, sizeof (block)->n_pointers); \ + MEM_MAKE_DEFINED(&(block)->n_pointers, sizeof (block)->n_pointers); \ assert_block_ahi_empty(block); \ } while (0) # define assert_block_ahi_valid(block) \ diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index e1c8986c2ed..3df17e8a978 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -824,7 +824,7 @@ buf_page_alloc_descriptor(void) bpage = (buf_page_t*) ut_zalloc_nokey(sizeof *bpage); ut_ad(bpage); - UNIV_MEM_ALLOC(bpage, sizeof *bpage); + MEM_UNDEFINED(bpage, sizeof *bpage); return(bpage); } diff --git a/storage/innobase/include/data0data.ic b/storage/innobase/include/data0data.ic index 295c786a583..9b7a3132873 100644 --- a/storage/innobase/include/data0data.ic +++ b/storage/innobase/include/data0data.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2019, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -50,10 +50,6 @@ dfield_set_len( dfield_t* field, /*!< in: field */ ulint len) /*!< in: length or UNIV_SQL_NULL */ { -#ifdef UNIV_VALGRIND_DEBUG - if (len != UNIV_SQL_NULL) UNIV_MEM_ASSERT_RW(field->data, len); -#endif /* UNIV_VALGRIND_DEBUG */ - field->ext = 0; field->len = static_cast(len); } @@ -96,9 +92,6 @@ dfield_set_data( const void* data, /*!< in: data */ ulint len) /*!< in: length or UNIV_SQL_NULL */ { -#ifdef UNIV_VALGRIND_DEBUG - if (len != UNIV_SQL_NULL) UNIV_MEM_ASSERT_RW(data, len); -#endif /* UNIV_VALGRIND_DEBUG */ field->data = (void*) data; field->ext = 0; field->len = static_cast(len); @@ -113,9 +106,7 @@ dfield_write_mbr( dfield_t* field, /*!< in: field */ const double* mbr) /*!< in: data */ { -#ifdef UNIV_VALGRIND_DEBUG - if (len != UNIV_SQL_NULL) UNIV_MEM_ASSERT_RW(data, len); -#endif /* UNIV_VALGRIND_DEBUG */ + MEM_CHECK_DEFINED(mbr, sizeof *mbr); field->ext = 0; for (unsigned i = 0; i < SPDIMS * 2; i++) { @@ -177,7 +168,7 @@ dfield_dup( mem_heap_t* heap) /*!< in: memory heap where allocated */ { if (!dfield_is_null(field)) { - UNIV_MEM_ASSERT_RW(field->data, field->len); + MEM_CHECK_DEFINED(field->data, field->len); field->data = mem_heap_dup(heap, field->data, field->len); } } @@ -334,8 +325,9 @@ dtuple_create_from_mem( } } #endif - UNIV_MEM_ASSERT_W(tuple->fields, n_t_fields * sizeof *tuple->fields); - UNIV_MEM_INVALID(tuple->fields, n_t_fields * sizeof *tuple->fields); + MEM_CHECK_ADDRESSABLE(tuple->fields, n_t_fields + * sizeof *tuple->fields); + MEM_UNDEFINED(tuple->fields, n_t_fields * sizeof *tuple->fields); return(tuple); } diff --git a/storage/innobase/include/dict0stats.ic b/storage/innobase/include/dict0stats.ic index 31065d15c45..98024935e16 100644 --- a/storage/innobase/include/dict0stats.ic +++ b/storage/innobase/include/dict0stats.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -187,41 +187,40 @@ dict_stats_deinit( table->stat_initialized = FALSE; -#ifdef UNIV_DEBUG_VALGRIND - UNIV_MEM_INVALID(&table->stat_n_rows, - sizeof(table->stat_n_rows)); - UNIV_MEM_INVALID(&table->stat_clustered_index_size, - sizeof(table->stat_clustered_index_size)); - UNIV_MEM_INVALID(&table->stat_sum_of_other_index_sizes, - sizeof(table->stat_sum_of_other_index_sizes)); - UNIV_MEM_INVALID(&table->stat_modified_counter, - sizeof(table->stat_modified_counter)); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(&table->stat_n_rows, sizeof table->stat_n_rows); + MEM_UNDEFINED(&table->stat_clustered_index_size, + sizeof table->stat_clustered_index_size); + MEM_UNDEFINED(&table->stat_sum_of_other_index_sizes, + sizeof table->stat_sum_of_other_index_sizes); + MEM_UNDEFINED(&table->stat_modified_counter, + sizeof table->stat_modified_counter); dict_index_t* index; for (index = dict_table_get_first_index(table); index != NULL; index = dict_table_get_next_index(index)) { - - ulint n_uniq = dict_index_get_n_unique(index); - - UNIV_MEM_INVALID( + MEM_UNDEFINED( index->stat_n_diff_key_vals, - n_uniq * sizeof(index->stat_n_diff_key_vals[0])); - UNIV_MEM_INVALID( + index->n_uniq + * sizeof(index->stat_n_diff_key_vals[0])); + MEM_UNDEFINED( index->stat_n_sample_sizes, - n_uniq * sizeof(index->stat_n_sample_sizes[0])); - UNIV_MEM_INVALID( + index->n_uniq + * sizeof(index->stat_n_sample_sizes[0])); + MEM_UNDEFINED( index->stat_n_non_null_key_vals, - n_uniq * sizeof(index->stat_n_non_null_key_vals[0])); - UNIV_MEM_INVALID( + index->n_uniq + * sizeof(index->stat_n_non_null_key_vals[0])); + MEM_UNDEFINED( &index->stat_index_size, sizeof(index->stat_index_size)); - UNIV_MEM_INVALID( + MEM_UNDEFINED( &index->stat_n_leaf_pages, sizeof(index->stat_n_leaf_pages)); } -#endif /* UNIV_DEBUG_VALGRIND */ +#endif /* HAVE_valgrind_or_MSAN */ dict_table_stats_unlock(table, RW_X_LATCH); } diff --git a/storage/innobase/include/mem0mem.ic b/storage/innobase/include/mem0mem.ic index 9c7ae8c28f8..bd0db9bf503 100644 --- a/storage/innobase/include/mem0mem.ic +++ b/storage/innobase/include/mem0mem.ic @@ -203,7 +203,7 @@ mem_heap_alloc( mem_block_set_free(block, free + MEM_SPACE_NEEDED(n)); buf = buf + REDZONE_SIZE; - UNIV_MEM_ALLOC(buf, n - REDZONE_SIZE); + MEM_UNDEFINED(buf, n - REDZONE_SIZE); return(buf); } @@ -267,7 +267,7 @@ mem_heap_free_heap_top( mem_block_set_free(block, old_top - (byte*) block); ut_ad(mem_block_get_start(block) <= mem_block_get_free(block)); - UNIV_MEM_FREE(old_top, (byte*) block + block->len - old_top); + MEM_NOACCESS(old_top, (byte*) block + block->len - old_top); /* If free == start, we may free the block if it is not the first one */ @@ -341,7 +341,7 @@ mem_heap_free_top( == mem_block_get_start(block))) { mem_heap_block_free(heap, block); } else { - UNIV_MEM_FREE((byte*) block + mem_block_get_free(block), n); + MEM_NOACCESS((byte*) block + mem_block_get_free(block), n); } } diff --git a/storage/innobase/include/page0zip.ic b/storage/innobase/include/page0zip.ic index 5345aa19dd5..5a3b500e2c8 100644 --- a/storage/innobase/include/page0zip.ic +++ b/storage/innobase/include/page0zip.ic @@ -231,7 +231,7 @@ page_zip_get_trailer_len( ulint uncompressed_size; ut_ad(page_zip_simple_validate(page_zip)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); if (!page_is_leaf(page_zip->data)) { uncompressed_size = PAGE_ZIP_DIR_SLOT_SIZE @@ -356,7 +356,7 @@ page_zip_write_header( ulint pos; ut_ad(page_zip_simple_validate(page_zip)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); pos = page_offset(str); diff --git a/storage/innobase/include/rem0rec.ic b/storage/innobase/include/rem0rec.ic index 2602b281e7d..f65bca8181d 100644 --- a/storage/innobase/include/rem0rec.ic +++ b/storage/innobase/include/rem0rec.ic @@ -928,7 +928,7 @@ rec_offs_get_n_alloc( ut_ad(offsets); n_alloc = offsets[0]; ut_ad(n_alloc > REC_OFFS_HEADER_SIZE); - UNIV_MEM_ASSERT_W(offsets, n_alloc * sizeof *offsets); + MEM_CHECK_ADDRESSABLE(offsets, n_alloc * sizeof *offsets); return(n_alloc); } @@ -944,7 +944,7 @@ rec_offs_set_n_alloc( ulint n_alloc) /*!< in: number of elements */ { ut_ad(n_alloc > REC_OFFS_HEADER_SIZE); - UNIV_MEM_ALLOC(offsets, n_alloc * sizeof *offsets); + MEM_UNDEFINED(offsets, n_alloc * sizeof *offsets); offsets[0] = static_cast(n_alloc); } diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 343cb0e741a..ccc70206ede 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -2,7 +2,7 @@ Copyright (c) 2010, 2015, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the @@ -652,14 +652,14 @@ Use MONITOR_DEC if appropriate mutex protection exists. } \ } -#ifdef UNIV_DEBUG_VALGRIND +#ifdef HAVE_valgrind_or_MSAN # define MONITOR_CHECK_DEFINED(value) do { \ mon_type_t m = value; \ - UNIV_MEM_ASSERT_RW(&m, sizeof m); \ + MEM_CHECK_DEFINED(&m, sizeof m); \ } while (0) -#else /* UNIV_DEBUG_VALGRIND */ +#else /* HAVE_valgrind_or_MSAN */ # define MONITOR_CHECK_DEFINED(value) (void) 0 -#endif /* UNIV_DEBUG_VALGRIND */ +#endif /* HAVE_valgrind_or_MSAN */ #define MONITOR_INC_VALUE(monitor, value) \ MONITOR_CHECK_DEFINED(value); \ diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 1c0166f4c3f..dbb1048a2e5 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -178,10 +178,6 @@ command. */ #define UNIV_ENABLE_UNIT_TEST_ROW_RAW_FORMAT_INT */ -#if defined HAVE_valgrind && defined HAVE_VALGRIND_MEMCHECK_H -# define UNIV_DEBUG_VALGRIND -#endif - #ifdef DBUG_OFF # undef UNIV_DEBUG #elif !defined UNIV_DEBUG @@ -189,8 +185,6 @@ command. */ #endif #if 0 -#define UNIV_DEBUG_VALGRIND /* Enable extra - Valgrind instrumentation */ #define UNIV_DEBUG_PRINT /* Enable the compilation of some debug print functions */ #define UNIV_AHI_DEBUG /* Enable adaptive hash index @@ -616,57 +610,6 @@ typedef void* os_thread_ret_t; #include "ut0ut.h" #include "sync0types.h" -#include -/* define UNIV macros in terms of my_valgrind.h */ -#define UNIV_MEM_INVALID(addr, size) MEM_UNDEFINED(addr, size) -#define UNIV_MEM_FREE(addr, size) MEM_NOACCESS(addr, size) -#define UNIV_MEM_ALLOC(addr, size) UNIV_MEM_INVALID(addr, size) -#ifdef UNIV_DEBUG_VALGRIND -# include -# define UNIV_MEM_VALID(addr, size) VALGRIND_MAKE_MEM_DEFINED(addr, size) -# define UNIV_MEM_DESC(addr, size) VALGRIND_CREATE_BLOCK(addr, size, #addr) -# define UNIV_MEM_UNDESC(b) VALGRIND_DISCARD(b) -# define UNIV_MEM_ASSERT_RW_LOW(addr, size, should_abort) do { \ - const void* _p = (const void*) (ulint) \ - VALGRIND_CHECK_MEM_IS_DEFINED(addr, size); \ - if (UNIV_LIKELY_NULL(_p)) { \ - fprintf(stderr, "%s:%d: %p[%u] undefined at %ld\n", \ - __FILE__, __LINE__, \ - (const void*) (addr), (unsigned) (size), (long) \ - (((const char*) _p) - ((const char*) (addr)))); \ - if (should_abort) { \ - ut_error; \ - } \ - } \ -} while (0) -# define UNIV_MEM_ASSERT_RW(addr, size) \ - UNIV_MEM_ASSERT_RW_LOW(addr, size, false) -# define UNIV_MEM_ASSERT_RW_ABORT(addr, size) \ - UNIV_MEM_ASSERT_RW_LOW(addr, size, true) -# define UNIV_MEM_ASSERT_W(addr, size) do { \ - const void* _p = (const void*) (ulint) \ - VALGRIND_CHECK_MEM_IS_ADDRESSABLE(addr, size); \ - if (UNIV_LIKELY_NULL(_p)) \ - fprintf(stderr, "%s:%d: %p[%u] unwritable at %ld\n", \ - __FILE__, __LINE__, \ - (const void*) (addr), (unsigned) (size), (long) \ - (((const char*) _p) - ((const char*) (addr)))); \ - } while (0) -# define UNIV_MEM_TRASH(addr, c, size) do { \ - ut_d(memset(addr, c, size)); \ - UNIV_MEM_INVALID(addr, size); \ - } while (0) -#else -# define UNIV_MEM_VALID(addr, size) do {} while(0) -# define UNIV_MEM_DESC(addr, size) do {} while(0) -# define UNIV_MEM_UNDESC(b) do {} while(0) -# define UNIV_MEM_ASSERT_RW_LOW(addr, size, should_abort) do {} while(0) -# define UNIV_MEM_ASSERT_RW(addr, size) do {} while(0) -# define UNIV_MEM_ASSERT_RW_ABORT(addr, size) do {} while(0) -# define UNIV_MEM_ASSERT_W(addr, size) do {} while(0) -# define UNIV_MEM_TRASH(addr, c, size) do {} while(0) -#endif - extern ulong srv_page_size_shift; extern ulong srv_page_size; diff --git a/storage/innobase/include/ut0pool.h b/storage/innobase/include/ut0pool.h index a7efc4e56a2..957157fa815 100644 --- a/storage/innobase/include/ut0pool.h +++ b/storage/innobase/include/ut0pool.h @@ -94,7 +94,7 @@ struct Pool { #ifdef HAVE_valgrind /* Declare the contents as initialized for Valgrind; we checked this in mem_free(). */ - UNIV_MEM_VALID(&elem->m_type, sizeof elem->m_type); + MEM_MAKE_DEFINED(&elem->m_type, sizeof elem->m_type); #endif Factory::destroy(&elem->m_type); } @@ -137,13 +137,12 @@ struct Pool { MEM_UNDEFINED(&elem->m_type, sizeof elem->m_type); # endif # ifdef HAVE_valgrind - /* Declare the memory initialized for Valgrind. The trx_t that are released to the pool are actually initialized; we checked that by - UNIV_MEM_ASSERT_RW() in mem_free() below. */ - UNIV_MEM_VALID(&elem->m_type, sizeof elem->m_type); + MEM_CHECK_DEFINED() in mem_free() below. */ # endif + MEM_MAKE_DEFINED(&elem->m_type, sizeof elem->m_type); } #endif @@ -159,7 +158,7 @@ struct Pool { byte* p = reinterpret_cast(ptr + 1); elem = reinterpret_cast(p - sizeof(*elem)); - UNIV_MEM_ASSERT_RW(&elem->m_type, sizeof elem->m_type); + MEM_CHECK_DEFINED(&elem->m_type, sizeof elem->m_type); elem->m_pool->m_lock_strategy.enter(); diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 2d0f3c83e3b..16980582c14 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2581,10 +2581,10 @@ recv_parse_log_rec( *body = NULL; - UNIV_MEM_INVALID(type, sizeof *type); - UNIV_MEM_INVALID(space, sizeof *space); - UNIV_MEM_INVALID(page_no, sizeof *page_no); - UNIV_MEM_INVALID(body, sizeof *body); + MEM_UNDEFINED(type, sizeof *type); + MEM_UNDEFINED(space, sizeof *space); + MEM_UNDEFINED(page_no, sizeof *page_no); + MEM_UNDEFINED(body, sizeof *body); if (ptr == end_ptr) { diff --git a/storage/innobase/mem/mem0mem.cc b/storage/innobase/mem/mem0mem.cc index 0f94ac0a491..783451abbf2 100644 --- a/storage/innobase/mem/mem0mem.cc +++ b/storage/innobase/mem/mem0mem.cc @@ -334,8 +334,7 @@ mem_heap_create_block_func( /* Not the first allocation for the heap. This block's total_length field should be set to undefined. */ ut_d(block->total_size = ULINT_UNDEFINED); - UNIV_MEM_INVALID(&block->total_size, - sizeof block->total_size); + MEM_UNDEFINED(&block->total_size, sizeof block->total_size); heap->total_size += len; } @@ -343,7 +342,7 @@ mem_heap_create_block_func( /* Poison all available memory. Individual chunks will be unpoisoned on every mem_heap_alloc() call. */ compile_time_assert(MEM_BLOCK_HEADER_SIZE >= sizeof *block); - UNIV_MEM_FREE(block + 1, len - sizeof *block); + MEM_NOACCESS(block + 1, len - sizeof *block); ut_ad((ulint)MEM_BLOCK_HEADER_SIZE < len); diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index b75a9c4cf02..9fdb10e230c 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -378,7 +378,7 @@ mtr_write_log( /** Start a mini-transaction. */ void mtr_t::start() { - UNIV_MEM_INVALID(this, sizeof *this); + MEM_UNDEFINED(this, sizeof *this); new(&m_memo) mtr_buf_t(); new(&m_log) mtr_buf_t(); diff --git a/storage/innobase/os/os0proc.cc b/storage/innobase/os/os0proc.cc index 60057880c18..508a13de2ca 100644 --- a/storage/innobase/os/os0proc.cc +++ b/storage/innobase/os/os0proc.cc @@ -101,7 +101,7 @@ os_mem_alloc_large( my_atomic_addlint( &os_total_large_mem_allocated, size); - UNIV_MEM_ALLOC(ptr, size); + MEM_UNDEFINED(ptr, size); return(ptr); } @@ -125,7 +125,7 @@ skip: } else { my_atomic_addlint( &os_total_large_mem_allocated, size); - UNIV_MEM_ALLOC(ptr, size); + MEM_UNDEFINED(ptr, size); } #else size = getpagesize(); @@ -141,7 +141,7 @@ skip: } else { my_atomic_addlint( &os_total_large_mem_allocated, size); - UNIV_MEM_ALLOC(ptr, size); + MEM_UNDEFINED(ptr, size); } #endif return(ptr); @@ -157,11 +157,13 @@ os_mem_free_large( { ut_a(os_total_large_mem_allocated >= size); +#ifdef __SANITIZE_ADDRESS__ // We could have manually poisoned that memory for ASAN. // And we must unpoison it by ourself as specified in documentation // for __asan_poison_memory_region() in sanitizer/asan_interface.h // munmap() doesn't do it for us automatically. - UNIV_MEM_ALLOC(ptr, size); + MEM_UNDEFINED(ptr, size); +#endif /* __SANITIZE_ADDRESS__ */ #ifdef HAVE_LINUX_LARGE_PAGES if (my_use_large_pages && opt_large_page_size && !shmdt(ptr)) { diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index db71c6cc63f..c8bf53fd9fc 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2018, 2019, MariaDB Corporation. +Copyright (c) 2018, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1217,7 +1217,7 @@ page_cur_insert_rec_low( /* 1. Get the size of the physical record in the page */ rec_size = rec_offs_size(offsets); -#ifdef UNIV_DEBUG_VALGRIND +#ifdef HAVE_valgrind_or_MSAN { const void* rec_start = rec - rec_offs_extra_size(offsets); @@ -1228,11 +1228,11 @@ page_cur_insert_rec_low( : REC_N_OLD_EXTRA_BYTES); /* All data bytes of the record must be valid. */ - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); /* The variable-length header must be valid. */ - UNIV_MEM_ASSERT_RW(rec_start, extra_size); + MEM_CHECK_DEFINED(rec_start, extra_size); } -#endif /* UNIV_DEBUG_VALGRIND */ +#endif /* HAVE_valgrind_or_MSAN */ /* 2. Try to find suitable space from page memory management */ @@ -1340,8 +1340,8 @@ use_heap: rec_set_heap_no_old(insert_rec, heap_no); } - UNIV_MEM_ASSERT_RW(rec_get_start(insert_rec, offsets), - rec_offs_size(offsets)); + MEM_CHECK_DEFINED(rec_get_start(insert_rec, offsets), + rec_offs_size(offsets)); /* 6. Update the last insertion info in page header */ last_insert = page_header_get_ptr(page, PAGE_LAST_INSERT); @@ -1469,7 +1469,7 @@ page_cur_insert_rec_zip( /* 1. Get the size of the physical record in the page */ rec_size = rec_offs_size(offsets); -#ifdef UNIV_DEBUG_VALGRIND +#ifdef HAVE_valgrind_or_MSAN { const void* rec_start = rec - rec_offs_extra_size(offsets); @@ -1480,11 +1480,11 @@ page_cur_insert_rec_zip( : REC_N_OLD_EXTRA_BYTES); /* All data bytes of the record must be valid. */ - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); /* The variable-length header must be valid. */ - UNIV_MEM_ASSERT_RW(rec_start, extra_size); + MEM_CHECK_DEFINED(rec_start, extra_size); } -#endif /* UNIV_DEBUG_VALGRIND */ +#endif /* HAVE_valgrind_or_MSAN */ const bool reorg_before_insert = page_has_garbage(page) && rec_size > page_get_max_insert_size(page, 1) @@ -1809,8 +1809,8 @@ use_heap: rec_set_n_owned_new(insert_rec, NULL, 0); rec_set_heap_no_new(insert_rec, heap_no); - UNIV_MEM_ASSERT_RW(rec_get_start(insert_rec, offsets), - rec_offs_size(offsets)); + MEM_CHECK_DEFINED(rec_get_start(insert_rec, offsets), + rec_offs_size(offsets)); page_zip_dir_insert(page_zip, cursor->rec, free_rec, insert_rec); diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index b507945f076..c722cdd619a 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -888,9 +888,9 @@ page_zip_compress_node_ptrs( /* Only leaf nodes may contain externally stored columns. */ ut_ad(!rec_offs_any_extern(offsets)); - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); /* Compress the extra bytes. */ c_stream->avail_in = static_cast( @@ -953,8 +953,8 @@ page_zip_compress_sec( - c_stream->next_in); if (UNIV_LIKELY(c_stream->avail_in != 0)) { - UNIV_MEM_ASSERT_RW(c_stream->next_in, - c_stream->avail_in); + MEM_CHECK_DEFINED(c_stream->next_in, + c_stream->avail_in); err = deflate(c_stream, Z_NO_FLUSH); if (UNIV_UNLIKELY(err != Z_OK)) { break; @@ -996,9 +996,9 @@ page_zip_compress_clust_ext( int err; ulint i; - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); for (i = 0; i < rec_offs_n_fields(offsets); i++) { ulint len; @@ -1136,9 +1136,9 @@ page_zip_compress_clust( ULINT_UNDEFINED, &heap); ut_ad(rec_offs_n_fields(offsets) == dict_index_get_n_fields(index)); - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); /* Compress the extra bytes. */ c_stream->avail_in = static_cast( @@ -1185,9 +1185,9 @@ page_zip_compress_clust( == rec_get_nth_field(rec, offsets, trx_id_col + 1, &len)); ut_ad(len == DATA_ROLL_PTR_LEN); - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); /* Compress any preceding bytes. */ c_stream->avail_in = static_cast( @@ -1293,7 +1293,7 @@ page_zip_compress( && dict_table_is_comp(index->table) && !dict_index_is_ibuf(index))); - UNIV_MEM_ASSERT_RW(page, UNIV_PAGE_SIZE); + MEM_CHECK_DEFINED(page, srv_page_size); /* Check the data that will be omitted. */ ut_a(!memcmp(page + (PAGE_NEW_INFIMUM - REC_N_NEW_EXTRA_BYTES), @@ -1490,7 +1490,7 @@ page_zip_compress( trx_id_col = ULINT_UNDEFINED; } - UNIV_MEM_ASSERT_RW(c_stream.next_in, c_stream.avail_in); + MEM_CHECK_DEFINED(c_stream.next_in, c_stream.avail_in); err = deflate(&c_stream, Z_FULL_FLUSH); if (err != Z_OK) { goto zlib_error; @@ -1544,7 +1544,7 @@ page_zip_compress( - (c_stream.next_in - page)); ut_a(c_stream.avail_in <= UNIV_PAGE_SIZE - PAGE_ZIP_START - PAGE_DIR); - UNIV_MEM_ASSERT_RW(c_stream.next_in, c_stream.avail_in); + MEM_CHECK_DEFINED(c_stream.next_in, c_stream.avail_in); err = deflate(&c_stream, Z_FINISH); if (UNIV_UNLIKELY(err != Z_STREAM_END)) { @@ -1579,9 +1579,11 @@ err_exit: ut_ad(buf + c_stream.total_out == c_stream.next_out); ut_ad((ulint) (storage - c_stream.next_out) >= c_stream.avail_out); +#ifdef HAVE_valgrind /* Valgrind believes that zlib does not initialize some bits in the last 7 or 8 bytes of the stream. Make Valgrind happy. */ - UNIV_MEM_VALID(buf, c_stream.total_out); + MEM_MAKE_DEFINED(buf, c_stream.total_out); +#endif /* HAVE_valgrind */ /* Zero out the area reserved for the modification log. Space for the end marker of the modification log is not @@ -1613,7 +1615,7 @@ err_exit: page_zip_compress_write_log(page_zip, page, index, mtr); } - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); #ifdef PAGE_ZIP_COMPRESS_DBG if (logfile) { @@ -3045,8 +3047,8 @@ page_zip_decompress_low( rec_offs* offsets; ut_ad(page_zip_simple_validate(page_zip)); - UNIV_MEM_ASSERT_W(page, UNIV_PAGE_SIZE); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_ADDRESSABLE(page, srv_page_size); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); /* The dense directory excludes the infimum and supremum records. */ n_dense = page_dir_get_n_heap(page_zip->data) - PAGE_HEAP_NO_USER_LOW; @@ -3090,9 +3092,9 @@ page_zip_decompress_low( #ifdef UNIV_ZIP_DEBUG /* Clear the uncompressed page, except the header. */ - memset(PAGE_DATA + page, 0x55, UNIV_PAGE_SIZE - PAGE_DATA); + memset(PAGE_DATA + page, 0x55, srv_page_size - PAGE_DATA); #endif /* UNIV_ZIP_DEBUG */ - UNIV_MEM_INVALID(PAGE_DATA + page, UNIV_PAGE_SIZE - PAGE_DATA); + MEM_UNDEFINED(PAGE_DATA + page, srv_page_size - PAGE_DATA); /* Copy the page directory. */ if (UNIV_UNLIKELY(!page_zip_dir_decode(page_zip, page, recs, @@ -3222,7 +3224,7 @@ err_exit: } ut_a(page_is_comp(page)); - UNIV_MEM_ASSERT_RW(page, UNIV_PAGE_SIZE); + MEM_CHECK_DEFINED(page, srv_page_size); page_zip_fields_free(index); mem_heap_free(heap); @@ -3365,8 +3367,8 @@ page_zip_validate_low( temp_page_buf = static_cast(ut_malloc_nokey(2 * UNIV_PAGE_SIZE)); temp_page = static_cast(ut_align(temp_page_buf, UNIV_PAGE_SIZE)); - UNIV_MEM_ASSERT_RW(page, UNIV_PAGE_SIZE); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page, srv_page_size); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); temp_page_zip = *page_zip; valid = page_zip_decompress_low(&temp_page_zip, temp_page, TRUE); @@ -3587,9 +3589,9 @@ page_zip_write_rec_ext( ulint n_ext = rec_offs_n_extern(offsets); ut_ad(rec_offs_validate(rec, index, offsets)); - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); externs -= (DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN) * (page_dir_get_n_heap(page) - PAGE_HEAP_NO_USER_LOW); @@ -3714,10 +3716,10 @@ page_zip_write_rec( ut_ad(page_zip_header_cmp(page_zip, page)); ut_ad(page_simple_validate_new((page_t*) page)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); slot = page_zip_dir_find(page_zip, page_offset(rec)); ut_a(slot); @@ -3972,10 +3974,10 @@ page_zip_write_blob_ptr( ut_ad(page_is_leaf(page)); ut_ad(dict_index_is_clust(index)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); blob_no = page_zip_get_n_prev_extern(page_zip, rec, index) + rec_get_n_extern_new(rec, index, n); @@ -4118,8 +4120,8 @@ page_zip_write_node_ptr( ut_ad(!page_is_leaf(page)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); - UNIV_MEM_ASSERT_RW(rec, size); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(rec, size); storage = page_zip_dir_start(page_zip) - (rec_get_heap_no_new(rec) - 1) * REC_NODE_PTR_SIZE; @@ -4184,7 +4186,7 @@ page_zip_write_trx_id_and_roll_ptr( ut_ad(page_is_leaf(page)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); storage = page_zip_dir_start(page_zip) - (rec_get_heap_no_new(rec) - 1) @@ -4211,10 +4213,10 @@ page_zip_write_trx_id_and_roll_ptr( mach_write_to_7(field + DATA_TRX_ID_LEN, roll_ptr); memcpy(storage, field, DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN); - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); } /**********************************************************************//** @@ -4244,10 +4246,10 @@ page_zip_clear_rec( heap_no = rec_get_heap_no_new(rec); ut_ad(heap_no >= PAGE_HEAP_NO_USER_LOW); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); if (!page_is_leaf(page)) { /* Clear node_ptr. On the compressed page, @@ -4316,7 +4318,7 @@ page_zip_rec_set_deleted( { byte* slot = page_zip_dir_find(page_zip, page_offset(rec)); ut_a(slot); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); if (flag) { *slot |= (PAGE_ZIP_DIR_SLOT_DEL >> 8); } else { @@ -4339,7 +4341,7 @@ page_zip_rec_set_owned( { byte* slot = page_zip_dir_find(page_zip, page_offset(rec)); ut_a(slot); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); if (flag) { *slot |= (PAGE_ZIP_DIR_SLOT_OWNED >> 8); } else { @@ -4366,7 +4368,7 @@ page_zip_dir_insert( ut_ad(page_rec_get_next((rec_t*) prev_rec) == rec); ut_ad(page_zip_simple_validate(page_zip)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); if (page_rec_is_infimum(prev_rec)) { /* Use the first slot. */ @@ -4445,10 +4447,10 @@ page_zip_dir_delete( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(rec_offs_comp(offsets)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); - UNIV_MEM_ASSERT_RW(rec, rec_offs_data_size(offsets)); - UNIV_MEM_ASSERT_RW(rec - rec_offs_extra_size(offsets), - rec_offs_extra_size(offsets)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); + MEM_CHECK_DEFINED(rec - rec_offs_extra_size(offsets), + rec_offs_extra_size(offsets)); slot_rec = page_zip_dir_find(page_zip, page_offset(rec)); @@ -4537,7 +4539,7 @@ page_zip_dir_add_slot( byte* stored; ut_ad(page_is_comp(page_zip->data)); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); /* Read the old n_dense (n_heap has already been incremented). */ n_dense = page_dir_get_n_heap(page_zip->data) @@ -4699,8 +4701,8 @@ page_zip_reorganize( ut_ad(!dict_index_is_ibuf(index)); ut_ad(!dict_table_is_temporary(index->table)); /* Note that page_zip_validate(page_zip, page, index) may fail here. */ - UNIV_MEM_ASSERT_RW(page, UNIV_PAGE_SIZE); - UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(page, srv_page_size); + MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); /* Disable logging */ mtr_log_t log_mode = mtr_set_log_mode(mtr, MTR_LOG_NONE); @@ -4787,10 +4789,10 @@ page_zip_copy_recs( ut_a(dict_index_is_clust(index)); } - UNIV_MEM_ASSERT_W(page, UNIV_PAGE_SIZE); - UNIV_MEM_ASSERT_W(page_zip->data, page_zip_get_size(page_zip)); - UNIV_MEM_ASSERT_RW(src, UNIV_PAGE_SIZE); - UNIV_MEM_ASSERT_RW(src_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_ADDRESSABLE(page, srv_page_size); + MEM_CHECK_ADDRESSABLE(page_zip->data, page_zip_get_size(page_zip)); + MEM_CHECK_DEFINED(src, srv_page_size); + MEM_CHECK_DEFINED(src_zip->data, page_zip_get_size(page_zip)); /* Copy those B-tree page header fields that are related to the records stored in the page. Also copy the field diff --git a/storage/innobase/row/row0ext.cc b/storage/innobase/row/row0ext.cc index f7e28981939..32e9aad9896 100644 --- a/storage/innobase/row/row0ext.cc +++ b/storage/innobase/row/row0ext.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -119,11 +120,6 @@ row_ext_create( ret->buf = static_cast( mem_heap_alloc(heap, n_ext * ret->max_len)); -#ifdef UNIV_DEBUG - memset(ret->buf, 0xaa, n_ext * ret->max_len); - UNIV_MEM_ALLOC(ret->buf, n_ext * ret->max_len); -#endif - /* Fetch the BLOB prefixes */ for (i = 0; i < n_ext; i++) { const dfield_t* dfield; diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index c7c86fb337b..4085ff17bd3 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -903,7 +903,7 @@ loop: goto func_exit; } - UNIV_MEM_INVALID(block[t_ctx.buf_used], srv_sort_buf_size); + MEM_UNDEFINED(block[t_ctx.buf_used], srv_sort_buf_size); buf[t_ctx.buf_used] = row_merge_buf_empty(buf[t_ctx.buf_used]); mycount[t_ctx.buf_used] += t_ctx.rows_added[t_ctx.buf_used]; t_ctx.rows_added[t_ctx.buf_used] = 0; @@ -997,12 +997,14 @@ exit: goto func_exit; } - UNIV_MEM_INVALID(block[i], srv_sort_buf_size); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(block[i], srv_sort_buf_size); if (crypt_block[i]) { - UNIV_MEM_INVALID(crypt_block[i], - srv_sort_buf_size); + MEM_UNDEFINED(crypt_block[i], + srv_sort_buf_size); } +#endif /* HAVE_valgrind_or_MSAN */ } buf[i] = row_merge_buf_empty(buf[i]); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 2ca54e90c4e..50394193a15 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1272,8 +1272,10 @@ row_ins_foreign_check_on_constraint( update->info_bits = 0; update->n_fields = foreign->n_fields; - UNIV_MEM_INVALID(update->fields, - update->n_fields * sizeof *update->fields); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(update->fields, + update->n_fields * sizeof *update->fields); +#endif /* HAVE_valgrind_or_MSAN */ bool affects_fulltext = false; diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index e46ee2c4f18..986cac54540 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -323,7 +323,9 @@ row_log_online_op( goto err_exit; } - UNIV_MEM_INVALID(log->tail.buf, sizeof log->tail.buf); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); +#endif /* HAVE_valgrind_or_MSAN */ ut_ad(log->tail.bytes < srv_sort_buf_size); avail_size = srv_sort_buf_size - log->tail.bytes; @@ -373,7 +375,7 @@ row_log_online_op( log->tail.buf, avail_size); } - UNIV_MEM_ASSERT_RW(buf, srv_sort_buf_size); + MEM_CHECK_DEFINED(buf, srv_sort_buf_size); if (row_log_tmpfile(log) < 0) { log->error = DB_OUT_OF_MEMORY; @@ -407,8 +409,10 @@ write_failed: index->type |= DICT_CORRUPT; } - UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); - UNIV_MEM_INVALID(buf, srv_sort_buf_size); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(log->tail.block, srv_sort_buf_size); + MEM_UNDEFINED(buf, srv_sort_buf_size); +#endif /* HAVE_valgrind_or_MSAN */ memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); @@ -418,7 +422,9 @@ write_failed: ut_ad(b == log->tail.block + log->tail.bytes); } - UNIV_MEM_INVALID(log->tail.buf, sizeof log->tail.buf); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); +#endif /* HAVE_valgrind_or_MSAN */ err_exit: mutex_exit(&log->mutex); } @@ -450,7 +456,9 @@ row_log_table_open( { mutex_enter(&log->mutex); - UNIV_MEM_INVALID(log->tail.buf, sizeof log->tail.buf); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); +#endif /* HAVE_valgrind_or_MSAN */ if (log->error != DB_SUCCESS) { err_exit: @@ -510,7 +518,7 @@ row_log_table_close_func( memcpy(buf + log->tail.bytes, log->tail.buf, avail); } - UNIV_MEM_ASSERT_RW(buf, srv_sort_buf_size); + MEM_CHECK_DEFINED(buf, srv_sort_buf_size); if (row_log_tmpfile(log) < 0) { log->error = DB_OUT_OF_MEMORY; @@ -541,8 +549,10 @@ row_log_table_close_func( write_failed: log->error = DB_ONLINE_LOG_TOO_BIG; } - UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); - UNIV_MEM_INVALID(buf, srv_sort_buf_size); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(log->tail.block, srv_sort_buf_size); + MEM_UNDEFINED(buf, srv_sort_buf_size); +#endif /* HAVE_valgrind_or_MSAN */ memcpy(log->tail.block, log->tail.buf + avail, size - avail); log->tail.bytes = size - avail; } else { @@ -551,7 +561,9 @@ write_failed: } log->tail.total += size; - UNIV_MEM_INVALID(log->tail.buf, sizeof log->tail.buf); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(log->tail.buf, sizeof log->tail.buf); +#endif /* HAVE_valgrind_or_MSAN */ err_exit: mutex_exit(&log->mutex); @@ -2557,7 +2569,9 @@ row_log_table_apply_ops( ut_ad(new_trx_id_col > 0); ut_ad(new_trx_id_col != ULINT_UNDEFINED); - UNIV_MEM_INVALID(&mrec_end, sizeof mrec_end); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(&mrec_end, sizeof mrec_end); +#endif /* HAVE_valgrind_or_MSAN */ offsets = static_cast(ut_malloc_nokey(i * sizeof *offsets)); rec_offs_set_n_alloc(offsets, i); @@ -3434,7 +3448,9 @@ row_log_apply_ops( ut_ad(!index->is_committed()); ut_ad(rw_lock_own(dict_index_get_lock(index), RW_LOCK_X)); ut_ad(index->online_log); - UNIV_MEM_INVALID(&mrec_end, sizeof mrec_end); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(&mrec_end, sizeof mrec_end); +#endif /* HAVE_valgrind_or_MSAN */ offsets = static_cast(ut_malloc_nokey(i * sizeof *offsets)); rec_offs_set_n_alloc(offsets, i); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 5cbfe0e3ddb..3a6d8cea4a8 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1027,11 +1027,11 @@ row_merge_buf_write( ut_a(b < &block[srv_sort_buf_size]); ut_a(b == &block[0] + buf->total_size); *b++ = 0; -#ifdef UNIV_DEBUG_VALGRIND +#ifdef HAVE_valgrind_or_MSAN /* The rest of the block is uninitialized. Initialize it to avoid bogus warnings. */ memset(b, 0xff, &block[srv_sort_buf_size] - b); -#endif /* UNIV_DEBUG_VALGRIND */ +#endif /* HAVE_valgrind_or_MSAN */ DBUG_LOG("ib_merge_sort", "write " << reinterpret_cast(b) << ',' << of->fd << ',' << of->offset << " EOF"); @@ -1425,7 +1425,9 @@ row_merge_write_rec( return(NULL); } - UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(&block[0], srv_sort_buf_size); +#endif /* HAVE_valgrind_or_MSAN */ /* Copy the rest. */ b = &block[0]; @@ -1466,20 +1468,19 @@ row_merge_write_eof( ",fd=" << fd << ',' << *foffs); *b++ = 0; - UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]); - UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size); + MEM_CHECK_DEFINED(&block[0], b - &block[0]); + MEM_CHECK_ADDRESSABLE(&block[0], srv_sort_buf_size); -#ifdef UNIV_DEBUG_VALGRIND - /* The rest of the block is uninitialized. Initialize it - to avoid bogus warnings. */ - memset(b, 0xff, &block[srv_sort_buf_size] - b); -#endif /* UNIV_DEBUG_VALGRIND */ + /* The rest of the block is uninitialized. Silence warnings. */ + MEM_MAKE_DEFINED(b, &block[srv_sort_buf_size] - b); if (!row_merge_write(fd, (*foffs)++, block, crypt_block, space)) { DBUG_RETURN(NULL); } - UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(&block[0], srv_sort_buf_size); +#endif DBUG_RETURN(&block[0]); } @@ -2550,8 +2551,10 @@ write_buffers: break; } - UNIV_MEM_INVALID( +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED( &block[0], srv_sort_buf_size); +#endif /* HAVE_valgrind_or_MSAN */ } } merge_buf[i] = row_merge_buf_empty(buf); @@ -3034,10 +3037,10 @@ row_merge( ulint n_run = 0; /*!< num of runs generated from this merge */ - UNIV_MEM_ASSERT_W(&block[0], 3 * srv_sort_buf_size); + MEM_CHECK_ADDRESSABLE(&block[0], 3 * srv_sort_buf_size); if (crypt_block) { - UNIV_MEM_ASSERT_W(&crypt_block[0], 3 * srv_sort_buf_size); + MEM_CHECK_ADDRESSABLE(&crypt_block[0], 3 * srv_sort_buf_size); } ut_ad(ihalf < file->offset); @@ -3058,7 +3061,9 @@ row_merge( foffs0 = 0; foffs1 = ihalf; - UNIV_MEM_INVALID(run_offset, *num_run * sizeof *run_offset); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(run_offset, *num_run * sizeof *run_offset); +#endif /* HAVE_valgrind_or_MSAN */ for (; foffs0 < ihalf && foffs1 < file->offset; foffs0++, foffs1++) { @@ -3139,7 +3144,9 @@ row_merge( *tmpfd = file->fd; *file = of; - UNIV_MEM_INVALID(&block[0], 3 * srv_sort_buf_size); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(&block[0], 3 * srv_sort_buf_size); +#endif /* HAVE_valgrind_or_MSAN */ return(DB_SUCCESS); } @@ -3252,7 +3259,7 @@ row_merge_sort( break; } - UNIV_MEM_ASSERT_RW(run_offset, num_runs * sizeof *run_offset); + MEM_CHECK_DEFINED(run_offset, num_runs * sizeof *run_offset); } while (num_runs > 1); ut_free(run_offset); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 3989095d6c6..b304c03f7ed 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -297,9 +297,7 @@ row_mysql_store_geometry( { /* MySQL might assume the field is set to zero except the length and the pointer fields */ - UNIV_MEM_ASSERT_RW(src, src_len); - UNIV_MEM_ASSERT_W(dest, dest_len); - UNIV_MEM_INVALID(dest, dest_len); + MEM_CHECK_DEFINED(src, src_len); memset(dest, '\0', dest_len); diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 15486500b37..9340d5060d9 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -984,9 +984,11 @@ row_sel_get_clust_rec( switch (err) { case DB_SUCCESS: case DB_SUCCESS_LOCKED_REC: - /* Declare the variable uninitialized in Valgrind. +#ifdef HAVE_valgrind_or_MSAN + /* Declare the variable uninitialized. It should be set to DB_SUCCESS at func_exit. */ - UNIV_MEM_INVALID(&err, sizeof err); + MEM_UNDEFINED(&err, sizeof err); +#endif /* HAVE_valgrind_or_MSAN */ break; default: goto err_exit; @@ -2811,9 +2813,11 @@ row_sel_field_store_in_mysql_format_func( #endif /* UNIV_DEBUG */ ut_ad(len != UNIV_SQL_NULL); - UNIV_MEM_ASSERT_RW(data, len); - UNIV_MEM_ASSERT_W(dest, templ->mysql_col_len); - UNIV_MEM_INVALID(dest, templ->mysql_col_len); + MEM_CHECK_DEFINED(data, len); + MEM_CHECK_ADDRESSABLE(dest, templ->mysql_col_len); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(dest, templ->mysql_col_len); +#endif /* HAVE_valgrind_or_MSAN */ switch (templ->type) { const byte* field_end; @@ -3075,9 +3079,9 @@ row_sel_store_mysql_field_func( NULL value is set to the default value. */ ut_ad(templ->mysql_null_bit_mask); - UNIV_MEM_ASSERT_RW(prebuilt->default_rec - + templ->mysql_col_offset, - templ->mysql_col_len); + MEM_CHECK_DEFINED(prebuilt->default_rec + + templ->mysql_col_offset, + templ->mysql_col_len); mysql_rec[templ->mysql_null_byte_offset] |= (byte) templ->mysql_null_bit_mask; memcpy(mysql_rec + templ->mysql_col_offset, @@ -3715,7 +3719,7 @@ row_sel_copy_cached_field_for_mysql( buf += templ->mysql_col_offset; cache += templ->mysql_col_offset; - UNIV_MEM_ASSERT_W(buf, templ->mysql_col_len); + MEM_CHECK_ADDRESSABLE(buf, templ->mysql_col_len); if (templ->mysql_type == DATA_MYSQL_TRUE_VARCHAR && (templ->type != DATA_INT)) { @@ -3725,7 +3729,9 @@ row_sel_copy_cached_field_for_mysql( row_mysql_read_true_varchar( &len, cache, templ->mysql_length_bytes); len += templ->mysql_length_bytes; - UNIV_MEM_INVALID(buf, templ->mysql_col_len); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(buf, templ->mysql_col_len); +#endif /* HAVE_valgrind_or_MSAN */ } else { len = templ->mysql_col_len; } @@ -3784,7 +3790,7 @@ row_sel_dequeue_cached_row_for_mysql( ut_ad(prebuilt->n_fetch_cached > 0); ut_ad(prebuilt->mysql_prefix_len <= prebuilt->mysql_row_len); - UNIV_MEM_ASSERT_W(buf, prebuilt->mysql_row_len); + MEM_CHECK_ADDRESSABLE(buf, prebuilt->mysql_row_len); cached_rec = prebuilt->fetch_cache[prebuilt->fetch_cache_first]; @@ -3794,7 +3800,9 @@ row_sel_dequeue_cached_row_for_mysql( /* The record is long. Copy it field by field, in case there are some long VARCHAR column of which only a small length is being used. */ - UNIV_MEM_INVALID(buf, prebuilt->mysql_prefix_len); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(buf, prebuilt->mysql_prefix_len); +#endif /* HAVE_valgrind_or_MSAN */ /* First copy the NULL bits. */ ut_memcpy(buf, cached_rec, prebuilt->null_bitmap_len); @@ -3878,8 +3886,10 @@ row_sel_fetch_last_buf( } ut_ad(prebuilt->fetch_cache_first == 0); - UNIV_MEM_INVALID(prebuilt->fetch_cache[prebuilt->n_fetch_cached], - prebuilt->mysql_row_len); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(prebuilt->fetch_cache[prebuilt->n_fetch_cached], + prebuilt->mysql_row_len); +#endif /* HAVE_valgrind_or_MSAN */ return(prebuilt->fetch_cache[prebuilt->n_fetch_cached]); } diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index a7cd06d25fd..63c1ea8d662 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1867,7 +1867,9 @@ row_upd_changes_ord_field_binary_func( /* Silence a compiler warning without silencing a Valgrind error. */ dfield_len = 0; - UNIV_MEM_INVALID(&dfield_len, sizeof dfield_len); +#ifdef HAVE_valgrind_or_MSAN + MEM_UNDEFINED(&dfield_len, sizeof dfield_len); +#endif /* HAVE_valgrind_or_MSAN */ /* See if the column is stored externally. */ buf = row_ext_lookup(ext, col_no, &dfield_len); diff --git a/storage/innobase/sync/sync0arr.cc b/storage/innobase/sync/sync0arr.cc index 0c942ada430..69d4692aa57 100644 --- a/storage/innobase/sync/sync0arr.cc +++ b/storage/innobase/sync/sync0arr.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -979,9 +979,9 @@ sync_array_print_long_waits_low( return(false); } -#ifdef UNIV_DEBUG_VALGRIND +#ifdef HAVE_valgrind /* Increase the timeouts if running under valgrind because it executes - extremely slowly. UNIV_DEBUG_VALGRIND does not necessary mean that + extremely slowly. HAVE_valgrind does not necessary mean that we are running under valgrind but we have no better way to tell. See Bug#58432 innodb.innodb_bug56143 fails under valgrind for an example */ diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index a19ab2c9c98..d4cd020b321 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -417,16 +417,16 @@ trx_free(trx_t*& trx) MEM_UNDEFINED(&trx->state, sizeof trx->state); MEM_UNDEFINED(&trx->mysql_thd, sizeof trx->mysql_thd); #endif -#ifdef HAVE_valgrind +#ifdef HAVE_valgrind_or_MSAN /* Unpoison the memory for innodb_monitor_set_option; it is operating also on the freed transaction objects. We checked that these were initialized in trx_pools->mem_free(trx). */ - UNIV_MEM_VALID(&trx->mutex, sizeof trx->mutex); - UNIV_MEM_VALID(&trx->undo_mutex, sizeof trx->undo_mutex); + MEM_MAKE_DEFINED(&trx->mutex, sizeof trx->mutex); + MEM_MAKE_DEFINED(&trx->undo_mutex, sizeof trx->undo_mutex); /* For innobase_kill_connection() */ - UNIV_MEM_VALID(&trx->state, sizeof trx->state); - UNIV_MEM_VALID(&trx->mysql_thd, sizeof trx->mysql_thd); + MEM_MAKE_DEFINED(&trx->state, sizeof trx->state); + MEM_MAKE_DEFINED(&trx->mysql_thd, sizeof trx->mysql_thd); #endif trx = NULL; diff --git a/storage/innobase/ut/ut0crc32.cc b/storage/innobase/ut/ut0crc32.cc index 44b1c4b30b4..a2ae570f4fe 100644 --- a/storage/innobase/ut/ut0crc32.cc +++ b/storage/innobase/ut/ut0crc32.cc @@ -200,15 +200,17 @@ ut_crc32_8_hw( const byte** data, ulint* len) { -#ifdef _MSC_VER +# ifdef _MSC_VER *crc = _mm_crc32_u8(*crc, (*data)[0]); -#else +# elif __has_feature(memory_sanitizer) + *crc = __builtin_ia32_crc32qi(*crc, (*data)[0]); +# else asm("crc32b %1, %0" /* output operands */ : "+r" (*crc) /* input operands */ : "rm" ((*data)[0])); -#endif +# endif (*data)++; (*len)--; @@ -225,22 +227,24 @@ ut_crc32_64_low_hw( uint64_t data) { uint64_t crc_64bit = crc; -#ifdef _MSC_VER -#ifdef _M_X64 +# ifdef _MSC_VER +# ifdef _M_X64 crc_64bit = _mm_crc32_u64(crc_64bit, data); -#elif defined(_M_IX86) +# elif defined(_M_IX86) crc = _mm_crc32_u32(crc, static_cast(data)); crc_64bit = _mm_crc32_u32(crc, static_cast(data >> 32)); -#else -#error Not Supported processors type. -#endif -#else +# else +# error Not Supported processors type. +# endif +# elif __has_feature(memory_sanitizer) + crc_64bit = __builtin_ia32_crc32di(crc_64bit, data); +# else asm("crc32q %1, %0" /* output operands */ : "+r" (crc_64bit) /* input operands */ : "rm" (data)); -#endif +# endif return(static_cast(crc_64bit)); } diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index c6822c1d74e..1b3d100f9ee 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -168,8 +168,6 @@ ut_print_buf( const byte* data; ulint i; - UNIV_MEM_ASSERT_RW(buf, len); - fprintf(file, " len " ULINTPF "; hex ", len); for (data = (const byte*) buf, i = 0; i < len; i++) { @@ -204,8 +202,6 @@ ut_print_buf_hex( '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F' }; - UNIV_MEM_ASSERT_RW(buf, len); - o << "(0x"; for (data = static_cast(buf), i = 0; i < len; i++) { @@ -228,8 +224,6 @@ ut_print_buf( const byte* data; ulint i; - UNIV_MEM_ASSERT_RW(buf, len); - for (data = static_cast(buf), i = 0; i < len; i++) { int c = static_cast(*data++); o << (isprint(c) ? static_cast(c) : ' '); diff --git a/strings/my_vsnprintf.c b/strings/my_vsnprintf.c index f9c0ad0a52c..fd2de08a38a 100644 --- a/strings/my_vsnprintf.c +++ b/strings/my_vsnprintf.c @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates. - Copyright (c) 2009-2011, Monty Program Ab + Copyright (c) 2009, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -704,7 +704,13 @@ size_t my_vsnprintf_ex(CHARSET_INFO *cs, char *to, size_t n, } else if (*fmt == 'f' || *fmt == 'g') { +#if __has_feature(memory_sanitizer) /* QQ: MSAN has double trouble? */ + __msan_check_mem_is_initialized(ap, sizeof(double)); +#endif double d= va_arg(ap, double); +#if __has_feature(memory_sanitizer) /* QQ: MSAN has double trouble? */ + __msan_unpoison(&d, sizeof(double)); +#endif to= process_dbl_arg(to, end, width, d, *fmt); continue; } From be51738465ff30eda627c72168e06e2ea9ca13f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 1 Jul 2020 17:43:44 +0300 Subject: [PATCH 67/73] MDEV-20428 after-merge fix: Stabilize the test --- .../rpl/r/rpl_binlog_dump_slave_gtid_state_info.result | 6 +++--- .../suite/rpl/t/rpl_binlog_dump_slave_gtid_state_info.test | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_binlog_dump_slave_gtid_state_info.result b/mysql-test/suite/rpl/r/rpl_binlog_dump_slave_gtid_state_info.result index b7f06a7dc46..98688df7273 100644 --- a/mysql-test/suite/rpl/r/rpl_binlog_dump_slave_gtid_state_info.result +++ b/mysql-test/suite/rpl/r/rpl_binlog_dump_slave_gtid_state_info.result @@ -8,7 +8,7 @@ CHANGE MASTER TO MASTER_USE_GTID=current_pos; include/start_slave.inc connection master; "Test Case 1: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('')" -FOUND 1 /using_gtid\(1\), gtid\(\'\'\)/ in mysqld.1.err +FOUND 1 /using_gtid\(1\), gtid\(\'\'\).*/ in mysqld.1.err connection slave; include/stop_slave.inc CHANGE MASTER TO MASTER_USE_GTID=no; @@ -25,7 +25,7 @@ CHANGE MASTER TO MASTER_USE_GTID=slave_pos; include/start_slave.inc connection master; "Test Case 3: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('0-1-2')" -FOUND 1 /using_gtid\(1\), gtid\(\'0-1-2\'\)/ in mysqld.1.err +FOUND 1 /using_gtid\(1\), gtid\(\'0-1-2\'\).*/ in mysqld.1.err SET @@SESSION.gtid_domain_id=10; INSERT INTO t VALUES(20); connection slave; @@ -35,7 +35,7 @@ CHANGE MASTER TO MASTER_USE_GTID=slave_pos; include/start_slave.inc connection master; "Test Case 4: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('0-1-2,10-1-1')" -FOUND 1 /using_gtid\(1\), gtid\(\'0-1-2,10-1-1\'\)/ in mysqld.1.err +FOUND 1 /using_gtid\(1\), gtid\(\'0-1-2,10-1-1\'\).*/ in mysqld.1.err "===== Clean up =====" connection slave; include/stop_slave.inc diff --git a/mysql-test/suite/rpl/t/rpl_binlog_dump_slave_gtid_state_info.test b/mysql-test/suite/rpl/t/rpl_binlog_dump_slave_gtid_state_info.test index 63d4256bc78..f26e9565671 100644 --- a/mysql-test/suite/rpl/t/rpl_binlog_dump_slave_gtid_state_info.test +++ b/mysql-test/suite/rpl/t/rpl_binlog_dump_slave_gtid_state_info.test @@ -58,7 +58,7 @@ if(!$log_error_) --echo "Test Case 1: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('')" --let SEARCH_FILE=$log_error_ --let SEARCH_RANGE=-50000 ---let SEARCH_PATTERN=using_gtid\(1\), gtid\(\'\'\) +--let SEARCH_PATTERN=using_gtid\(1\), gtid\(\'\'\).* --source include/search_pattern_in_file.inc --connection slave @@ -88,7 +88,7 @@ CHANGE MASTER TO MASTER_USE_GTID=slave_pos; --echo "Test Case 3: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('0-1-2')" --let SEARCH_FILE=$log_error_ --let SEARCH_RANGE=-50000 ---let SEARCH_PATTERN=using_gtid\(1\), gtid\(\'0-1-2\'\) +--let SEARCH_PATTERN=using_gtid\(1\), gtid\(\'0-1-2\'\).* --source include/search_pattern_in_file.inc SET @@SESSION.gtid_domain_id=10; INSERT INTO t VALUES(20); @@ -106,7 +106,7 @@ CHANGE MASTER TO MASTER_USE_GTID=slave_pos; --echo "Test Case 4: Start binlog_dump to slave_server(#), pos(master-bin.000001, ###), using_gtid(1), gtid('0-1-2,10-1-1')" --let SEARCH_FILE=$log_error_ --let SEARCH_RANGE=-50000 ---let SEARCH_PATTERN=using_gtid\(1\), gtid\(\'0-1-2,10-1-1\'\) +--let SEARCH_PATTERN=using_gtid\(1\), gtid\(\'0-1-2,10-1-1\'\).* --source include/search_pattern_in_file.inc --echo "===== Clean up =====" From 41b0d98e69740f6d894a369a83efbbe5f7bffdfd Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 1 Jul 2020 18:43:21 +0200 Subject: [PATCH 68/73] MDEV-23067 Windows : manually registered services rejected mysql_upgrade_service - service not using "--defaults-file" can have any name not just "MySQL" - service with "--defaults-file", without datadir in them use default datadir (install_root\data) --- sql/winservice.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/sql/winservice.c b/sql/winservice.c index d9605f3a6c4..fe1fb0cb2bb 100644 --- a/sql/winservice.c +++ b/sql/winservice.c @@ -150,10 +150,7 @@ int get_mysql_service_properties(const wchar_t *bin_path, There are rare cases where service config does not have --defaults-filein the binary parth . There services were registered with plain mysqld --install, the data directory is next to "bin" in this case. - Service name (second parameter) must be MySQL. */ - if(wcscmp(args[1], L"MySQL") != 0) - goto end; have_inifile= FALSE; } else if(numargs == 3) @@ -211,7 +208,7 @@ int get_mysql_service_properties(const wchar_t *bin_path, } } - if(!have_inifile) + if(!have_inifile || props->datadir[0] == 0) { /* Hard, although a rare case, we're guessing datadir and defaults-file. @@ -235,22 +232,25 @@ int get_mysql_service_properties(const wchar_t *bin_path, *p= 0; } - /* Look for my.ini, my.cnf in the install root */ - sprintf_s(props->inifile, MAX_PATH, "%s\\my.ini", install_root); - if (GetFileAttributes(props->inifile) == INVALID_FILE_ATTRIBUTES) + if (!have_inifile) { - sprintf_s(props->inifile, MAX_PATH, "%s\\my.cnf", install_root); - } - if (GetFileAttributes(props->inifile) != INVALID_FILE_ATTRIBUTES) - { - /* Ini file found, get datadir from there */ - GetPrivateProfileString("mysqld", "datadir", NULL, props->datadir, - MAX_PATH, props->inifile); - } - else - { - /* No ini file */ - props->inifile[0]= 0; + /* Look for my.ini, my.cnf in the install root */ + sprintf_s(props->inifile, MAX_PATH, "%s\\my.ini", install_root); + if (GetFileAttributes(props->inifile) == INVALID_FILE_ATTRIBUTES) + { + sprintf_s(props->inifile, MAX_PATH, "%s\\my.cnf", install_root); + } + if (GetFileAttributes(props->inifile) != INVALID_FILE_ATTRIBUTES) + { + /* Ini file found, get datadir from there */ + GetPrivateProfileString("mysqld", "datadir", NULL, props->datadir, + MAX_PATH, props->inifile); + } + else + { + /* No ini file */ + props->inifile[0]= 0; + } } /* Try datadir in install directory.*/ From 69df4f834b09d2405610afc81f055658188ab8c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 1 Jul 2020 20:34:06 +0300 Subject: [PATCH 69/73] MDEV-20377: Fix -Wunused-but-set-variable --- storage/innobase/data/data0data.cc | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/storage/innobase/data/data0data.cc b/storage/innobase/data/data0data.cc index b4d41f47060..fba06e78987 100644 --- a/storage/innobase/data/data0data.cc +++ b/storage/innobase/data/data0data.cc @@ -194,29 +194,20 @@ dtuple_validate( /*============*/ const dtuple_t* tuple) /*!< in: tuple */ { - const dfield_t* field; - ulint n_fields; - ulint len; - ulint i; - ut_ad(tuple->magic_n == DATA_TUPLE_MAGIC_N); +#ifdef HAVE_valgrind_or_MSAN + const ulint n_fields = dtuple_get_n_fields(tuple); - n_fields = dtuple_get_n_fields(tuple); - - /* We dereference all the data of each field to test - for memory traps */ - - for (i = 0; i < n_fields; i++) { - - field = dtuple_get_nth_field(tuple, i); - len = dfield_get_len(field); + for (ulint i = 0; i < n_fields; i++) { + const dfield_t* field = dtuple_get_nth_field(tuple, i); if (!dfield_is_null(field)) { - MEM_CHECK_DEFINED(dfield_get_data(field), len); + MEM_CHECK_DEFINED(dfield_get_data(field), + dfield_get_len(field)); } } - - ut_a(dtuple_check_typed(tuple)); +#endif /* HAVE_valgrind_or_MSAN */ + ut_ad(dtuple_check_typed(tuple)); return(TRUE); } From b0f836053b094b09999c102d10bf0ad6ed761ac6 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Tue, 23 Jun 2020 17:07:03 +0200 Subject: [PATCH 70/73] MDEV-22983: Mariabackup's --help option disappeared return --help option --- extra/mariabackup/xtrabackup.cc | 6 ++++++ mysql-test/suite/mariabackup/options_check.result | 1 + mysql-test/suite/mariabackup/options_check.test | 7 ++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 3946879415d..0f30e02fb7b 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -119,6 +119,7 @@ my_bool xtrabackup_move_back; my_bool xtrabackup_decrypt_decompress; my_bool xtrabackup_print_param; my_bool xtrabackup_mysqld_args; +my_bool xtrabackup_help; my_bool xtrabackup_export; @@ -1458,6 +1459,11 @@ struct my_option xb_server_options[] = (G_PTR *) &xtrabackup_mysqld_args, (G_PTR *) &xtrabackup_mysqld_args, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"help", '?', + "Display this help and exit.", + (G_PTR *) &xtrabackup_help, (G_PTR *) &xtrabackup_help, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; diff --git a/mysql-test/suite/mariabackup/options_check.result b/mysql-test/suite/mariabackup/options_check.result index 6b9925a0a8e..59666754f04 100644 --- a/mysql-test/suite/mariabackup/options_check.result +++ b/mysql-test/suite/mariabackup/options_check.result @@ -5,3 +5,4 @@ # Check for options overwriting # Check if uknown options that follow --mysqld-args are ingored # Check if [mariadb-client] group is not loaded (MDEV-22894) +# Check if --help presents diff --git a/mysql-test/suite/mariabackup/options_check.test b/mysql-test/suite/mariabackup/options_check.test index 7483453d19b..022bcbd5d10 100644 --- a/mysql-test/suite/mariabackup/options_check.test +++ b/mysql-test/suite/mariabackup/options_check.test @@ -7,7 +7,6 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --unknown-option=xxx --target-dir=$targetdir; --error 2 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --unknown-option --target-dir=$targetdir; ---enable_result_log --echo # Check for unknown options in "mariabackup" group --write_file $custom_cnf @@ -62,3 +61,9 @@ EOF exec $XTRABACKUP --defaults-file=$custom_cnf --backup --target-dir=$targetdir; --remove_file $custom_cnf --rmdir $targetdir + +--echo # Check if --help presents +exec $XTRABACKUP --help; +exec $XTRABACKUP -?; +--enable_result_log + From 838a1046b247e0c70089d3b5cf609c0a40fa3e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 2 Jul 2020 06:03:59 +0300 Subject: [PATCH 71/73] MDEV-20377: Fix cmake -DPLUGIN_PERFSCHEMA=NO --- storage/innobase/ut/ut0crc32.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/innobase/ut/ut0crc32.cc b/storage/innobase/ut/ut0crc32.cc index a2ae570f4fe..5c62309ee89 100644 --- a/storage/innobase/ut/ut0crc32.cc +++ b/storage/innobase/ut/ut0crc32.cc @@ -84,6 +84,7 @@ mysys/my_perf.c, contributed by Facebook under the following license. #include #include "ut0crc32.h" +#include "my_valgrind.h" #ifdef _MSC_VER #include From 90d1e58ed0a148cf850f187e948e9ca24ab3a88d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 2 Jul 2020 06:04:31 +0300 Subject: [PATCH 72/73] MDEV-22941: Fix the DBUG_ENTER name --- sql/sql_show.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 70cab968f43..8eba6ef6027 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2009, 2017, MariaDB + Copyright (c) 2009, 2020, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -4962,16 +4962,16 @@ bool store_schema_shemata(THD* thd, TABLE *table, LEX_STRING *db_name, */ static bool verify_database_directory_exists(const LEX_STRING &dbname) { - DBUG_ENTER("verity_database_exists"); + DBUG_ENTER("verify_database_directory_exists"); char path[FN_REFLEN + 16]; uint path_len; MY_STAT stat_info; if (!dbname.str[0]) - DBUG_RETURN(true); // Empty database name: does not exits. + DBUG_RETURN(true); // Empty database name: does not exist. path_len= build_table_filename(path, sizeof(path) - 1, dbname.str, "", "", 0); path[path_len - 1]= 0; if (!mysql_file_stat(key_file_misc, path, &stat_info, MYF(0))) - DBUG_RETURN(true); // The database directory was not found: does not exists. + DBUG_RETURN(true); // The database directory was not found: does not exist. DBUG_RETURN(false); // The database directory was found. } From c43a666662ac71e70bde83e6673ebcb2811887af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 2 Jul 2020 06:04:42 +0300 Subject: [PATCH 73/73] Revert "Fix result of merge." This reverts commit e0793d386517f4ff9c0267830d558f91c75263aa. In idiomatic C++, accessor functions should not discard qualifiers. --- sql/sql_class.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/sql_class.h b/sql/sql_class.h index 4d14b42b065..7ca3896a69d 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3408,7 +3408,11 @@ public: inline bool is_error() const { return m_stmt_da->is_error(); } /// Returns Diagnostics-area for the current statement. - Diagnostics_area *get_stmt_da() const + Diagnostics_area *get_stmt_da() + { return m_stmt_da; } + + /// Returns Diagnostics-area for the current statement. + const Diagnostics_area *get_stmt_da() const { return m_stmt_da; } /// Sets Diagnostics-area for the current statement.