From 4e667b96389a0d77836770b05d78110caec66f9d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 17 Feb 2022 18:17:05 +0100 Subject: [PATCH 01/11] MDEV-27877 considerably speed up innodb_force_recovery test decrease innodb_lock_wait_timeout for the current session. --- mysql-test/suite/innodb/r/innodb_force_recovery.result | 2 +- mysql-test/suite/innodb/t/innodb_force_recovery.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_force_recovery.result b/mysql-test/suite/innodb/r/innodb_force_recovery.result index 8e700f9489d..38892e4c838 100644 --- a/mysql-test/suite/innodb/r/innodb_force_recovery.result +++ b/mysql-test/suite/innodb/r/innodb_force_recovery.result @@ -113,7 +113,7 @@ SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; select * from t2; f1 f2 1 2 -SET GLOBAL innodb_lock_wait_timeout=1; +SET SESSION innodb_lock_wait_timeout=1; insert into t2 values(1,2); ERROR HY000: Lock wait timeout exceeded; try restarting transaction insert into t2 values(9,10); diff --git a/mysql-test/suite/innodb/t/innodb_force_recovery.test b/mysql-test/suite/innodb/t/innodb_force_recovery.test index 9d7cf9d80b7..b8b8cc9c900 100644 --- a/mysql-test/suite/innodb/t/innodb_force_recovery.test +++ b/mysql-test/suite/innodb/t/innodb_force_recovery.test @@ -162,7 +162,7 @@ SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; select * from t2; SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; select * from t2; -SET GLOBAL innodb_lock_wait_timeout=1; +SET SESSION innodb_lock_wait_timeout=1; --error ER_LOCK_WAIT_TIMEOUT insert into t2 values(1,2); insert into t2 values(9,10); From 24ec144c63b36402adaff2bc12aaabefa40bdd51 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 20 Feb 2022 20:48:31 +0100 Subject: [PATCH 02/11] MDEV-27901 Windows : expensive system calls used to calculate file system block size The result is not used anywhere but in the output of Innodb information schema, but this can take as much as 7%CPU (only) on a benchmark. Fix to move fs blocksize calculate to where it is used. --- storage/innobase/handler/i_s.cc | 29 ++++--------- storage/innobase/os/os0file.cc | 72 ++++++++++++++------------------- 2 files changed, 37 insertions(+), 64 deletions(-) diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 1111d974ad0..fb7f0e5ecbb 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -6745,6 +6745,9 @@ static ST_FIELD_INFO innodb_sys_tablespaces_fields_info[]= END_OF_ST_FIELD_INFO }; + +extern size_t os_file_get_fs_block_size(const char *path); + /**********************************************************************//** Function to fill INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES with information collected by scanning SYS_TABLESPACESS table. @@ -6814,11 +6817,10 @@ i_s_dict_fill_sys_tablespaces( OK(fields[SYS_TABLESPACES_ZIP_PAGE_SIZE]->store( page_size.physical(), true)); - os_file_stat_t stat; + size_t fs_block_size = 0; os_file_size_t file; memset(&file, 0xff, sizeof(file)); - memset(&stat, 0x0, sizeof(stat)); if (fil_space_t* s = fil_space_acquire_silent(space)) { const char *filepath = s->chain.start @@ -6828,36 +6830,19 @@ i_s_dict_fill_sys_tablespaces( } file = os_file_get_size(filepath); - - /* Get the file system (or Volume) block size. */ - switch (dberr_t err = os_file_get_status(filepath, &stat, - false, false)) { - case DB_FAIL: - ib::warn() - << "File '" << filepath << "', failed to get " - << "stats"; - break; - - case DB_SUCCESS: - case DB_NOT_FOUND: - break; - - default: - ib::error() << "File '" << filepath << "' " << err; - break; - } + fs_block_size= os_file_get_fs_block_size(filepath); file_done: fil_space_release(s); } if (file.m_total_size == static_cast(~0)) { - stat.block_size = 0; + fs_block_size = 0; file.m_total_size = 0; file.m_alloc_size = 0; } - OK(fields[SYS_TABLESPACES_FS_BLOCK_SIZE]->store(stat.block_size, true)); + OK(fields[SYS_TABLESPACES_FS_BLOCK_SIZE]->store(fs_block_size, true)); OK(fields[SYS_TABLESPACES_FILE_SIZE]->store(file.m_total_size, true)); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 48be5256ab1..fbfbdb27c0c 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -4795,48 +4795,6 @@ os_file_get_status_win32( CloseHandle(fh); } } - stat_info->block_size = 0; - - /* What follows, is calculation of FS block size, which is not important - (it is just shown in I_S innodb tables). The error to calculate it will be ignored.*/ - char volname[MAX_PATH]; - BOOL result = GetVolumePathName(path, volname, MAX_PATH); - static bool warned_once = false; - if (!result) { - if (!warned_once) { - ib::warn() - << "os_file_get_status_win32: " - << "Failed to get the volume path name for: " - << path - << "- OS error number " << GetLastError(); - warned_once = true; - } - return(DB_SUCCESS); - } - - DWORD sectorsPerCluster; - DWORD bytesPerSector; - DWORD numberOfFreeClusters; - DWORD totalNumberOfClusters; - - result = GetDiskFreeSpace( - (LPCSTR) volname, - §orsPerCluster, - &bytesPerSector, - &numberOfFreeClusters, - &totalNumberOfClusters); - - if (!result) { - if (!warned_once) { - ib::warn() - << "GetDiskFreeSpace(" << volname << ",...) " - << "failed " - << "- OS error number " << GetLastError(); - warned_once = true; - } - return(DB_SUCCESS); - } - stat_info->block_size = bytesPerSector * sectorsPerCluster; } else { stat_info->type = OS_FILE_TYPE_UNKNOWN; } @@ -5736,6 +5694,36 @@ os_is_sparse_file_supported(os_file_t fh) #endif /* _WIN32 */ } +/* + Get file system block size, by path. + + This is expensive on Windows, and not very useful in general, + (only shown in some I_S table), so we keep that out of usual + stat. +*/ +size_t os_file_get_fs_block_size(const char *path) +{ +#ifdef _WIN32 + char volname[MAX_PATH]; + if (!GetVolumePathName(path, volname, MAX_PATH)) + return 0; + DWORD sectorsPerCluster; + DWORD bytesPerSector; + DWORD numberOfFreeClusters; + DWORD totalNumberOfClusters; + + if (GetDiskFreeSpace(volname, §orsPerCluster, &bytesPerSector, + &numberOfFreeClusters, &totalNumberOfClusters)) + return ((size_t) bytesPerSector) * sectorsPerCluster; +#else + os_file_stat_t info; + if (os_file_get_status(path, &info, false, false) == DB_SUCCESS) + return info.block_size; +#endif + return 0; +} + + /** This function returns information about the specified file @param[in] path pathname of the file @param[out] stat_info information of a file in a directory From d140d276244508d7f2e69a3c1a23e0b09201f71a Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 16 Feb 2022 21:04:56 +0300 Subject: [PATCH 03/11] MDEV-22377: Subquery in an UPDATE query uses full scan instead of range [Patch idea by Igor Babaev] Symptom: for IN (SELECT ...) subqueries using IN-to-EXISTS transformation, the optimizer was unable to make inferences using multiple equalities. The cause is code Item_in_subselect::inject_in_to_exists_cond() which may break invariants that Multiple-Equality code relies on. In particular, it may produce a WHERE condition with an empty Item_cond::m_cond_equal. Fixed this by making Item_cond::m_cond_equal. --- mysql-test/r/subselect2.result | 2 +- mysql-test/r/subselect3.result | 4 +- mysql-test/r/subselect3_jcl6.result | 4 +- mysql-test/r/subselect4.result | 40 ++++++++++++++++++- mysql-test/r/subselect_mat_cost_bugs.result | 4 +- mysql-test/t/subselect4.test | 43 +++++++++++++++++++++ sql/item_subselect.cc | 9 ++--- 7 files changed, 92 insertions(+), 14 deletions(-) diff --git a/mysql-test/r/subselect2.result b/mysql-test/r/subselect2.result index 155d5ab2a50..91d3e445b48 100644 --- a/mysql-test/r/subselect2.result +++ b/mysql-test/r/subselect2.result @@ -262,7 +262,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2c ALL NULL NULL NULL NULL 2 100.00 Using where; Using join buffer (incremental, BNL join) Warnings: Note 1276 Field or reference 'test.t1.a' of SELECT #2 was resolved in SELECT #1 -Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`c1` AS `c1` from `test`.`t1` where !<`test`.`t1`.`c1`,`test`.`t1`.`a`>((`test`.`t1`.`c1`,(select `test`.`t2a`.`c2` from `test`.`t2` `t2a` join `test`.`t2` `t2b` join `test`.`t2` `t2c` where (`test`.`t2b`.`m` <> `test`.`t1`.`a` or `test`.`t2b`.`m` = `test`.`t2a`.`m`) and trigcond((`test`.`t1`.`c1`) = `test`.`t2a`.`c2` or `test`.`t2a`.`c2` is null) and `test`.`t2c`.`c2` = `test`.`t2b`.`c2` and `test`.`t2b`.`n` = `test`.`t2a`.`m` having trigcond(`test`.`t2a`.`c2` is null)))) +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`c1` AS `c1` from `test`.`t1` where !<`test`.`t1`.`c1`,`test`.`t1`.`a`>((`test`.`t1`.`c1`,(select `test`.`t2a`.`c2` from `test`.`t2` `t2a` join `test`.`t2` `t2b` join `test`.`t2` `t2c` where `test`.`t2c`.`c2` = `test`.`t2b`.`c2` and `test`.`t2b`.`n` = `test`.`t2a`.`m` and (`test`.`t2b`.`m` <> `test`.`t1`.`a` or `test`.`t2b`.`m` = `test`.`t2a`.`m`) and trigcond((`test`.`t1`.`c1`) = `test`.`t2a`.`c2` or `test`.`t2a`.`c2` is null) having trigcond(`test`.`t2a`.`c2` is null)))) DROP TABLE t1,t2; # # MDEV-614, also MDEV-536, also LP:1050806: diff --git a/mysql-test/r/subselect3.result b/mysql-test/r/subselect3.result index 8d33e4e1606..d92626380ef 100644 --- a/mysql-test/r/subselect3.result +++ b/mysql-test/r/subselect3.result @@ -169,7 +169,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 ref a a 5 test.t1.b 1 100.00 Using where Warnings: Note 1276 Field or reference 'test.t3.oref' of SELECT #2 was resolved in SELECT #1 -Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<`test`.`t3`.`a`,`test`.`t3`.`oref`>((`test`.`t3`.`a`,(select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond((`test`.`t3`.`a`) = `test`.`t1`.`a` or `test`.`t1`.`a` is null) and `test`.`t2`.`a` = `test`.`t1`.`b` having trigcond(`test`.`t1`.`a` is null)))) AS `Z` from `test`.`t3` +Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<`test`.`t3`.`a`,`test`.`t3`.`oref`>((`test`.`t3`.`a`,(select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`a` = `test`.`t1`.`b` and `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond((`test`.`t3`.`a`) = `test`.`t1`.`a` or `test`.`t1`.`a` is null) having trigcond(`test`.`t1`.`a` is null)))) AS `Z` from `test`.`t3` drop table t1, t2, t3; create table t1 (a int NOT NULL, b int NOT NULL, key(a)); insert into t1 values @@ -197,7 +197,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 ref a a 4 test.t1.b 1 100.00 Using where Warnings: Note 1276 Field or reference 'test.t3.oref' of SELECT #2 was resolved in SELECT #1 -Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<`test`.`t3`.`a`,`test`.`t3`.`oref`>((`test`.`t3`.`a`,(select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond((`test`.`t3`.`a`) = `test`.`t1`.`a`) and `test`.`t2`.`a` = `test`.`t1`.`b`))) AS `Z` from `test`.`t3` +Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<`test`.`t3`.`a`,`test`.`t3`.`oref`>((`test`.`t3`.`a`,(select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`a` = `test`.`t1`.`b` and `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond((`test`.`t3`.`a`) = `test`.`t1`.`a`)))) AS `Z` from `test`.`t3` drop table t1,t2,t3; create table t1 (oref int, grp int); insert into t1 (oref, grp) values diff --git a/mysql-test/r/subselect3_jcl6.result b/mysql-test/r/subselect3_jcl6.result index 6d1e305bc49..2df3f2eea35 100644 --- a/mysql-test/r/subselect3_jcl6.result +++ b/mysql-test/r/subselect3_jcl6.result @@ -172,7 +172,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 ref a a 5 test.t1.b 1 100.00 Using where; Using join buffer (flat, BKA join); Key-ordered Rowid-ordered scan Warnings: Note 1276 Field or reference 'test.t3.oref' of SELECT #2 was resolved in SELECT #1 -Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<`test`.`t3`.`a`,`test`.`t3`.`oref`>((`test`.`t3`.`a`,(select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond((`test`.`t3`.`a`) = `test`.`t1`.`a` or `test`.`t1`.`a` is null) and `test`.`t2`.`a` = `test`.`t1`.`b` having trigcond(`test`.`t1`.`a` is null)))) AS `Z` from `test`.`t3` +Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<`test`.`t3`.`a`,`test`.`t3`.`oref`>((`test`.`t3`.`a`,(select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`a` = `test`.`t1`.`b` and `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond((`test`.`t3`.`a`) = `test`.`t1`.`a` or `test`.`t1`.`a` is null) having trigcond(`test`.`t1`.`a` is null)))) AS `Z` from `test`.`t3` drop table t1, t2, t3; create table t1 (a int NOT NULL, b int NOT NULL, key(a)); insert into t1 values @@ -200,7 +200,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 ref a a 4 test.t1.b 1 100.00 Using where; Using join buffer (flat, BKA join); Key-ordered Rowid-ordered scan Warnings: Note 1276 Field or reference 'test.t3.oref' of SELECT #2 was resolved in SELECT #1 -Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<`test`.`t3`.`a`,`test`.`t3`.`oref`>((`test`.`t3`.`a`,(select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond((`test`.`t3`.`a`) = `test`.`t1`.`a`) and `test`.`t2`.`a` = `test`.`t1`.`b`))) AS `Z` from `test`.`t3` +Note 1003 select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<`test`.`t3`.`a`,`test`.`t3`.`oref`>((`test`.`t3`.`a`,(select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`a` = `test`.`t1`.`b` and `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond((`test`.`t3`.`a`) = `test`.`t1`.`a`)))) AS `Z` from `test`.`t3` drop table t1,t2,t3; create table t1 (oref int, grp int); insert into t1 (oref, grp) values diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 2657977dae7..43e698d2b56 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2351,7 +2351,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 system NULL NULL NULL NULL 1 100.00 2 DEPENDENT SUBQUERY t1 ALL NULL NULL NULL NULL 2 100.00 Using where Warnings: -Note 1003 select 3 AS `f` from dual where !<3>((3,(select `test`.`t1`.`b` from `test`.`t1` where (`test`.`t1`.`c` = 'USA' or `test`.`t1`.`c` <> 'USA') and trigcond((3) = `test`.`t1`.`b` or `test`.`t1`.`b` is null) and `test`.`t1`.`b` = `test`.`t1`.`a` having trigcond(`test`.`t1`.`b` is null)))) +Note 1003 select 3 AS `f` from dual where !<3>((3,(select `test`.`t1`.`b` from `test`.`t1` where `test`.`t1`.`b` = `test`.`t1`.`a` and (`test`.`t1`.`c` = 'USA' or `test`.`t1`.`c` <> 'USA') and trigcond((3) = `test`.`t1`.`b` or `test`.`t1`.`b` is null) having trigcond(`test`.`t1`.`b` is null)))) SELECT * FROM t2 WHERE f NOT IN (SELECT b FROM t1 WHERE 0 OR (c IN ('USA') OR c NOT IN ('USA')) AND a = b); @@ -2803,4 +2803,42 @@ FROM (t1 JOIN t1 AS ref_t1 ON (t1.i1 > (SELECT ref_t1.i1 AS c0 FROM t1 b ORDER BY -c0))); ERROR 21000: Subquery returns more than 1 row DROP TABLE t1; +# +# MDEV-22377: Subquery in an UPDATE query uses full scan instead of range +# +CREATE TABLE t1 ( +key1 varchar(30) NOT NULL, +col1 int(11) NOT NULL, +filler char(100) +); +insert into t1 select seq, seq, seq from seq_1_to_100; +CREATE TABLE t10 ( +key1 varchar(30) NOT NULL, +col1 int, +filler char(100), +PRIMARY KEY (key1) +); +insert into t10 select seq, seq, seq from seq_1_to_1000; +CREATE TABLE t11 ( +key1 varchar(30) NOT NULL, +filler char(100), +PRIMARY KEY (key1) +); +insert into t11 select seq, seq from seq_1_to_1000; +set @tmp_os=@@optimizer_switch; +set optimizer_switch='semijoin=off,materialization=off'; +# Must use range access (not full scan) for table tms: +explain select * from t1 hist +WHERE +key1 IN ('1','2','3','4','5','6','7','8','9','10') AND +hist.col1 NOT IN (SELECT tn.col1 +FROM t10 tn JOIN t11 tms ON tms.key1 = tn.key1 +WHERE tn.key1 IN ('1','2','3','4','5','6','7','8','9','10') +); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY hist ALL NULL NULL NULL NULL 100 Using where +2 DEPENDENT SUBQUERY tms range PRIMARY PRIMARY 32 NULL 10 Using where; Using index +2 DEPENDENT SUBQUERY tn eq_ref PRIMARY PRIMARY 32 test.tms.key1 1 Using where +set optimizer_switch=@tmp_os; +drop table t1, t10, t11; # End of 10.2 tests diff --git a/mysql-test/r/subselect_mat_cost_bugs.result b/mysql-test/r/subselect_mat_cost_bugs.result index fd768b1efd4..87cbc1bb078 100644 --- a/mysql-test/r/subselect_mat_cost_bugs.result +++ b/mysql-test/r/subselect_mat_cost_bugs.result @@ -100,7 +100,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 index c3 c3 9 NULL 2 100.00 Using where; Using index; Using join buffer (flat, BNL join) Warnings: Note 1276 Field or reference 'test.t1.pk' of SELECT #2 was resolved in SELECT #1 -Note 1003 select `test`.`t1`.`pk` AS `pk` from `test`.`t1` where <`test`.`t1`.`c1`,`test`.`t1`.`pk`>((`test`.`t1`.`c1`,(select `test`.`t1a`.`c1` from `test`.`t1b` join `test`.`t2` left join `test`.`t1a` on(`test`.`t1a`.`c2` = `test`.`t1b`.`pk` and 2) where `test`.`t1`.`pk` <> 0 and (`test`.`t1`.`c1`) = `test`.`t1a`.`c1` and `test`.`t2`.`c3` = `test`.`t1b`.`c4`))) +Note 1003 select `test`.`t1`.`pk` AS `pk` from `test`.`t1` where <`test`.`t1`.`c1`,`test`.`t1`.`pk`>((`test`.`t1`.`c1`,(select `test`.`t1a`.`c1` from `test`.`t1b` join `test`.`t2` left join `test`.`t1a` on(`test`.`t1a`.`c2` = `test`.`t1b`.`pk` and 2) where `test`.`t2`.`c3` = `test`.`t1b`.`c4` and `test`.`t1`.`pk` <> 0 and (`test`.`t1`.`c1`) = `test`.`t1a`.`c1`))) SELECT pk FROM t1 WHERE c1 IN @@ -363,7 +363,7 @@ AND a = SOME (SELECT b FROM t5)); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t3 ALL NULL NULL NULL NULL 2 Using where 2 DEPENDENT SUBQUERY t5 index c c 10 NULL 2 Using where; Using index; Start temporary -2 DEPENDENT SUBQUERY t4 eq_ref PRIMARY PRIMARY 4 test.t5.b 1 Using index condition; Using where; End temporary +2 DEPENDENT SUBQUERY t4 eq_ref PRIMARY PRIMARY 4 test.t5.b 1 Using where; End temporary SELECT * FROM t3 WHERE t3.b > ALL ( diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index 93389571c5c..1313d2e49e7 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -2308,4 +2308,47 @@ FROM (t1 JOIN t1 AS ref_t1 ON DROP TABLE t1; +--echo # +--echo # MDEV-22377: Subquery in an UPDATE query uses full scan instead of range +--echo # + +CREATE TABLE t1 ( + key1 varchar(30) NOT NULL, + col1 int(11) NOT NULL, + filler char(100) +); +insert into t1 select seq, seq, seq from seq_1_to_100; + +CREATE TABLE t10 ( + key1 varchar(30) NOT NULL, + col1 int, + filler char(100), + PRIMARY KEY (key1) +); +insert into t10 select seq, seq, seq from seq_1_to_1000; + +CREATE TABLE t11 ( + key1 varchar(30) NOT NULL, + filler char(100), + PRIMARY KEY (key1) +); +insert into t11 select seq, seq from seq_1_to_1000; + + +set @tmp_os=@@optimizer_switch; +set optimizer_switch='semijoin=off,materialization=off'; + +--echo # Must use range access (not full scan) for table tms: +explain select * from t1 hist +WHERE + key1 IN ('1','2','3','4','5','6','7','8','9','10') AND + hist.col1 NOT IN (SELECT tn.col1 + FROM t10 tn JOIN t11 tms ON tms.key1 = tn.key1 + WHERE tn.key1 IN ('1','2','3','4','5','6','7','8','9','10') + ); + +set optimizer_switch=@tmp_os; + +drop table t1, t10, t11; + --echo # End of 10.2 tests diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 25621dfe104..92ba085af5b 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -2769,12 +2769,9 @@ bool Item_in_subselect::inject_in_to_exists_cond(JOIN *join_arg) { /* The argument list of the top-level AND may change after fix fields. */ and_args= ((Item_cond*) join_arg->conds)->argument_list(); - List_iterator li(join_arg->cond_equal->current_level); - Item_equal *elem; - while ((elem= li++)) - { - and_args->push_back(elem, thd->mem_root); - } + ((Item_cond_and *) (join_arg->conds))->m_cond_equal= + *join_arg->cond_equal; + and_args->append((List *)&join_arg->cond_equal->current_level); } } From 942a9791b2231273ba20649a658c856641268fae Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Mon, 14 Dec 2020 14:34:14 +0530 Subject: [PATCH 04/11] MDEV-15208: server crashed, when using ORDER BY with window function and UNION SELECTs inside a UNION can have window function but not the global ORDER BY clause of the UNION. --- mysql-test/r/win.result | 8 ++++++++ mysql-test/t/win.test | 10 ++++++++++ sql/sql_select.cc | 4 +++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result index 30650f29555..1ddafbd4f8f 100644 --- a/mysql-test/r/win.result +++ b/mysql-test/r/win.result @@ -4230,5 +4230,13 @@ i LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j) 4 2 DROP TABLE t1; # +# MDEV-15208: server crashed, when using ORDER BY with window function and UNION +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(1),(1),(1),(1),(2),(2),(2),(2),(2),(2); +SELECT 1 UNION SELECT a FROM t1 ORDER BY (row_number() over ()); +ERROR HY000: Expression #1 of ORDER BY contains aggregate function and applies to a UNION +DROP TABLE t1; +# # End of 10.2 tests # diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test index 126ed735c88..37f09a6e850 100644 --- a/mysql-test/t/win.test +++ b/mysql-test/t/win.test @@ -2730,6 +2730,16 @@ INSERT INTO t1 VALUES (1,1), (1,5),(1,4), (2,2),(2,5), (3,3),(4,4); SELECT i, LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j) FROM t1 GROUP BY i; DROP TABLE t1; +--echo # +--echo # MDEV-15208: server crashed, when using ORDER BY with window function and UNION +--echo # + +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(1),(1),(1),(1),(2),(2),(2),(2),(2),(2); +--error ER_AGGREGATE_ORDER_FOR_UNION +SELECT 1 UNION SELECT a FROM t1 ORDER BY (row_number() over ()); +DROP TABLE t1; + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index db8a63eeb48..a3e8b453897 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -22796,7 +22796,9 @@ int setup_order(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables, an ORDER BY clause */ - if (for_union && (*order->item)->with_sum_func) + if (for_union && + ((*order->item)->with_sum_func || + (*order->item)->with_window_func)) { my_error(ER_AGGREGATE_ORDER_FOR_UNION, MYF(0), number); return 1; From 647e952315740286df3cf6643732ad8deb1ff7fe Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Mon, 14 Dec 2020 14:34:14 +0530 Subject: [PATCH 05/11] MDEV-15208: server crashed, when using ORDER BY with window function and UNION (Edits by SergeiP: fix encryption.tempfiles_encrypted, re-word comment) Global ORDER BY clause of a UNION may not refer to 1) aggregate functions or 2) window functions. setup_order() checked for #1 but not for #2. --- mysql-test/suite/encryption/r/tempfiles_encrypted.result | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mysql-test/suite/encryption/r/tempfiles_encrypted.result b/mysql-test/suite/encryption/r/tempfiles_encrypted.result index aaba6bb044a..f7b5081f625 100644 --- a/mysql-test/suite/encryption/r/tempfiles_encrypted.result +++ b/mysql-test/suite/encryption/r/tempfiles_encrypted.result @@ -4236,6 +4236,14 @@ i LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j) 4 2 DROP TABLE t1; # +# MDEV-15208: server crashed, when using ORDER BY with window function and UNION +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(1),(1),(1),(1),(2),(2),(2),(2),(2),(2); +SELECT 1 UNION SELECT a FROM t1 ORDER BY (row_number() over ()); +ERROR HY000: Expression #1 of ORDER BY contains aggregate function and applies to a UNION +DROP TABLE t1; +# # End of 10.2 tests # # From c753b3f6f855721ea0760b33c1d760d35776748d Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 18 Jan 2022 13:00:31 +0100 Subject: [PATCH 06/11] MDEV-27534: mariabackup: missing line in the compress_qpress.result This commit adds the missing line to the .result file for compress_qpress (in the mariabackup test suite). --- mysql-test/suite/mariabackup/compress_qpress.result | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/mariabackup/compress_qpress.result b/mysql-test/suite/mariabackup/compress_qpress.result index f8dfb46e4db..69427f5fc4d 100644 --- a/mysql-test/suite/mariabackup/compress_qpress.result +++ b/mysql-test/suite/mariabackup/compress_qpress.result @@ -3,6 +3,7 @@ INSERT INTO t VALUES(1); # xtrabackup backup INSERT INTO t VALUES(2); # xtrabackup prepare +db.opt.qp t.frm.qp t.ibd.qp # shutdown server From 571eb9d7759a198e57f1efd03e50e0b15c627631 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 18 Jan 2022 19:07:01 +0100 Subject: [PATCH 07/11] mariabackup: cosmetic changes (whitespaces and indentation) --- extra/mariabackup/xtrabackup.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 12cfdb6c284..bb1e2eaf42d 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1503,14 +1503,14 @@ struct my_option xb_server_options[] = (G_PTR*)&opt_encrypted_backup, 0, GET_BOOL, NO_ARG, TRUE, 0, 0, 0, 0, 0}, - {"log", OPT_LOG, "Ignored option for MySQL option compatibility", + {"log", OPT_LOG, "Ignored option for MySQL option compatibility", (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, - {"log_bin", OPT_LOG, "Base name for the log sequence", + {"log_bin", OPT_LOG, "Base name for the log sequence", &opt_log_bin, &opt_log_bin, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, - {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility", + {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility", (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, #ifdef BTR_CUR_HASH_ADAPT @@ -1635,10 +1635,10 @@ struct my_option xb_server_options[] = 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"plugin-dir", OPT_PLUGIN_DIR, - "Server plugin directory. Used to load encryption plugin during 'prepare' phase." - "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", - &xb_plugin_dir, &xb_plugin_dir, - 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + "Server plugin directory. Used to load encryption plugin during 'prepare' phase." + "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", + &xb_plugin_dir, &xb_plugin_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, {"innodb-log-checksums", OPT_INNODB_LOG_CHECKSUMS, "Whether to require checksums for InnoDB redo log blocks", @@ -1660,12 +1660,12 @@ struct my_option xb_server_options[] = &xb_rocksdb_datadir, &xb_rocksdb_datadir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, - { "rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed." + {"rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed." "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data", &xb_backup_rocksdb, &xb_backup_rocksdb, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, - {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user " + {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user " "privileges fro the backup user", &opt_check_privileges, &opt_check_privileges, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, From 17e0f5224c8339ec08707a6ad0397bbf8c19bbd3 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 22 Feb 2022 10:45:06 +0100 Subject: [PATCH 08/11] MDEV-27524: Incorrect binlogs after Galera SST using rsync and mariabackup This commit adds correct handling of binlogs for SST using rsync or mariabackup. Before this fix, binlogs were handled incorrectly - - only one (last) binary log file was transferred during SST, which then led to various failures (for example, when trying to list all events from the binary log). These bugs were long masked by flaws in the primitive binlogs handling code in the SST scripts, which causing binary logs files to be erased after transfer or not added to the binlog index on the joiner node. Now the correct transfer of all binary logs (not just the last of the binary log files) has been implemented both for the rsync (at the script level) and for the mariabackup (at the level of the main utility code). This commit also adds a new sst_max_binlogs= parameter, which can be located in the [sst] section or in the [xtrabackup] section (historically, supported for mariabackup only, not for rsync), or in one of the server sections. This parameter specifies the number of binary log files to be sent to the joiner node during SST. This option is added for compatibility with old SST scripting behavior, which can be emulated by setting the sst_max_binlogs=1 (although in general this can cause problems for the reasons described above). In addition, setting the sst_max_binlogs=0 can be used to suppress the transmission of binary logs to the joiner nodes during SST (although sometimes a single file with the current binary log can still be transmitted to the joiner, even with sst_max_binlogs=0, because this sometimes necessary in modes that involve the use of GTIDs with Galera). Also, this commit ensures correct handling of paths to various innodb files and directories in the SST scripts, and fixes some problems with this that existed in mariabackup utility (which were associated with incorrect handling of the innodb_data_dir parameter in some scenarios). In addition, this commit contains the following enhancements: 1) Added tests for mtr, which check the correct work with binlogs after SST (using rsync and mariabackup); 2) Added correct handling of slashes at the end of all paths that the SST script receives as parameters; 3) Improved parsing code for --mysqld-args parameters. Now it correctly processes the sequence "--" after the name of the one-letter option; 4) Checking the secret signature during joiner authentication is made independent of presence of bash (as a unix shell) in the system and diff utility no longer needed to check certificates compliance; 5) All directories that are necessary for the correct placement of various logs are automatically created by SST scripts in advance (before running mariabackup on the joiner node); 6) Removal of old binary logs on joiner is done using the binlog index (if it exists) (not only by fixed pattern that based on the current binlog name, as before); 7) Paths for placing binary logs are correctly processed if they are set as relative paths (to the datadir); 8) SST scripts are made even more resistant to spaces in filenames (now for binlogs); 9) In case of failure, SST scripts now always end with an exit code other than zero; 10) SST script for rsync now correctly create a tar file with the binlogs, even if the paths to them (in the binlog index file) are specified as a mix of absolute and relative paths, and even if they do not match with the datadir path specified in the current configuration settings. --- extra/mariabackup/backup_copy.cc | 84 ++-- extra/mariabackup/backup_copy.h | 7 + extra/mariabackup/backup_mysql.cc | 248 ++++++++--- extra/mariabackup/backup_mysql.h | 6 +- extra/mariabackup/common.h | 10 + extra/mariabackup/xtrabackup.cc | 62 ++- extra/mariabackup/xtrabackup.h | 2 + .../galera/include/galera_wsrep_recover.inc | 7 + .../suite/galera/r/galera_log_bin_ext.result | 9 + .../r/galera_log_bin_ext_mariabackup.result | 82 ++++ .../galera/r/galera_sst_rsync,debug.rdiff | 4 +- .../galera/r/galera_sst_rsync2,debug.rdiff | 4 +- .../r/galera_sst_xtrabackup-v2,debug.rdiff | 4 +- ...era_sst_xtrabackup-v2_data_dir,debug.rdiff | 19 +- .../galera_sst_xtrabackup-v2_data_dir.result | 2 + .../galera/r/galera_wan_restart_sst.result | 11 +- .../suite/galera/t/galera_log_bin_ext.cnf | 3 + .../suite/galera/t/galera_log_bin_ext.test | 2 +- .../t/galera_log_bin_ext_mariabackup.cnf | 19 + .../t/galera_log_bin_ext_mariabackup.test | 2 + .../suite/galera/t/galera_log_bin_sst.inc | 84 ++++ .../suite/galera/t/galera_sst_rsync.test | 1 + .../galera/t/galera_sst_rsync_logbasename.cnf | 1 - .../galera/t/galera_wan_restart_sst.test | 19 +- scripts/wsrep_sst_common.sh | 230 ++++++---- scripts/wsrep_sst_mariabackup.sh | 238 ++++++---- scripts/wsrep_sst_mysqldump.sh | 7 +- scripts/wsrep_sst_rsync.sh | 419 +++++++++++------- scripts/wsrep_sst_xtrabackup-v2.sh | 242 ++++++---- 29 files changed, 1310 insertions(+), 518 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result create mode 100644 mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf create mode 100644 mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test create mode 100644 mysql-test/suite/galera/t/galera_log_bin_sst.inc diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index c46c32213d6..9c6f3665a6a 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -582,7 +582,6 @@ datafile_read(datafile_cur_t *cursor) Check to see if a file exists. Takes name of the file to check. @return true if file exists. */ -static bool file_exists(const char *filename) { @@ -1547,13 +1546,14 @@ bool backup_start(CorruptedPages &corrupted_pages) if (!write_galera_info(mysql_connection)) { return(false); } - write_current_binlog_file(mysql_connection); } - if (opt_binlog_info == BINLOG_INFO_ON) { + bool with_binlogs = opt_binlog_info == BINLOG_INFO_ON; - lock_binlog_maybe(mysql_connection); - write_binlog_info(mysql_connection); + if (with_binlogs || opt_galera_info) { + if (!write_current_binlog_file(mysql_connection, with_binlogs)) { + return(false); + } } if (have_flush_engine_logs && !opt_no_lock) { @@ -1587,15 +1587,34 @@ void backup_release() } } +static const char *default_buffer_pool_file = "ib_buffer_pool"; + +static +const char * get_buffer_pool_filename(size_t *length) +{ + /* If mariabackup is run for Galera, then the file + name is changed to the default so that the receiving + node can find this file and rename it according to its + settings, otherwise we keep the original file name: */ + size_t dir_length = 0; + const char *dst_name = default_buffer_pool_file; + if (!opt_galera_info) { + dir_length = dirname_length(buffer_pool_filename); + dst_name = buffer_pool_filename + dir_length; + } + if (length) { + *length=dir_length; + } + return dst_name; +} + /** Finish after backup_start() and backup_release() */ bool backup_finish() { /* Copy buffer pool dump or LRU dump */ if (!opt_rsync) { if (buffer_pool_filename && file_exists(buffer_pool_filename)) { - const char *dst_name; - - dst_name = trim_dotslash(buffer_pool_filename); + const char *dst_name = get_buffer_pool_filename(NULL); copy_file(ds_data, buffer_pool_filename, dst_name, 0); } if (file_exists("ib_lru_dump")) { @@ -1684,17 +1703,14 @@ ibx_copy_incremental_over_full() /* copy buffer pool dump */ if (innobase_buffer_pool_filename) { - const char *src_name; - - src_name = trim_dotslash(innobase_buffer_pool_filename); + const char *src_name = get_buffer_pool_filename(NULL); snprintf(path, sizeof(path), "%s/%s", xtrabackup_incremental_dir, src_name); if (file_exists(path)) { - copy_file(ds_data, path, - innobase_buffer_pool_filename, 0); + copy_file(ds_data, path, src_name, 0); } } @@ -1929,6 +1945,14 @@ copy_back() datadir_node_init(&node); + /* If mariabackup is run for Galera, then the file + name is changed to the default so that the receiving + node can find this file and rename it according to its + settings, otherwise we keep the original file name: */ + size_t dir_length; + const char *src_buffer_pool; + src_buffer_pool = get_buffer_pool_filename(&dir_length); + while (datadir_iter_next(it, &node)) { const char *ext_list[] = {"backup-my.cnf", "xtrabackup_binary", "xtrabackup_binlog_info", @@ -1991,6 +2015,11 @@ copy_back() continue; } + /* skip buffer pool dump */ + if (!strcmp(filename, src_buffer_pool)) { + continue; + } + /* skip innodb data files */ is_ibdata_file = false; for (Tablespace::const_iterator iter(srv_sys_space.begin()), @@ -2013,23 +2042,18 @@ copy_back() /* copy buffer pool dump */ - if (innobase_buffer_pool_filename) { - const char *src_name; - char path[FN_REFLEN]; - - src_name = trim_dotslash(innobase_buffer_pool_filename); - - snprintf(path, sizeof(path), "%s/%s", - mysql_data_home, - src_name); - - /* could be already copied with other files - from data directory */ - if (file_exists(src_name) && - !file_exists(innobase_buffer_pool_filename)) { - copy_or_move_file(src_name, - innobase_buffer_pool_filename, - mysql_data_home, 0); + if (file_exists(src_buffer_pool)) { + char dst_dir[FN_REFLEN]; + while (IS_TRAILING_SLASH(buffer_pool_filename, dir_length)) { + dir_length--; + } + memcpy(dst_dir, buffer_pool_filename, dir_length); + dst_dir[dir_length] = 0; + if (!(ret = copy_or_move_file(src_buffer_pool, + src_buffer_pool, + dst_dir, 1))) + { + goto cleanup; } } diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index 62b2b1bc232..858182001ce 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -32,6 +32,13 @@ copy_file(ds_ctxt_t *datasink, const char *dst_file_path, uint thread_n); +/************************************************************************ +Check to see if a file exists. +Takes name of the file to check. +@return true if file exists. */ +bool +file_exists(const char *filename); + /** Start --backup */ bool backup_start(CorruptedPages &corrupted_pages); /** Release resources after backup_start() */ diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 824b8b6d5f3..78af2b7f5ac 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -83,7 +83,6 @@ os_event_t kill_query_thread_stop; bool sql_thread_started = false; char *mysql_slave_position = NULL; char *mysql_binlog_position = NULL; -char *buffer_pool_filename = NULL; /* History on server */ time_t history_start_time; @@ -1333,27 +1332,29 @@ cleanup: } +static +bool +write_binlog_info(MYSQL *connection, char *log_bin_dir, + MYSQL_RES *mysql_result, my_ulonglong n_rows, + my_ulonglong start); + /*********************************************************************//** Flush and copy the current binary log file into the backup, if GTID is enabled */ bool -write_current_binlog_file(MYSQL *connection) +write_current_binlog_file(MYSQL *connection, bool write_binlogs) { + char *log_bin = NULL; + char *filename = NULL; + char *position = NULL; char *executed_gtid_set = NULL; char *gtid_binlog_state = NULL; - char *log_bin_file = NULL; char *log_bin_dir = NULL; bool gtid_exists; bool result = true; - char filepath[FN_REFLEN]; - mysql_variable status[] = { - {"Executed_Gtid_Set", &executed_gtid_set}, - {NULL, NULL} - }; - - mysql_variable status_after_flush[] = { - {"File", &log_bin_file}, + mysql_variable log_bin_var[] = { + {"@@GLOBAL.log_bin", &log_bin}, {NULL, NULL} }; @@ -1363,21 +1364,36 @@ write_current_binlog_file(MYSQL *connection) {NULL, NULL} }; + mysql_variable status[] = { + {"File", &filename}, + {"Position", &position}, + {"Executed_Gtid_Set", &executed_gtid_set}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SELECT @@GLOBAL.log_bin", log_bin_var, false); + + /* Do not create xtrabackup_binlog_info if binary log is disabled: */ + if (strncmp(log_bin, "1", 2) != 0) { + goto binlog_disabled; + } + + lock_binlog_maybe(connection); + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + + /* Do not create xtrabackup_binlog_info if replication + has not started yet: */ + if (filename == NULL || position == NULL) { + goto no_replication; + } + read_mysql_variables(connection, "SHOW VARIABLES", vars, true); gtid_exists = (executed_gtid_set && *executed_gtid_set) || (gtid_binlog_state && *gtid_binlog_state); - if (gtid_exists) { - size_t log_bin_dir_length; - - lock_binlog_maybe(connection); - - xb_mysql_query(connection, "FLUSH BINARY LOGS", false); - - read_mysql_variables(connection, "SHOW MASTER STATUS", - status_after_flush, false); + if (write_binlogs || gtid_exists) { if (opt_log_bin != NULL && strchr(opt_log_bin, FN_LIBCHAR)) { /* If log_bin is set, it has priority */ @@ -1387,34 +1403,89 @@ write_current_binlog_file(MYSQL *connection) log_bin_dir = strdup(opt_log_bin); } else if (log_bin_dir == NULL) { /* Default location is MySQL datadir */ - log_bin_dir = strdup("./"); + log_bin_dir = static_cast(malloc(3)); + ut_a(log_bin_dir); + log_bin_dir[0] = '.'; + log_bin_dir[1] = FN_LIBCHAR; + log_bin_dir[2] = 0; } + size_t log_bin_dir_length; + dirname_part(log_bin_dir, log_bin_dir, &log_bin_dir_length); /* strip final slash if it is not the only path component */ - if (log_bin_dir_length > 1 && - log_bin_dir[log_bin_dir_length - 1] == FN_LIBCHAR) { - log_bin_dir[log_bin_dir_length - 1] = 0; + while (IS_TRAILING_SLASH(log_bin_dir, log_bin_dir_length)) { + log_bin_dir_length--; } + log_bin_dir[log_bin_dir_length] = 0; - if (log_bin_dir == NULL || log_bin_file == NULL) { - msg("Failed to get master binlog coordinates from " - "SHOW MASTER STATUS"); + if (log_bin_dir == NULL) { + msg("Failed to locate binary log files"); result = false; goto cleanup; } - snprintf(filepath, sizeof(filepath), "%s%c%s", - log_bin_dir, FN_LIBCHAR, log_bin_file); - result = copy_file(ds_data, filepath, log_bin_file, 0); + uint max_binlogs; + max_binlogs = opt_max_binlogs; + if (max_binlogs == 0) { + if (gtid_exists) { + max_binlogs = 1; + } else { + goto cleanup; + } + } + + xb_mysql_query(connection, "FLUSH BINARY LOGS", false); + + MYSQL_RES *mysql_result; + + mysql_result = xb_mysql_query(connection, "SHOW BINARY LOGS", true); + + ut_ad(mysql_num_fields(mysql_result) >= 2); + + my_ulonglong n_rows; + my_ulonglong start; + + n_rows = mysql_num_rows(mysql_result); + + start = 0; + if (max_binlogs < n_rows) { + start = n_rows - max_binlogs; + } + if (start) { + mysql_data_seek(mysql_result, start); + } + + MYSQL_ROW row; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + char filepath[FN_REFLEN]; + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, binlog_name); + if (file_exists(filepath)) { + result = copy_file(ds_data, filepath, binlog_name, 0); + if (!result) break; + } + } + + if (result) { + write_binlog_info(connection, log_bin_dir, + mysql_result, n_rows, start); + } + + mysql_free_result(mysql_result); } cleanup: - free_mysql_variables(status_after_flush); - free_mysql_variables(status); free_mysql_variables(vars); +no_replication: + free_mysql_variables(status); + +binlog_disabled: + free_mysql_variables(log_bin_var); + return(result); } @@ -1422,8 +1493,11 @@ cleanup: /*********************************************************************//** Retrieves MySQL binlog position and saves it in a file. It also prints it to stdout. */ +static bool -write_binlog_info(MYSQL *connection) +write_binlog_info(MYSQL *connection, char *log_bin_dir, + MYSQL_RES *mysql_result, my_ulonglong n_rows, + my_ulonglong start) { char *filename = NULL; char *position = NULL; @@ -1431,9 +1505,13 @@ write_binlog_info(MYSQL *connection) char *gtid_current_pos = NULL; char *gtid_executed = NULL; char *gtid = NULL; - bool result; + char *buffer; + char *buf; + size_t total; + bool result = true; bool mysql_gtid; bool mariadb_gtid; + bool with_gtid; mysql_variable status[] = { {"File", &filename}, @@ -1451,39 +1529,106 @@ write_binlog_info(MYSQL *connection) read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); read_mysql_variables(connection, "SHOW VARIABLES", vars, true); - if (filename == NULL || position == NULL) { - /* Do not create xtrabackup_binlog_info if binary - log is disabled */ - result = true; - goto cleanup; - } + mysql_gtid = gtid_mode && (strcmp(gtid_mode, "ON") == 0); + mariadb_gtid = gtid_current_pos && *gtid_current_pos; - mysql_gtid = ((gtid_mode != NULL) && (strcmp(gtid_mode, "ON") == 0)); - mariadb_gtid = (gtid_current_pos != NULL); + gtid = (gtid_executed && *gtid_executed) ? gtid_executed : gtid_current_pos; - gtid = (gtid_executed != NULL ? gtid_executed : gtid_current_pos); - - if (mariadb_gtid || mysql_gtid) { + with_gtid = mariadb_gtid || mysql_gtid; + if (with_gtid) { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s', " "GTID of the last change '%s'", filename, position, gtid) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, - "%s\t%s\t%s\n", filename, position, - gtid); } else { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s'", filename, position) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, - "%s\t%s\n", filename, position); } + mysql_data_seek(mysql_result, start); + + MYSQL_ROW row; + my_ulonglong current; + + total = 1; + current = start; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + /* The position in the current binlog is taken from + the global variable, but for the previous ones it is + determined by their length: */ + const char *binlog_pos = + ++current == n_rows ? position : row[1]; + total += strlen(binlog_name) + strlen(binlog_pos) + 2; + if (with_gtid && current != n_rows) { + /* Add the "\t[]" length to the buffer size: */ + total += 3; + } + } + /* For the last of the binray log files, also add + the length of the GTID (+ one character for '\t'): */ + if (with_gtid) { + total += strlen(gtid) + 1; + } + + buffer = static_cast(malloc(total)); + if (!buffer) { + msg("Failed to allocate memory for temporary buffer"); + result = false; + goto cleanup; + } + + mysql_data_seek(mysql_result, start); + + buf = buffer; + current = start; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + char filepath[FN_REFLEN]; + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, binlog_name); + current++; + if (file_exists(filepath)) { + /* The position in the current binlog is taken from + the global variable, but for the previous ones it is + determined by their length: */ + char *binlog_pos = + current == n_rows ? position : row[1]; + int bytes; + if (with_gtid) { + bytes = snprintf(buf, total, "%s\t%s\t%s\n", + binlog_name, binlog_pos, + current == n_rows ? gtid : "[]"); + } else { + bytes = snprintf(buf, total, "%s\t%s\n", + binlog_name, binlog_pos); + } + if (bytes <= 0) { + goto buffer_overflow; + } + buf += bytes; + total -= bytes; + } + } + + if (buf != buffer) { + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, "%s", buffer); + } + +cleanup2: + free(buffer); + cleanup: - free_mysql_variables(status); free_mysql_variables(vars); + free_mysql_variables(status); return(result); + +buffer_overflow: + msg("Internal error: buffer overflow in the write_binlog_info()"); + result = false; + goto cleanup2; } struct escape_and_quote @@ -1811,7 +1956,6 @@ backup_cleanup() { free(mysql_slave_position); free(mysql_binlog_position); - free(buffer_pool_filename); if (mysql_connection) { mysql_close(mysql_connection); diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index b61fa2362c6..5e78281e1cb 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -28,7 +28,6 @@ extern time_t history_lock_time; extern bool sql_thread_started; extern char *mysql_slave_position; extern char *mysql_binlog_position; -extern char *buffer_pool_filename; /** connection to mysql server */ extern MYSQL *mysql_connection; @@ -62,10 +61,7 @@ void unlock_all(MYSQL *connection); bool -write_current_binlog_file(MYSQL *connection); - -bool -write_binlog_info(MYSQL *connection); +write_current_binlog_file(MYSQL *connection, bool write_binlogs); bool write_xtrabackup_info(MYSQL *connection, const char * filename, bool history, diff --git a/extra/mariabackup/common.h b/extra/mariabackup/common.h index 1973512ad82..beb49524608 100644 --- a/extra/mariabackup/common.h +++ b/extra/mariabackup/common.h @@ -187,4 +187,14 @@ xb_read_full(File fd, uchar *buf, size_t len) return tlen; } +#ifdef _WIN32 +#define IS_TRAILING_SLASH(name, length) \ + ((length) > 1 && \ + (name[(length) - 1] == '/' || \ + name[(length) - 1] == '\\')) +#else +#define IS_TRAILING_SLASH(name, length) \ + ((length) > 1 && name[(length) - 1] == FN_LIBCHAR) +#endif + #endif diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index bb1e2eaf42d..c1b2a1a9c85 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -240,7 +240,8 @@ long innobase_log_buffer_size = 1024*1024L; long innobase_open_files = 300L; longlong innobase_page_size = (1LL << 14); /* 16KB */ -char* innobase_buffer_pool_filename = NULL; +char *innobase_buffer_pool_filename = NULL; +char *buffer_pool_filename = NULL; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ @@ -347,6 +348,7 @@ uint opt_lock_wait_timeout = 0; uint opt_lock_wait_threshold = 0; uint opt_debug_sleep_before_unlock = 0; uint opt_safe_slave_backup_timeout = 0; +uint opt_max_binlogs = UINT_MAX; const char *opt_history = NULL; @@ -1051,7 +1053,8 @@ enum options_xtrabackup OPT_BACKUP_ROCKSDB, OPT_XTRA_CHECK_PRIVILEGES, OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION, - OPT_INNODB_FORCE_RECOVERY + OPT_INNODB_FORCE_RECOVERY, + OPT_MAX_BINLOGS }; struct my_option xb_client_options[]= { @@ -1454,6 +1457,17 @@ struct my_option xb_client_options[]= { &opt_log_innodb_page_corruption, &opt_log_innodb_page_corruption, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"sst_max_binlogs", OPT_MAX_BINLOGS, + "Number of recent binary logs to be included in the backup. " + "Setting this parameter to zero normally disables transmission " + "of binary logs to the joiner nodes during SST using Galera. " + "But sometimes a single current binlog can still be transmitted " + "to the joiner even with sst_max_binlogs=0, because it is " + "required for Galera to work properly with GTIDs support.", + (G_PTR *) &opt_max_binlogs, + (G_PTR *) &opt_max_binlogs, 0, GET_UINT, OPT_ARG, + UINT_MAX, 0, UINT_MAX, 0, 1, 0}, + #define MYSQL_CLIENT #include "sslopt-longopts.h" #undef MYSQL_CLIENT @@ -6272,6 +6286,44 @@ check_all_privileges() } } +static +void +xb_init_buffer_pool(const char * filename) +{ + if (filename && +#ifdef _WIN32 + (filename[0] == '/' || + filename[0] == '\\' || + strchr(filename, ':'))) +#else + filename[0] == FN_LIBCHAR) +#endif + { + buffer_pool_filename = strdup(filename); + } else { + char filepath[FN_REFLEN]; + char *dst_dir = + (innobase_data_home_dir && *innobase_data_home_dir) ? + innobase_data_home_dir : mysql_data_home; + size_t dir_length; + if (dst_dir && *dst_dir) { + dir_length = strlen(dst_dir); + while (IS_TRAILING_SLASH(dst_dir, dir_length)) { + dir_length--; + } + memcpy(filepath, dst_dir, dir_length); + } + else { + filepath[0] = '.'; + dir_length = 1; + } + snprintf(filepath + dir_length, + sizeof(filepath) - dir_length, "%c%s", FN_LIBCHAR, + filename ? filename : "ib_buffer_pool"); + buffer_pool_filename = strdup(filepath); + } +} + bool xb_init() { @@ -6336,11 +6388,15 @@ xb_init() if (!get_mysql_vars(mysql_connection)) { return(false); } + xb_init_buffer_pool(buffer_pool_filename); + if (opt_check_privileges) { check_all_privileges(); } history_start_time = time(NULL); + } else { + xb_init_buffer_pool(innobase_buffer_pool_filename); } return(true); @@ -6634,6 +6690,8 @@ int main(int argc, char **argv) free_error_messages(); mysql_mutex_destroy(&LOCK_error_log); + free(buffer_pool_filename); + if (status == EXIT_SUCCESS) { msg("completed OK!"); } diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index e2955f58d6a..dbcd0d83f9b 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -69,6 +69,7 @@ extern char *xtrabackup_incremental_dir; extern char *xtrabackup_incremental_basedir; extern char *innobase_data_home_dir; extern char *innobase_buffer_pool_filename; +extern char *buffer_pool_filename; extern char *xb_plugin_dir; extern char *xb_rocksdb_datadir; extern my_bool xb_backup_rocksdb; @@ -164,6 +165,7 @@ extern uint opt_lock_wait_timeout; extern uint opt_lock_wait_threshold; extern uint opt_debug_sleep_before_unlock; extern uint opt_safe_slave_backup_timeout; +extern uint opt_max_binlogs; extern const char *opt_history; diff --git a/mysql-test/suite/galera/include/galera_wsrep_recover.inc b/mysql-test/suite/galera/include/galera_wsrep_recover.inc index d2956ea99e6..aa2f0e2e777 100644 --- a/mysql-test/suite/galera/include/galera_wsrep_recover.inc +++ b/mysql-test/suite/galera/include/galera_wsrep_recover.inc @@ -1,5 +1,12 @@ --echo Performing --wsrep-recover ... +if ($wsrep_recover_additional) +{ +--exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover $wsrep_recover_additional > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 +} +if (!$wsrep_recover_additional) +{ --exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 +} --perl use strict; diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext.result b/mysql-test/suite/galera/r/galera_log_bin_ext.result index f5276b7d1ac..c5ff66d17b3 100644 --- a/mysql-test/suite/galera/r/galera_log_bin_ext.result +++ b/mysql-test/suite/galera/r/galera_log_bin_ext.result @@ -1,4 +1,6 @@ connection node_1; +connection node_2; +connection node_1; reset master; connection node_2; reset master; @@ -40,6 +42,12 @@ hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; COUNT(*) = 2 1 @@ -66,6 +74,7 @@ hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 DROP TABLE t1; DROP TABLE t2; #cleanup diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result new file mode 100644 index 00000000000..c5ff66d17b3 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result @@ -0,0 +1,82 @@ +connection node_1; +connection node_2; +connection node_1; +reset master; +connection node_2; +reset master; +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); +connection node_2; +SELECT COUNT(*) = 1 FROM t1; +COUNT(*) = 1 +1 +SELECT COUNT(*) = 2 FROM t2; +COUNT(*) = 2 +1 +connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 +DROP TABLE t1; +DROP TABLE t2; +#cleanup +connection node_1; +RESET MASTER; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff index 94dd8c2e502..7b10cd23b60 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff @@ -1,6 +1,6 @@ --- galera_sst_rsync.result -+++ galera_sst_rsync,debug.reject -@@ -284,3 +284,111 @@ ++++ galera_sst_rsync.reject +@@ -286,3 +286,111 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff index 525156d88da..23ed9862ea4 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff @@ -1,5 +1,5 @@ ---- suite/galera/r/galera_sst_rsync2.result 2018-09-12 13:09:35.352229478 +0200 -+++ suite/galera/r/galera_sst_rsync2,debug.reject 2018-09-12 17:00:51.601974979 +0200 +--- suite/galera/r/galera_sst_rsync2.result ++++ suite/galera/r/galera_sst_rsync2.reject @@ -286,3 +286,111 @@ DROP TABLE t1; COMMIT; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff index 8b091eb370a..412f8996f15 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff @@ -1,5 +1,5 @@ ---- galera_sst_mariabackup.result -+++ galera_sst_mariabackup,debug.reject +--- galera_sst_xtrabackup-v2.result ++++ galera_sst_xtrabackup-v2.reject @@ -286,5 +286,113 @@ DROP TABLE t1; COMMIT; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff index ac232020037..1cb0725177b 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff @@ -1,11 +1,12 @@ ---- r/galera_sst_xtrabackup-v2_data_dir.result 2018-11-19 12:27:24.795221479 +0200 -+++ r/galera_sst_xtrabackup-v2_data_dir.reject 2018-11-19 19:15:38.774008404 +0200 -@@ -260,3 +260,100 @@ +--- galera_sst_xtrabackup-v2_data_dir.result ++++ galera_sst_xtrabackup-v2_data_dir.reject +@@ -286,5 +286,113 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +while a DDL was in progress on it ++connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; @@ -14,6 +15,7 @@ +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); ++connection node_2; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); @@ -22,9 +24,12 @@ +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; ++connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; ++connection node_2; +SET wsrep_sync_wait = 0; +Killing server ... ++connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); @@ -39,6 +44,7 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); ++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -46,7 +52,9 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); ++connection node_2; +Performing --wsrep-recover ... ++connection node_2; +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; @@ -57,6 +65,7 @@ +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +COMMIT; ++connection node_1; +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); @@ -71,6 +80,7 @@ +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +COMMIT; ++connection node_1a_galera_st_kill_slave_ddl; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -88,6 +98,7 @@ +1 +COMMIT; +SET AUTOCOMMIT=ON; ++connection node_1; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 @@ -101,3 +112,5 @@ +COMMIT; +SET AUTOCOMMIT=ON; +SET GLOBAL debug_dbug = $debug_orig; + disconnect node_2; + disconnect node_1; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result index ff85a7d6c0f..fdb5883b590 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result @@ -286,3 +286,5 @@ COUNT(*) = 0 DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +disconnect node_2; +disconnect node_1; diff --git a/mysql-test/suite/galera/r/galera_wan_restart_sst.result b/mysql-test/suite/galera/r/galera_wan_restart_sst.result index 1adcbfd1d50..2744c91a06b 100644 --- a/mysql-test/suite/galera/r/galera_wan_restart_sst.result +++ b/mysql-test/suite/galera/r/galera_wan_restart_sst.result @@ -1,3 +1,9 @@ +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4; +connection node_1; +connection node_2; +connection node_3; +connection node_4; SELECT VARIABLE_VALUE AS EXPECT_4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; EXPECT_4 4 @@ -6,10 +12,8 @@ CREATE TABLE t1 (f1 INTEGER); INSERT INTO t1 VALUES (1); connection node_2; INSERT INTO t1 VALUES (2); -connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; connection node_3; INSERT INTO t1 VALUES (3); -connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4; connection node_4; INSERT INTO t1 VALUES (4); connection node_3; @@ -81,3 +85,6 @@ CALL mtr.add_suppression("There are no nodes in the same segment that will ever CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); connection node_4; CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); +connection node_1; +disconnect node_3; +disconnect node_4; diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf index 012209610ea..ae47de90bb8 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf @@ -9,3 +9,6 @@ log-slave-updates log-bin = hostname2-bin log-bin-index = hostname2.bdx log-slave-updates + +[sst] +sst_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.test b/mysql-test/suite/galera/t/galera_log_bin_ext.test index 923bd623a8a..752073aecdb 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.test +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.test @@ -1 +1 @@ ---source galera_log_bin.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf new file mode 100644 index 00000000000..c988136b8fb --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf @@ -0,0 +1,19 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=mariabackup +wsrep_sst_auth="root:" + +[mysqld.1] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.1/data/hostname1-bin +log-bin-index = hostname1.bdx +log-slave-updates + +[mysqld.2] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.2/data/hostname2-bin +log-bin-index = hostname2.bdx +log-slave-updates + +[sst] +transferfmt=@ENV.MTR_GALERA_TFMT +sst_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test new file mode 100644 index 00000000000..47df45b4c71 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test @@ -0,0 +1,2 @@ +--source include/have_mariabackup.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_sst.inc b/mysql-test/suite/galera/t/galera_log_bin_sst.inc new file mode 100644 index 00000000000..5d543f6f8b6 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_sst.inc @@ -0,0 +1,84 @@ +--source include/galera_cluster.inc +--source include/force_restart.inc + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_1 +reset master; +--connection node_2 +reset master; + +# +# Test Galera with --log-bin --log-slave-updates . +# This way the actual MySQL binary log is used, +# rather than Galera's own implementation +# + +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); + +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); + +--connection node_2 +SELECT COUNT(*) = 1 FROM t1; +SELECT COUNT(*) = 2 FROM t2; + +--connection node_1 +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +--let $MASTER_MYPORT=$NODE_MYPORT_1 +--source include/show_binlog_events.inc + +--connection node_2 + +#--connection node_2 +#--source suite/galera/include/galera_stop_replication.inc + +--echo Shutting down server ... +--source include/shutdown_mysqld.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +# +# Force SST +# +--echo Cleaning var directory ... +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mtr +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/performance_schema +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/test +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mysql +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data + +--connection node_2 + +--echo Starting server ... +let $restart_noprint=2; +--source include/start_mysqld.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +--connection node_2 +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +--let $MASTER_MYPORT=$NODE_MYPORT_2 +--source include/show_binlog_events.inc + +DROP TABLE t1; +DROP TABLE t2; + +--echo #cleanup +--connection node_1 +RESET MASTER; + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync.test b/mysql-test/suite/galera/t/galera_sst_rsync.test index 5c08707e870..c944c8d84b7 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync.test @@ -10,4 +10,5 @@ --source suite/galera/include/galera_st_kill_slave.inc --source suite/galera/include/galera_st_kill_slave_ddl.inc + --source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf index 4f25af7cd8b..3913ab6660f 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf +++ b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf @@ -12,4 +12,3 @@ log_bin wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' log_basename=server2 log_bin - diff --git a/mysql-test/suite/galera/t/galera_wan_restart_sst.test b/mysql-test/suite/galera/t/galera_wan_restart_sst.test index 16e073e7164..2048977b4ef 100644 --- a/mysql-test/suite/galera/t/galera_wan_restart_sst.test +++ b/mysql-test/suite/galera/t/galera_wan_restart_sst.test @@ -12,6 +12,16 @@ --source include/galera_cluster.inc --source include/force_restart.inc +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--let $node_4=node_4 +--source include/auto_increment_offset_save.inc + --let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; --source include/wait_condition.inc SELECT VARIABLE_VALUE AS EXPECT_4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; @@ -23,11 +33,9 @@ INSERT INTO t1 VALUES (1); --connection node_2 INSERT INTO t1 VALUES (2); ---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 --connection node_3 INSERT INTO t1 VALUES (3); ---connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 --connection node_4 INSERT INTO t1 VALUES (4); @@ -156,3 +164,10 @@ CALL mtr.add_suppression("Action message in non-primary configuration from membe --connection node_4 CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc + +--connection node_1 +--disconnect node_3 +--disconnect node_4 diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index deebe7cf820..f1d332a9822 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2012-2015 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -22,6 +22,62 @@ set -ue # Setting the path for some utilities on CentOS export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" +trim_string() +{ + if [ -n "$BASH_VERSION" ]; then + local pattern="[![:space:]${2:-}]" + local x="${1#*$pattern}" + local z=${#1} + x=${#x} + if [ $x -ne $z ]; then + local y="${1%$pattern*}" + y=${#y} + x=$(( z-x-1 )) + y=$(( y-x+1 )) + printf '%s' "${1:$x:$y}" + else + printf '' + fi + else + local pattern="[[:space:]${2:-}]" + echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" + fi +} + +trim_dir() +{ + local t=$(trim_string "$1") + if [ "$t" != '/' ]; then + if [ "${t%/}" != "$t" ]; then + t=$(trim_string "${t%/}") + fi + else + t='.' + fi + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$t" + else + echo "$t" + fi +} + +to_minuses() +{ + local x="$1" + local t="${1#*_}" + local r="" + while [ "$t" != "$x" ]; do + r="$r${x%%_*}-" + x="$t" + t="${t#*_}" + done + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$r$x" + else + echo "$r$x" + fi +} + WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" WSREP_SST_OPT_BINLOG_INDEX="" @@ -44,9 +100,9 @@ WSREP_SST_OPT_ADDR_PORT="" WSREP_SST_OPT_HOST="" WSREP_SST_OPT_HOST_UNESCAPED="" WSREP_SST_OPT_HOST_ESCAPED="" -INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}" -INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}" -INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}" +INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}") +INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}") +INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}") INNODB_FORCE_RECOVERY="" INNOEXTRA="" @@ -138,22 +194,22 @@ case "$1" in ;; '--datadir') # Let's remove the trailing slash: - readonly WSREP_SST_OPT_DATA="${2%/}" + readonly WSREP_SST_OPT_DATA=$(trim_dir "$2") shift ;; '--innodb-data-home-dir') # Let's remove the trailing slash: - readonly INNODB_DATA_HOME_DIR="${2%/}" + readonly INNODB_DATA_HOME_DIR=$(trim_dir "$2") shift ;; '--innodb-log-group-home-dir') # Let's remove the trailing slash: - readonly INNODB_LOG_GROUP_HOME="${2%/}" + readonly INNODB_LOG_GROUP_HOME=$(trim_dir "$2") shift ;; '--innodb-undo-directory') # Let's remove the trailing slash: - readonly INNODB_UNDO_DIR="${2%/}" + readonly INNODB_UNDO_DIR=$(trim_dir "$2") shift ;; '--defaults-file') @@ -247,6 +303,7 @@ case "$1" in '--mysqld-args') original_cmd="" shift + cmd_tail=0 while [ $# -gt 0 ]; do lname="${1#--}" # "--" is interpreted as the end of the list of options: @@ -261,7 +318,7 @@ case "$1" in shift done fi - break; + break fi # Make sure the argument does not start with "--", otherwise it # is a long option, which is processed after this "if": @@ -301,15 +358,25 @@ case "$1" in if [ "${2#-}" = "$2" ]; then shift value="$1" + elif [ "$2" = '--' ]; then + shift + if [ $# -gt 1 ]; then + cmd_tail=1 + shift + value="$1" + fi fi fi - if [ $option = 'h' ]; then + if [ "$option" = 'h' ]; then if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi - elif [ $option != 'u' -a \ - $option != 'P' ] + elif [ "$option" != 'u' -a \ + "$option" != 'P' ] then + if [ $cmd_tail -ne 0 ]; then + option="$option --" + fi if [ -z "$value" ]; then slist="$slist$option" elif [ -z "$slist" ]; then @@ -317,9 +384,16 @@ case "$1" in else slist="$slist -$option '$value'" fi + break + fi + if [ $cmd_tail -ne 0 ]; then + if [ -n "$slist" ]; then + slist="$slist --" + else + slist='-' + fi fi break - else slist="$slist$option" fi @@ -329,7 +403,7 @@ case "$1" in original_cmd="$original_cmd -$slist" fi elif [ -z "$options" ]; then - # We found an equal sign without any characters after it: + # We found an minus sign without any characters after it: original_cmd="$original_cmd -" else # We found a value that does not start with a minus - @@ -338,12 +412,25 @@ case "$1" in original_cmd="$original_cmd '$1'" fi shift - continue; + if [ $cmd_tail -ne 0 ]; then + # All other arguments must be copied unchanged: + while [ $# -gt 0 ]; do + original_cmd="$original_cmd '$1'" + shift + done + break + fi + continue fi # Now we are sure that we are working with an option # that has a "long" name, so remove all characters after # the first equal sign: option="${1%%=*}" + # If the option name contains underscores, then replace + # them to minuses: + if [ "${option#*_}" != "$option" ]; then + option=$(to_minuses "$option") + fi # The "--loose-" prefix should not affect the recognition # of the option name: if [ "${option#--loose-}" != "$option" ]; then @@ -370,19 +457,19 @@ case "$1" in case "$option" in '--innodb-data-home-dir') if [ -z "$INNODB_DATA_HOME_DIR" ]; then - MYSQLD_OPT_INNODB_DATA_HOME_DIR="${value%/}" + MYSQLD_OPT_INNODB_DATA_HOME_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-log-group-home-dir') if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - MYSQLD_OPT_INNODB_LOG_GROUP_HOME="${value%/}" + MYSQLD_OPT_INNODB_LOG_GROUP_HOME=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-undo-directory') if [ -z "$INNODB_UNDO_DIR" ]; then - MYSQLD_OPT_INNODB_UNDO_DIR="${value%/}" + MYSQLD_OPT_INNODB_UNDO_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -412,7 +499,7 @@ case "$1" in ;; '--datadir') if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -471,8 +558,8 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments to mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else # Take the default name: @@ -525,26 +612,23 @@ get_binlog() WSREP_SST_OPT_BINLOG_INDEX=$(parse_cnf '--mysqld' 'log-bin-index') fi # if no command line argument and WSREP_SST_OPT_LOG_BASENAME is not set, - # try to get it from my.cnf: + # then try to get it from my.cnf: if [ -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then WSREP_SST_OPT_LOG_BASENAME=$(parse_cnf '--mysqld' 'log-basename') fi if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - # If the --log-bin option is specified without a parameter, + # If the log-bin option is specified without a parameter, # then we need to build the name of the index file according # to the rules described in the server documentation: - if [ -n "${MYSQLD_OPT_LOG_BIN+x}" -o \ - $(in_config '--mysqld' 'log-bin') -eq 1 ] - then + if [ $(in_config '--mysqld' 'log-bin') -ne 0 ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # If the --log-bin option is present without a value, then - # we take the default name: + # Take the default name: readonly WSREP_SST_OPT_BINLOG='mysql-bin' fi fi @@ -554,13 +638,13 @@ get_binlog() # it according to the specifications for the server: if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then - # If the WSREP_SST_OPT_BINLOG variable is not set, but + # If the WSREP_SST_OPT_BINLOG_INDEX variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG_INDEX equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else - # the default name (note that base of this name + # Use the default name (note that base of this name # is already defined above): readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi @@ -632,7 +716,7 @@ commandex() # try to use my_print_defaults, mysql and mysqldump that come # with the sources (for MTR suite): script_binary=$(dirname "$0") -SCRIPTS_DIR=$(cd "$script_binary"; pwd -P) +SCRIPTS_DIR=$(cd "$script_binary"; pwd) EXTRA_DIR="$SCRIPTS_DIR/../extra" CLIENT_DIR="$SCRIPTS_DIR/../client" @@ -754,7 +838,11 @@ parse_cnf() if [ -z "$reval" ]; then [ -n "${3:-}" ] && reval="$3" fi - echo "$reval" + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$reval" + else + echo "$reval" + fi } # @@ -804,7 +892,11 @@ in_config() break fi done - echo $found + if [ -n "$BASH_VERSION" ]; then + printf '%s' $found + else + echo $found + fi } wsrep_auth_not_set() @@ -937,11 +1029,22 @@ wsrep_gen_secret() { get_openssl if [ -n "$OPENSSL_BINARY" ]; then - echo $("$OPENSSL_BINARY" rand -hex 16) - else - printf "%04x%04x%04x%04x%04x%04x%04x%04x" \ + "$OPENSSL_BINARY" rand -hex 16 + elif [ -n "$BASH_VERSION" ]; then + printf '%04x%04x%04x%04x%04x%04x%04x%04x' \ $RANDOM $RANDOM $RANDOM $RANDOM \ $RANDOM $RANDOM $RANDOM $RANDOM + elif [ -n "$(commandex cksum)" -a \ + -n "$(commandex printf)" ] + then + printf '%08x%08x%08x%08x' \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) + else + wsrep_log_error "Unable to generate 16-byte secret" + exit 22 fi } @@ -979,14 +1082,14 @@ is_local_ip() if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6]
/": "$ip_util" address show \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+/' -o \ | grep -F " $1/" >/dev/null && return 0 else local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6]
...": "$ifconfig_util" \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+ ' -o \ | grep -F " $1 " >/dev/null && return 0 fi fi @@ -1049,7 +1152,7 @@ check_port() ss -nlpH "( sport = :$port )" 2>/dev/null | \ grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 else - wsrep_log_error "unknown sockets utility" + wsrep_log_error "Unknown sockets utility" exit 2 # ENOENT fi @@ -1158,13 +1261,6 @@ verify_cert_matches_key() exit 22 fi - # If the diff utility is not installed, then - # we will not do this certificate check: - if [ -z "$(commandex diff)" ]; then - wsrep_log_info "diff utility not found" - return - fi - # If the openssl utility is not installed, then # we will not do this certificate check: get_openssl @@ -1175,9 +1271,9 @@ verify_cert_matches_key() # Generate the public key from the cert and the key. # They should match (otherwise we can't create an SSL connection). - if ! diff <("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null) \ - <("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null) >/dev/null 2>&1 - then + local pk1=$("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null || :) + local pk2=$("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null || :) + if [ "$pk1" != "$pk2" ]; then wsrep_log_error "******************* FATAL ERROR *****************" wsrep_log_error "* The certificate and private key do not match. *" wsrep_log_error "* Please check your certificate and key files. *" @@ -1220,28 +1316,6 @@ check_for_version() return 0 } -trim_string() -{ - if [ -n "$BASH_VERSION" ]; then - local pattern="[![:space:]${2:-}]" - local x="${1#*$pattern}" - local z=${#1} - x=${#x} - if [ $x -ne $z ]; then - local y="${1%$pattern*}" - y=${#y} - x=$(( z-x-1 )) - y=$(( y-x+1 )) - printf '%s' "${1:$x:$y}" - else - printf '' - fi - else - local pattern="[[:space:]${2:-}]" - echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" - fi -} - # # Check whether process is still running. # The first parameter contains the name of the PID file. @@ -1272,6 +1346,10 @@ check_pid() rm -f "$pid_file" || : fi fi + local config="${3:-}" + if [ -n "$config" -a -f "$config" ]; then + rm -f "$config" || : + fi CHECK_PID=0 return 1 } diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 471b0e5f5b0..2eedff554e6 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify @@ -40,7 +40,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode='DISABLED' sockopt="" progress="" ttime=0 @@ -85,13 +85,13 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' # Required for backup locks # For backup locks it is 1 sent by joiner sst_ver=1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -104,17 +104,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/mariabackup.prepare.log" INNOMOVELOG="$DATA/mariabackup.move.log" INNOBACKUPLOG="$DATA/mariabackup.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -154,7 +151,7 @@ get_keys() return fi - if [ $sfmt = 'tar' ]; then + if [ "$sfmt" = 'tar' ]; then wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 @@ -184,11 +181,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -229,15 +226,15 @@ get_keys() get_transfer() { - if [ $tfmt = 'nc' ]; then + if [ "$tfmt" = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q 'ncat'; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -259,14 +256,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -- '-N'; then + if nc -h 2>&1 | grep -qw -F -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q ncat; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -456,7 +453,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -583,8 +580,14 @@ get_stream() sig_joiner_cleanup() { + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + exit $estatus } cleanup_at_exit() @@ -595,6 +598,8 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -624,7 +629,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -715,7 +720,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -- '-k'; then + if timeout --help | grep -qw -F -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -815,7 +820,9 @@ monitor_process() [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ + "$WSREP_SST_OPT_ROLE" != 'donor' ] +then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -823,25 +830,17 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd)" cd "$OLD_PWD" @@ -869,7 +868,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -929,7 +928,7 @@ setup_commands() recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY" - INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP" } @@ -1052,6 +1051,11 @@ then iopts="--parallel=$backup_threads${iopts:+ }$iopts" fi + max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') + if [ -n "$max_binlogs" ]; then + iopts="--sst-max-binlogs=$max_binlogs${iopts:+ }$iopts" + fi + setup_commands set +e timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -1096,6 +1100,7 @@ then echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" + wsrep_log_info "mariabackup SST/IST completed on donor" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then @@ -1103,22 +1108,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1142,14 +1178,13 @@ then stagemsg='Joiner-Recv' - MODULE="xtrabackup_sst" + MODULE='xtrabackup_sst' [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" # May need xtrabackup_checkpoints later on [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" - [ -f "$DATA/ib_logfile0" ] && rm -f "$DATA/ib_logfile0" ADDR="$WSREP_SST_OPT_ADDR" @@ -1165,7 +1200,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1219,6 +1254,36 @@ then jpid=$! wsrep_log_info "Proceeding with SST" + get_binlog + + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + cd "$DATA" + wsrep_log_info "Cleaning the old binary logs" + # If there is a file with binlogs state, delete it: + [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2 + # Clean up the old binlog files and index: + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 || : + done < "$binlog_index" + rm -fv "$binlog_index" 1>&2 + fi + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + cd "$binlog_dir" + if [ "$(pwd)" != "$DATA_DIR" ]; then + wsrep_log_info \ + "Cleaning the binlog directory '$binlog_dir' as well" + fi + fi + rm -fv "$binlog_base".[0-9]* 1>&2 || : + cd "$OLD_PWD" + fi + wsrep_log_info \ "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then @@ -1235,20 +1300,6 @@ then -o -exec rm -rfv {} 1>&2 \+ fi - get_binlog - - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" - fi - fi - TDATA="$DATA" DATA="$DATA/.sst" @@ -1282,11 +1333,13 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then + if [ -n "$progress" ] && \ + pv --help | grep -qw -F -- '--line-mode' + then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -- '-F'; then + if pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1296,8 +1349,9 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit 'Joiner-Decompression' \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ + $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1314,25 +1368,9 @@ then fi fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done - cd "$OLD_PWD" - - fi - wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "mariabackup prepare stage" "$INNOAPPLY" + timeit 'mariabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "mariabackup apply finished with errors." \ @@ -1340,10 +1378,43 @@ then exit 22 fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + cd "$DATA_DIR" + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + fi + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + if [ -n "$binlogs" ]; then + wsrep_log_info "Moving binary logs to $binlog_dir" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + mv "$DATA/$bin_file" "$binlog_dir" + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi + cd "$OLD_PWD" + fi + MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "mariabackup move stage" "$INNOMOVE" + timeit 'mariabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" @@ -1370,6 +1441,7 @@ then cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id wsrep_log_info "Total time on joiner: $totime seconds" + wsrep_log_info "mariabackup SST/IST completed on joiner" fi exit 0 diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 1c8fc181328..3c92f489cb5 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -3,7 +3,7 @@ set -ue # Copyright (C) 2009-2015 Codership Oy -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,8 +40,7 @@ then fi # Check client version -if ! $MYSQL_CLIENT --version | grep 'Distrib 10\.[1-9]' >/dev/null -then +if ! $MYSQL_CLIENT --version | grep -q -E 'Distrib 10\.[1-9]'; then $MYSQL_CLIENT --version >&2 wsrep_log_error "this operation requires MySQL client version 10.1 or newer" exit $EINVAL @@ -95,7 +94,7 @@ DROP PREPARE stmt;" SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" SET_WSREP_GTID_DOMAIN_ID="" -if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ]; then +if [ -n "$WSREP_SST_OPT_GTID_DOMAIN_ID" ]; then SET_WSREP_GTID_DOMAIN_ID=" SET @val = (SELECT GLOBAL_VALUE FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES WHERE VARIABLE_NAME = 'WSREP_GTID_STRICT_MODE' AND GLOBAL_VALUE > 0); SET @stmt = IF (@val IS NOT NULL, 'SET GLOBAL WSREP_GTID_DOMAIN_ID=$WSREP_SST_OPT_GTID_DOMAIN_ID', 'SET @dummy = 0'); diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index cc73db5da87..7845f143088 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2010-2014 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -36,6 +36,8 @@ cleanup_joiner() { local failure=0 + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID," \ "stunnel PID=$STUNNEL_REAL_PID" @@ -58,6 +60,7 @@ cleanup_joiner() if [ $failure -eq 0 ]; then if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" else wsrep_log_warning "rsync cleanup failed." fi @@ -140,66 +143,77 @@ STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" -BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" -BINLOG_N_FILES=1 - get_binlog if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") -fi - -# if no command line argument and INNODB_LOG_GROUP_HOME is not set, -# try to get it from my.cnf: -if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") fi OLD_PWD="$(pwd)" -WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME" - -cd "$WSREP_SST_OPT_DATA" -if [ -n "$WSREP_LOG_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR" - cd "$WSREP_LOG_DIR" +DATA="$WSREP_SST_OPT_DATA" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -WSREP_LOG_DIR=$(pwd -P) +DATA_DIR="$(pwd)" cd "$OLD_PWD" -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: +BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" + +ib_log_dir="$DATA_DIR" +ib_home_dir="$DATA_DIR" +ib_undo_dir="$DATA_DIR" + +# if no command line argument and INNODB_LOG_GROUP_HOME is not set, +# then try to get it from the my.cnf: +if [ -z "$INNODB_LOG_GROUP_HOME" ]; then + INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") +fi + +if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" +fi + +# if no command line argument and INNODB_DATA_HOME_DIR environment +# variable is not set, try to get it from the my.cnf: if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" fi -INNODB_DATA_HOME_DIR=$(pwd -P) -cd "$OLD_PWD" - -# if no command line argument then try to get it from my.cnf: +# if no command line argument and INNODB_UNDO_DIR is not set, +# then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_UNDO_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" fi -INNODB_UNDO_DIR=$(pwd -P) - -cd "$OLD_PWD" encgroups='--mysqld|sst' @@ -276,7 +290,7 @@ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST" fi if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then - CHECK_OPT_LOCAL="checkHost = localhost" + CHECK_OPT_LOCAL='checkHost = localhost' fi fi fi @@ -293,14 +307,58 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then fi fi -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' -if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] -then +SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" +# give some time for previous SST to complete: +check_round=0 +while check_pid "$SST_PID" 0; do + wsrep_log_info "Previous SST is not completed, waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 20 ]; then + wsrep_log_error "previous SST script still running." + exit 114 # EALREADY + fi + sleep 1 +done + +echo $$ > "$SST_PID" + +# give some time for stunnel from the previous SST to complete: +check_round=0 +while check_pid "$STUNNEL_PID" 1 "$STUNNEL_CONF"; do + wsrep_log_info "Lingering stunnel daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "stunnel daemon still running." + exit 114 # EALREADY + fi + sleep 1 +done + +MODULE='rsync_sst' +RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" +RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" + +# give some time for rsync from the previous SST to complete: +check_round=0 +while check_pid "$RSYNC_PID" 1 "$RSYNC_CONF"; do + wsrep_log_info "Lingering rsync daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "rsync daemon still running." + exit 114 # EALREADY + fi + sleep 1 +done + +[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" +[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" + +if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then if [ -n "$STUNNEL" ] then @@ -319,8 +377,6 @@ ${VERIFY_OPT} ${CHECK_OPT} ${CHECK_OPT_LOCAL} EOF - else - [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" fi RC=0 @@ -333,7 +389,7 @@ EOF [ -f "$FLUSHED" ] && rm -f "$FLUSHED" [ -f "$ERROR" ] && rm -f "$ERROR" - echo "flush tables" + echo 'flush tables' # Wait for : # (a) Tables to be flushed, AND @@ -357,32 +413,78 @@ EOF sync - if [ -n "$WSREP_SST_OPT_BINLOG" -a -d "${BINLOG_DIRNAME:-}" ] - then - # Prepare binlog files - cd "$BINLOG_DIRNAME" - - binlog_files_full=$(tail -n $BINLOG_N_FILES \ - "$WSREP_SST_OPT_BINLOG_INDEX") - binlog_files="" - for file in $binlog_files_full; do - binlog_file=$(basename "$file") - binlog_files="$binlog_files${binlog_files:+ }'$binlog_file'" - done - - if [ -n "$binlog_files" ]; then - wsrep_log_info "Preparing binlog files for transfer:" - eval tar -cvf "'$BINLOG_TAR_FILE'" $binlog_files >&2 + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + # Change the directory to binlog base (if possible): + cd "$DATA" + # Let's check the existence of the file with the index: + if [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ]; then + # Let's read the binlog index: + max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') + if [ -n "$max_binlogs" ]; then + binlog_files="" + if [ $max_binlogs -gt 0 ]; then + binlog_files=$(tail -n $max_binlogs \ + "$WSREP_SST_OPT_BINLOG_INDEX") + fi + else + binlog_files=$(cat "$WSREP_SST_OPT_BINLOG_INDEX") + fi + if [ -n "$binlog_files" ]; then + # Preparing binlog files for transfer: + wsrep_log_info "Preparing binlog files for transfer:" + tar_type=0 + if tar --help | grep -qw -F -- '--transform'; then + tar_type=1 + elif tar --help | grep -qw -E -- '-s[[:space:]]pattern' + then + tar_type=2 + fi + if [ $tar_type -ne 0 ]; then + # Preparing list of the binlog file names: + echo "$binlog_files" | { + binlogs="" + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + binlogs="$binlogs${binlogs:+ }'$bin_file'" + done + if [ -n "$binlogs" ]; then + tar_options='/^.*\///g' + if [ $tar_type -eq 1 ]; then + tar_options="--transform='s$tar_options'" + else + tar_options="-s '$tar_options'" + fi + eval tar -P $tar_options \ + -cvf "'$BINLOG_TAR_FILE'" $binlogs >&2 + fi + } + else + tar_options='-cvf' + echo "$binlog_files" | \ + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + bin_dir=$(dirname "$bin_file") + bin_base=$(basename "$bin_file") + if [ -n "$bin_dir" -a "$bin_dir" != '.' ]; then + tar $tar_options "$BINLOG_TAR_FILE" \ + -C "$bin_dir" "$bin_base" >&2 + else + tar $tar_options "$BINLOG_TAR_FILE" \ + "$bin_base" >&2 + fi + tar_options='-rvf' + done + fi + fi fi - cd "$OLD_PWD" fi - # Use deltaxfer only for WAN + # Use deltaxfer only for WAN: inv=$(basename "$0") WHOLE_FILE_OPT="" if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then - WHOLE_FILE_OPT="--whole-file" + WHOLE_FILE_OPT='--whole-file' fi # Old filter - include everything except selected @@ -399,9 +501,9 @@ FILTER="-f '- /lost+found' -f '- /.pid' -f '- /.conf' -f '+ /wsrep_sst_binlog.tar' - -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' - -f '- $INNODB_DATA_HOME_DIR/ibdata*' - -f '+ $INNODB_UNDO_DIR/undo*' + -f '- $ib_home_dir/ib_lru_dump' + -f '- $ib_home_dir/ibdata*' + -f '+ $ib_undo_dir/undo*' -f '+ /*/' -f '- /*'" @@ -435,7 +537,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ - -f '- **' "$INNODB_DATA_HOME_DIR/" \ + -f '- **' "$ib_home_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -448,7 +550,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '+ /aria_log.*' \ - -f '+ /aria_log_control' -f '- **' "$WSREP_LOG_DIR/" \ + -f '+ /aria_log_control' -f '- **' "$ib_log_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -459,7 +561,7 @@ FILTER="-f '- /lost+found' # then, we parallelize the transfer of database directories, # use '.' so that path concatenation works: - cd "$WSREP_SST_OPT_DATA" + cd "$DATA" backup_threads=$(parse_cnf '--mysqld|sst' 'backup-threads') if [ -z "$backup_threads" ]; then @@ -518,68 +620,21 @@ FILTER="-f '- /lost+found' [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" fi + [ -f "$SST_PID" ] && rm -f "$SST_PID" + + wsrep_log_info "rsync SST/IST completed on donor" + elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then check_sockets_utils - SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" - - # give some time for previous SST to complete: - check_round=0 - while check_pid "$SST_PID" 0 'wsrep_sst_'; do - wsrep_log_info "previous SST is not completed, waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "previous SST script still running." - exit 114 # EALREADY - fi - sleep 1 - done - - echo $$ > "$SST_PID" - - # give some time for stunnel from the previous SST to complete: - check_round=0 - while check_pid "$STUNNEL_PID" 1; do - wsrep_log_info "Lingering stunnel daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "stunnel daemon already running." - exit 114 # EALREADY - fi - sleep 1 - done - - MODULE="rsync_sst" - RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" - RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" - - # give some time for rsync from the previous SST to complete: - check_round=0 - while check_pid "$RSYNC_PID" 1; do - wsrep_log_info "Lingering rsync daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "rsync daemon already running." - exit 114 # EALREADY - fi - sleep 1 - done - - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - - [ -z "$STUNNEL" -a -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" - ADDR="$WSREP_SST_OPT_ADDR" RSYNC_PORT="$WSREP_SST_OPT_PORT" RSYNC_ADDR="$WSREP_SST_OPT_HOST" RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED" - trap "exit 32" HUP PIPE - trap "exit 3" INT TERM ABRT + trap 'exit 32' HUP PIPE + trap 'exit 3' INT TERM ABRT trap cleanup_joiner EXIT touch "$SST_PROGRESS_FILE" @@ -600,13 +655,11 @@ $SILENT path = $WSREP_SST_OPT_DATA exclude = .zfs [$MODULE-log_dir] - path = $WSREP_LOG_DIR + path = $ib_log_dir [$MODULE-data_dir] - path = $INNODB_DATA_HOME_DIR + path = $ib_home_dir EOF -# rm -rf "$DATA/ib_logfile"* # we don't want old logs around - # If the IP is local, listen only on it: if is_local_ip "$RSYNC_ADDR_UNESCAPED" then @@ -617,7 +670,7 @@ EOF RSYNC_EXTRA_ARGS="" STUNNEL_ACCEPT="$RSYNC_PORT" # Overwrite address with all: - RSYNC_ADDR="*" + RSYNC_ADDR='*' fi if [ -z "$STUNNEL" ]; then @@ -675,11 +728,10 @@ EOF TRANSFER_PID="$STUNNEL_PID" fi - if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ] - then # backward-incompatible behavior + if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then + # backward-incompatible behavior CN="" - if [ -n "$SSTCERT" ] - then + if [ -n "$SSTCERT" ]; then # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then @@ -688,7 +740,7 @@ EOF exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -725,16 +777,53 @@ EOF exit 32 fi + if [ -r "$MAGIC_FILE" ]; then + if [ -n "$MY_SECRET" ]; then + # Check donor supplied secret: + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) + if [ "$SECRET" != "$MY_SECRET" ]; then + wsrep_log_error "Donor does not know my secret!" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" + exit 32 + fi + fi + else + # This message should cause joiner to abort: + wsrep_log_info "rsync process ended without creating magic file" + echo "rsync process ended without creating '$MAGIC_FILE'" + exit 32 + fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - if [ -f "$BINLOG_TAR_FILE" ]; then - cd "$BINLOG_DIRNAME" - + binlog_tar_present=0 + [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1 + # If it is SST (not an IST) or tar with binlogs is present + # among the transferred files, then we need to remove the + # old binlogs: + if [ $WSREP_SST_OPT_BYPASS -eq 0 -o $binlog_tar_present -ne 0 ]; then + cd "$DATA" + # Clean up the old binlog files and index: binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - - # Clean up old binlog files first - rm -f "$BINLOG_FILENAME".[0-9]* - [ -f "$binlog_index" ] && rm -f "$binlog_index" - + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -f "$bin_file" || : + done < "$binlog_index" + rm -f "$binlog_index" + fi + binlog_cd=0 + # Change the directory to binlog base (if possible): + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + binlog_cd=1 + cd "$binlog_dir" + fi + # Clean up unindexed binlog files: + rm -f "$binlog_base".[0-9]* || : + [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" + fi + if [ $binlog_tar_present -ne 0 ]; then # Create a temporary file: tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') if [ -z "$tmpdir" ]; then @@ -744,46 +833,44 @@ EOF else tmpfile=$(TMPDIR="$tmpdir"; mktemp) fi - + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + binlog_cd=0 + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + cd "$binlog_dir" + binlog_cd=1 + fi + # Extracting binlog files: wsrep_log_info "Extracting binlog files:" if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then - wsrep_log_error "Error unpacking tar file with binlog files" rm -f "$tmpfile" + wsrep_log_error "Error unpacking tar file with binlog files" exit 32 fi - # Rebuild binlog index: - while read bin_file; do - echo "$BINLOG_DIRNAME/$bin_file" >> "$binlog_index" + [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" done < "$tmpfile" rm -f "$tmpfile" - cd "$OLD_PWD" fi fi - if [ -r "$MAGIC_FILE" ]; then - if [ -n "$MY_SECRET" ]; then - # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) - if [ "$SECRET" != "$MY_SECRET" ]; then - wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" - exit 32 - fi - # remove secret from the magic file, and output - # the UUID:seqno & wsrep_gtid_domain_id: - grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" - else - # Output the UUID:seqno and wsrep_gtid_domain_id: - cat "$MAGIC_FILE" - fi + if [ -n "$MY_SECRET" ]; then + # remove secret from the magic file, and output + # the UUID:seqno & wsrep_gtid_domain_id: + grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" else - # this message should cause joiner to abort - echo "rsync process ended without creating '$MAGIC_FILE'" + # Output the UUID:seqno and wsrep_gtid_domain_id: + cat "$MAGIC_FILE" fi + wsrep_log_info "rsync SST/IST completed on joiner" + # wsrep_cleanup_progress_file # cleanup_joiner else diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 569d5084c98..978c4805205 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -1,5 +1,8 @@ -#!/bin/bash -ue -# Copyright (C) 2017-2021 MariaDB +#!/usr/bin/env bash + +set -ue + +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify @@ -38,7 +41,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode='DISABLED' sockopt="" progress="" ttime=0 @@ -85,11 +88,11 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' sst_ver=-1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -102,17 +105,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/innobackupex.prepare.log" INNOMOVELOG="$DATA/innobackupex.move.log" INNOBACKUPLOG="$DATA/innobackupex.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -152,7 +152,7 @@ get_keys() return fi - if [ $sfmt = 'tar' ]; then + if [ "$sfmt" = 'tar' ]; then wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 @@ -182,11 +182,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -231,15 +231,15 @@ get_keys() get_transfer() { - if [ $tfmt = 'nc' ]; then + if [ "$tfmt" = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q 'ncat'; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -261,14 +261,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -- '-N'; then + if nc -h 2>&1 | grep -qw -F -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q ncat; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -458,7 +458,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -592,8 +592,14 @@ get_stream() sig_joiner_cleanup() { + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + exit $estatus } cleanup_at_exit() @@ -604,6 +610,8 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -633,7 +641,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -724,7 +732,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -- '-k'; then + if timeout --help | grep -qw -F -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -824,9 +832,10 @@ monitor_process() # check the version, we require XB-2.3.5 to ensure that we can pass the # datadir via the command-line option -XB_REQUIRED_VERSION="2.3.5" +XB_REQUIRED_VERSION='2.3.5' -XB_VERSION=`$BACKUP_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` +XB_VERSION=$($BACKUP_BIN --version 2>&1 | \ + grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1) if [ -z "$XB_VERSION" ]; then wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version." \ "Needs xtrabackup-$XB_REQUIRED_VERSION or higher to" \ @@ -843,7 +852,9 @@ fi [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ + "$WSREP_SST_OPT_ROLE" != 'donor' ] +then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -851,25 +862,17 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd)" cd "$OLD_PWD" @@ -897,7 +900,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -953,7 +956,7 @@ setup_commands() recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi INNOAPPLY="$BACKUP_BIN$disver$recovery${iapts:+ }$iapts$INNOEXTRA --apply-log${rebuildcmd:+ }$rebuildcmd --datadir='$DATA' '$DATA' $INNOAPPLY" - INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --datadir='${TDATA:-$DATA}' '$DATA' $INNOMOVE" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --datadir='${TDATA:-$DATA}' '$DATA' $INNOMOVE" local sfmt_work="$sfmt" if [ "$sfmt" = 'mbstream' ]; then sfmt_work='xbstream' @@ -1124,6 +1127,7 @@ then echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" + wsrep_log_info "xtrabackup SST/IST completed on donor" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then @@ -1131,22 +1135,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1170,7 +1205,7 @@ then stagemsg='Joiner-Recv' - MODULE="xtrabackup_sst" + MODULE='xtrabackup_sst' [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" @@ -1193,7 +1228,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1247,6 +1282,36 @@ then jpid=$! wsrep_log_info "Proceeding with SST" + get_binlog + + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + cd "$DATA" + wsrep_log_info "Cleaning the old binary logs" + # If there is a file with binlogs state, delete it: + [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2 + # Clean up the old binlog files and index: + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 || : + done < "$binlog_index" + rm -fv "$binlog_index" 1>&2 + fi + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + cd "$binlog_dir" + if [ "$(pwd)" != "$DATA_DIR" ]; then + wsrep_log_info \ + "Cleaning the binlog directory '$binlog_dir' as well" + fi + fi + rm -fv "$binlog_base".[0-9]* 1>&2 || : + cd "$OLD_PWD" + fi + wsrep_log_info \ "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then @@ -1263,20 +1328,6 @@ then -o -exec rm -rfv {} 1>&2 \+ fi - get_binlog - - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" - fi - fi - TDATA="$DATA" DATA="$DATA/.sst" @@ -1312,11 +1363,13 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then + if [ -n "$progress" ] && \ + pv --help | grep -qw -F -- '--line-mode' + then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -- '-F'; then + if pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1326,8 +1379,9 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit 'Joiner-Decompression' \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ + $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1344,25 +1398,9 @@ then fi fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done - cd "$OLD_PWD" - - fi - wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "Xtrabackup prepare stage" "$INNOAPPLY" + timeit 'Xtrabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "xtrabackup apply finished with errors." \ @@ -1370,10 +1408,43 @@ then exit 22 fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + cd "$DATA_DIR" + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + fi + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + if [ -n "$binlogs" ]; then + wsrep_log_info "Moving binary logs to $binlog_dir" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + mv "$DATA/$bin_file" "$binlog_dir" + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi + cd "$OLD_PWD" + fi + MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "Xtrabackup move stage" "$INNOMOVE" + timeit 'Xtrabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" @@ -1400,6 +1471,7 @@ then cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id wsrep_log_info "Total time on joiner: $totime seconds" + wsrep_log_info "xtrabackup SST/IST completed on joiner" fi exit 0 From fac9224d85e879ff06e549c18446375d850ef83a Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Sat, 12 Feb 2022 00:59:15 +0100 Subject: [PATCH 09/11] MDEV-27777: Some Galera tests fail on FreeBSD This commit fixes problems due to bugs and quirks in bsdtar (the FreeBSD version of tar). Separate tests are not required, because without these fixes, many other tests fail when tested in the FreeBSD environment. Also, the grep patterns for reading utility version numbers has been made more robust. The notation of some options of the "cut" utility has been changed. --- scripts/wsrep_sst_common.sh | 2 +- scripts/wsrep_sst_mariabackup.sh | 5 ++-- scripts/wsrep_sst_rsync.sh | 48 ++++++++++++++++++++++++------ scripts/wsrep_sst_xtrabackup-v2.sh | 8 ++--- 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index f1d332a9822..6d4def74b86 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -336,7 +336,7 @@ case "$1" in else # If it's not bash, then we need to use slow # external utilities: - option=$(echo "$options" | cut -c1-1) + option=$(echo "$options" | cut -c1) fi # And the subsequent characters consider option value: value="" diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 2eedff554e6..a15a3f7b7a5 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -323,7 +323,8 @@ get_transfer() if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then if [ -z "$ssl_dhparams" ]; then # Determine the socat version - SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') + SOCAT_VERSION=$(socat -V 2>&1 | \ + grep -m1 -owE '[0-9]+(\.[0-9]+)+' | head -n1) if [ -z "$SOCAT_VERSION" ]; then wsrep_log_error "******** FATAL ERROR ******************" wsrep_log_error "* Cannot determine the socat version. *" @@ -766,7 +767,7 @@ recv_joiner() # check donor supplied secret SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) + cut -d ' ' -f2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 7845f143088..54160fd19a2 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -435,24 +435,38 @@ EOF tar_type=0 if tar --help | grep -qw -F -- '--transform'; then tar_type=1 - elif tar --help | grep -qw -E -- '-s[[:space:]]pattern' - then + elif tar --version | grep -q -E '^bsdtar\>'; then tar_type=2 fi + if [ $tar_type -ne 2 ]; then + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$binlog_files" >&2 + else + echo "$binlog_files" >&2 + fi + fi if [ $tar_type -ne 0 ]; then # Preparing list of the binlog file names: echo "$binlog_files" | { binlogs="" while read bin_file || [ -n "$bin_file" ]; do [ ! -f "$bin_file" ] && continue + if [ -n "$BASH_VERSION" ]; then + first="${bin_file:0:1}" + else + first=$(echo "$bin_file" | cut -c1) + fi + if [ "$first" = '-' -o "$first" = '@' ]; then + bin_file="./$bin_file" + fi binlogs="$binlogs${binlogs:+ }'$bin_file'" done if [ -n "$binlogs" ]; then - tar_options='/^.*\///g' if [ $tar_type -eq 1 ]; then - tar_options="--transform='s$tar_options'" + tar_options="--transform='s/^.*\///g'" else - tar_options="-s '$tar_options'" + # bsdtar handles backslash incorrectly: + tar_options="-s '?^.*/??g'" fi eval tar -P $tar_options \ -cvf "'$BINLOG_TAR_FILE'" $binlogs >&2 @@ -465,6 +479,14 @@ EOF [ ! -f "$bin_file" ] && continue bin_dir=$(dirname "$bin_file") bin_base=$(basename "$bin_file") + if [ -n "$BASH_VERSION" ]; then + first="${bin_base:0:1}" + else + first=$(echo "$bin_base" | cut -c1) + fi + if [ "$first" = '-' -o "$first" = '@' ]; then + bin_base="./$bin_base" + fi if [ -n "$bin_dir" -a "$bin_dir" != '.' ]; then tar $tar_options "$BINLOG_TAR_FILE" \ -C "$bin_dir" "$bin_base" >&2 @@ -740,7 +762,7 @@ EOF exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ - tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -781,7 +803,7 @@ EOF if [ -n "$MY_SECRET" ]; then # Check donor supplied secret: SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) + cut -d ' ' -f2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" @@ -840,12 +862,20 @@ EOF binlog_cd=0 if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" - cd "$binlog_dir" binlog_cd=1 + cd "$binlog_dir" fi # Extracting binlog files: wsrep_log_info "Extracting binlog files:" - if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then + RC=0 + if tar --version | grep -q -E '^bsdtar\>'; then + tar -tf "$BINLOG_TAR_FILE" > "$tmpfile" && \ + tar -xvf "$BINLOG_TAR_FILE" > /dev/null || RC=$? + else + tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile" && \ + cat "$tmpfile" >&2 || RC=$? + fi + if [ $RC -ne 0 ]; then rm -f "$tmpfile" wsrep_log_error "Error unpacking tar file with binlog files" exit 32 diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 978c4805205..e7cd77396fe 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -328,7 +328,8 @@ get_transfer() if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then if [ -z "$ssl_dhparams" ]; then # Determine the socat version - SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') + SOCAT_VERSION=$(socat -V 2>&1 | \ + grep -m1 -owE '[0-9]+(\.[0-9]+)+' | head -n1) if [ -z "$SOCAT_VERSION" ]; then wsrep_log_error "******** FATAL ERROR ******************" wsrep_log_error "* Cannot determine the socat version. *" @@ -778,7 +779,7 @@ recv_joiner() # check donor supplied secret SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) + cut -d ' ' -f2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" @@ -835,7 +836,7 @@ monitor_process() XB_REQUIRED_VERSION='2.3.5' XB_VERSION=$($BACKUP_BIN --version 2>&1 | \ - grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1) + grep -m1 -owE '[0-9]+(\.[0-9]+)+' | head -n1) if [ -z "$XB_VERSION" ]; then wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version." \ "Needs xtrabackup-$XB_REQUIRED_VERSION or higher to" \ @@ -1212,7 +1213,6 @@ then # May need xtrabackup_checkpoints later on [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" - [ -f "$DATA/ib_logfile0" ] && rm -f "$DATA/ib_logfile0" ADDR="$WSREP_SST_OPT_ADDR" From 8b7abe21e041d616ddb5a68652bc96e97b3eed44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 23 Feb 2022 06:00:01 +0200 Subject: [PATCH 10/11] MDEV-23888 fixup: GCC 12 -Wunused-value --- storage/innobase/handler/ha_innodb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 0c0900bd744..48a1944f0e1 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2811,7 +2811,7 @@ innobase_trx_init( while holding lock_sys.mutex, by lock_rec_enqueue_waiting(), will not end up acquiring LOCK_global_system_variables in intern_sys_var_ptr(). */ - THDVAR(thd, lock_wait_timeout); + (void) THDVAR(thd, lock_wait_timeout); trx->check_foreigns = !thd_test_options( thd, OPTION_NO_FOREIGN_KEY_CHECKS); From a3da3c8a0b1113d941f837cce0615673b95fb9f2 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 22 Feb 2022 12:34:58 +0100 Subject: [PATCH 11/11] MDEV-26377: Stricter validation of ssl-mode values This commit adds validation of the values of the ssl-mode parameter in SSL scripts, since now only a basic check for the presence of the "VERIFY_" prefix is performed there to detect "VERIFY_IDENTITY" and "VERIFY_CA", but all other values are not checked at all. In addition, this commit removes leading and trailing spaces from parameter values that SST scripts read from configuration files or from the command line so that they do not interfere with parameter checks and substitutions. Parameter substitution has been made more robust against characters in strings that the shell might erroneously interpret as regexp. --- scripts/wsrep_sst_common.sh | 91 +++++++++++++----------------- scripts/wsrep_sst_mariabackup.sh | 26 +++++---- scripts/wsrep_sst_rsync.sh | 30 +++++----- scripts/wsrep_sst_xtrabackup-v2.sh | 26 +++++---- 4 files changed, 87 insertions(+), 86 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 6d4def74b86..d4d9a58897d 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -99,7 +99,6 @@ WSREP_SST_OPT_ADDR="" WSREP_SST_OPT_ADDR_PORT="" WSREP_SST_OPT_HOST="" WSREP_SST_OPT_HOST_UNESCAPED="" -WSREP_SST_OPT_HOST_ESCAPED="" INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}") INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}") INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}") @@ -122,20 +121,22 @@ case "$1" in # without square brackets: readonly WSREP_SST_OPT_HOST_UNESCAPED="${addr_no_bracket%%\]*}" # Square brackets are needed in most cases: - readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" - readonly WSREP_SST_OPT_HOST_ESCAPED="\\[${WSREP_SST_OPT_HOST_UNESCAPED}\\]" + readonly WSREP_SST_OPT_HOST="[$WSREP_SST_OPT_HOST_UNESCAPED]" # Mark this address as IPv6: readonly WSREP_SST_OPT_HOST_IPv6=1 + # Let's remove the leading part that contains the host address: + remain="${WSREP_SST_OPT_ADDR#*\]}" ;; *) readonly WSREP_SST_OPT_HOST="${WSREP_SST_OPT_ADDR%%[:/]*}" readonly WSREP_SST_OPT_HOST_UNESCAPED="$WSREP_SST_OPT_HOST" - readonly WSREP_SST_OPT_HOST_ESCAPED="$WSREP_SST_OPT_HOST" readonly WSREP_SST_OPT_HOST_IPv6=0 + # Let's remove the leading part that contains the host address: + remain="${WSREP_SST_OPT_ADDR#*[:/]}" ;; esac - # Let's remove the leading part that contains the host address: - remain="${WSREP_SST_OPT_ADDR#$WSREP_SST_OPT_HOST_ESCAPED}" + # If there is nothing but the address, then the remainder is empty: + [ "$remain" = "$WSREP_SST_OPT_ADDR" ] && remain="" # Let's remove the ":" character that separates the port number # from the hostname: remain="${remain#:}" @@ -143,39 +144,32 @@ case "$1" in # up to "/" (if present): WSREP_SST_OPT_ADDR_PORT="${remain%%/*}" # If the "/" character is present, then the path is not empty: - if [ "${remain#*/}" != "$remain" ]; then + if [ "$WSREP_SST_OPT_ADDR_PORT" != "$remain" ]; then # This operation removes everything up to the "/" character, # effectively removing the port number from the string: readonly WSREP_SST_OPT_PATH="${remain#*/}" else readonly WSREP_SST_OPT_PATH="" fi - # The rest of the string is the same as the path (for now): - remain="$WSREP_SST_OPT_PATH" - # If there is one more "/" in the string, then everything before - # it will be the module name, otherwise the module name is empty: - if [ "${remain%%/*}" != "$remain" ]; then - # This operation removes the tail after the very first - # occurrence of the "/" character (inclusively): - readonly WSREP_SST_OPT_MODULE="${remain%%/*}" - else - readonly WSREP_SST_OPT_MODULE="" - fi # Remove the module name part from the string, which ends with "/": remain="${WSREP_SST_OPT_PATH#*/}" - # If the rest of the string does not match the original, then there - # was something else besides the module name: + # This operation removes the tail after the very first occurrence + # of the "/" character, inclusively: + readonly WSREP_SST_OPT_MODULE="${WSREP_SST_OPT_PATH%%/*}" + # If there is one more "/" in the string, then everything before + # it will be the LSN, otherwise the LSN is empty: if [ "$remain" != "$WSREP_SST_OPT_PATH" ]; then # Extract the part that matches the LSN by removing all # characters starting from the very first "/": readonly WSREP_SST_OPT_LSN="${remain%%/*}" # Exctract everything after the first occurrence of # the "/" character in the string: + source="$remain" remain="${remain#*/}" # If the remainder does not match the original string, # then there is something else (the version number in # our case): - if [ "$remain" != "$WSREP_SST_OPT_LSN" ]; then + if [ "$remain" != "$source" ]; then # Let's extract the version number by removing the tail # after the very first occurence of the "/" character # (inclusively): @@ -238,14 +232,12 @@ case "$1" in readonly WSREP_SST_OPT_HOST_UNESCAPED="${addr_no_bracket%%\]*}" # Square brackets are needed in most cases: readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" - readonly WSREP_SST_OPT_HOST_ESCAPED="\\[${WSREP_SST_OPT_HOST_UNESCAPED}\\]" # Mark this address as IPv6: readonly WSREP_SST_OPT_HOST_IPv6=1 ;; *) readonly WSREP_SST_OPT_HOST="$2" readonly WSREP_SST_OPT_HOST_UNESCAPED="$2" - readonly WSREP_SST_OPT_HOST_ESCAPED="$2" readonly WSREP_SST_OPT_HOST_IPv6=0 ;; esac @@ -475,25 +467,25 @@ case "$1" in ;; '--innodb-force-recovery') if [ -n "$value" -a "$value" != "0" ]; then - INNODB_FORCE_RECOVERY="$value" + INNODB_FORCE_RECOVERY=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--log-bin') if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - MYSQLD_OPT_LOG_BIN="$value" + MYSQLD_OPT_LOG_BIN=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--log-bin-index') if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then - MYSQLD_OPT_LOG_BIN_INDEX="$value" + MYSQLD_OPT_LOG_BIN_INDEX=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--log-basename') if [ -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then - MYSQLD_OPT_LOG_BASENAME="$value" + MYSQLD_OPT_LOG_BASENAME=$(trim_string "$value") fi skip_mysqld_arg=1 ;; @@ -678,27 +670,18 @@ if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then # the corresponding variable: readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" fi -elif [ -n "$WSREP_SST_OPT_ADDR" ]; then +else # If the port is missing, take the default port: if [ -z "$WSREP_SST_OPT_PORT" ]; then readonly WSREP_SST_OPT_PORT=4444 fi WSREP_SST_OPT_ADDR_PORT="$WSREP_SST_OPT_PORT" - # Let's remove the leading part that contains the host address: - remain="${WSREP_SST_OPT_ADDR#$WSREP_SST_OPT_HOST_ESCAPED}" - # Let's remove the ":" character that separates the port number - # from the hostname: - remain="${remain#:}" - # Let's remove all characters upto first "/" character that - # separates the hostname with port number from the path: - remain="${remain#/}" - # Let's construct a new value for the address with the port: - WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT" - if [ -n "$remain" ]; then - WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_ADDR/$remain" - fi fi +# Let's construct a new value for the address with the port: +sst_path="${WSREP_SST_OPT_PATH:+/}$WSREP_SST_OPT_PATH" +WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT$sst_path" + readonly WSREP_SST_OPT_ADDR readonly WSREP_SST_OPT_ADDR_PORT @@ -809,8 +792,11 @@ parse_cnf() local group="${groups%%\|*}" # Remove the remainder (the group name) from the rest # of the groups list (as if it were a prefix): - groups="${groups#$group}" - groups="${groups#\|}" + if [ "$group" != "$groups" ]; then + groups="${groups#*\|}" + else + groups="" + fi # If the group name is the same as the "mysqld" without "--" prefix, # then try to use it together with the group suffix: if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then @@ -835,9 +821,11 @@ parse_cnf() done # Use default if we haven't found a value: - if [ -z "$reval" ]; then - [ -n "${3:-}" ] && reval="$3" - fi + [ -z "$reval" ] && reval="${3:-}" + + # Truncate spaces: + [ -n "$reval" ] && reval=$(trim_string "$reval") + if [ -n "$BASH_VERSION" ]; then printf '%s' "$reval" else @@ -868,8 +856,11 @@ in_config() local group="${groups%%\|*}" # Remove the remainder (the group name) from the rest # of the groups list (as if it were a prefix): - groups="${groups#$group}" - groups="${groups#\|}" + if [ "$group" != "$groups" ]; then + groups="${groups#*\|}" + else + groups="" + fi # If the group name is the same as the "mysqld" without "--" prefix, # then try to use it together with the group suffix: if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then @@ -1455,13 +1446,9 @@ check_server_ssl_config() fi fi if [ -n "$tcert" ]; then - tcert=$(trim_string "$tcert") if [ "${tcert%/}" != "$tcert" -o -d "$tcert" ]; then tcap="$tcert" tcert="" fi fi - if [ -n "$tcap" ]; then - tcap=$(trim_string "$tcap") - fi } diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index a15a3f7b7a5..ce4001fdc56 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -40,7 +40,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode='DISABLED' +tmode="" sockopt="" progress="" ttime=0 @@ -474,6 +474,15 @@ read_cnf() tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | \ tr [:lower:] [:upper:]) + case "$tmode" in + 'VERIFY_IDENTITY'|'VERIFY_CA'|'REQUIRED'|'DISABLED') + ;; + *) + wsrep_log_error "Unrecognized ssl-mode option: '$tmode'" + exit 22 # EINVAL + ;; + esac + if [ $encrypt -eq 0 -o $encrypt -ge 2 ]; then if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then check_server_ssl_config @@ -667,16 +676,13 @@ setup_ports() # wait_for_listen() { - local PORT="$1" - local ADDR="$2" - local MODULE="$3" for i in {1..150}; do - if check_port "" "$PORT" 'socat|nc'; then + if check_port "" "$SST_PORT" 'socat|nc'; then break fi sleep 0.2 done - echo "ready $ADDR/$MODULE//$sst_ver" + echo "ready $ADDR:$SST_PORT/$MODULE/$lsn/$sst_ver" } check_extra() @@ -733,7 +739,7 @@ recv_joiner() set +e if [ $wait -ne 0 ]; then - wait_for_listen "$SST_PORT" "$ADDR" "$MODULE" & + wait_for_listen & fi timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -1179,7 +1185,7 @@ then stagemsg='Joiner-Recv' - MODULE='xtrabackup_sst' + MODULE="${WSREP_SST_OPT_MODULE:-xtrabackup_sst}" [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" @@ -1187,7 +1193,7 @@ then [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" - ADDR="$WSREP_SST_OPT_ADDR" + ADDR="$WSREP_SST_OPT_HOST" if [ "${tmode#VERIFY}" != "$tmode" ]; then # backward-incompatible behavior: @@ -1201,7 +1207,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 54160fd19a2..67a7afc638f 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -238,6 +238,15 @@ if [ -z "$SSLMODE" ]; then elif [ -n "$SSTKEY" -a -n "$SSTCERT" ]; then SSLMODE='REQUIRED' fi +else + case "$SSLMODE" in + 'VERIFY_IDENTITY'|'VERIFY_CA'|'REQUIRED'|'DISABLED') + ;; + *) + wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'" + exit 22 # EINVAL + ;; + esac fi if [ -n "$SSTKEY" -a -n "$SSTCERT" ]; then @@ -262,18 +271,11 @@ VERIFY_OPT="" CHECK_OPT="" CHECK_OPT_LOCAL="" if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then - case "$SSLMODE" in - 'VERIFY_IDENTITY') + if [ "$SSLMODE" = 'VERIFY_IDENTITY' ]; then VERIFY_OPT='verifyPeer = yes' - ;; - 'VERIFY_CA') + else VERIFY_OPT='verifyChain = yes' - ;; - *) - wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'" - exit 22 # EINVAL - ;; - esac + fi if [ -z "$SSTCA$SSTCAP" ]; then wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file or path" exit 22 # EINVAL @@ -338,7 +340,8 @@ while check_pid "$STUNNEL_PID" 1 "$STUNNEL_CONF"; do sleep 1 done -MODULE='rsync_sst' +MODULE="${WSREP_SST_OPT_MODULE:-rsync_sst}" + RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" @@ -650,7 +653,7 @@ elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then check_sockets_utils - ADDR="$WSREP_SST_OPT_ADDR" + ADDR="$WSREP_SST_OPT_HOST" RSYNC_PORT="$WSREP_SST_OPT_PORT" RSYNC_ADDR="$WSREP_SST_OPT_HOST" RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED" @@ -751,7 +754,7 @@ EOF fi if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then - # backward-incompatible behavior + # backward-incompatible behavior: CN="" if [ -n "$SSTCERT" ]; then # find out my Common Name @@ -770,7 +773,6 @@ EOF ADDR="$CN:$MY_SECRET@$WSREP_SST_OPT_HOST" else MY_SECRET="" # for check down in recv_joiner() - ADDR="$WSREP_SST_OPT_HOST" fi until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID \ diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index e7cd77396fe..747378aced8 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -41,7 +41,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode='DISABLED' +tmode="" sockopt="" progress="" ttime=0 @@ -479,6 +479,15 @@ read_cnf() tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | \ tr [:lower:] [:upper:]) + case "$tmode" in + 'VERIFY_IDENTITY'|'VERIFY_CA'|'REQUIRED'|'DISABLED') + ;; + *) + wsrep_log_error "Unrecognized ssl-mode option: '$tmode'" + exit 22 # EINVAL + ;; + esac + if [ $encrypt -eq 0 -o $encrypt -ge 2 ]; then if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then check_server_ssl_config @@ -679,16 +688,13 @@ setup_ports() # wait_for_listen() { - local PORT="$1" - local ADDR="$2" - local MODULE="$3" for i in {1..150}; do - if check_port "" "$PORT" 'socat|nc'; then + if check_port "" "$SST_PORT" 'socat|nc'; then break fi sleep 0.2 done - echo "ready $ADDR/$MODULE//$sst_ver" + echo "ready $ADDR:$SST_PORT/$MODULE/$lsn/$sst_ver" } check_extra() @@ -745,7 +751,7 @@ recv_joiner() set +e if [ $wait -ne 0 ]; then - wait_for_listen "$SST_PORT" "$ADDR" "$MODULE" & + wait_for_listen & fi timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -1206,7 +1212,7 @@ then stagemsg='Joiner-Recv' - MODULE='xtrabackup_sst' + MODULE="${WSREP_SST_OPT_MODULE:-xtrabackup_sst}" [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" @@ -1214,7 +1220,7 @@ then [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" - ADDR="$WSREP_SST_OPT_ADDR" + ADDR="$WSREP_SST_OPT_HOST" if [ "${tmode#VERIFY}" != "$tmode" ]; then # backward-incompatible behavior: @@ -1228,7 +1234,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)"