From d5ceddb39140b34dd6fc9ac68ef3f5f17cae857d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub <wlad@mariadb.com> Date: Tue, 7 Dec 2021 01:28:51 +0100 Subject: [PATCH 01/21] Appveyor - cache chocolatey packages --- appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 9797aec6b44..e8902eec589 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,6 +6,10 @@ init: version: build-{build}~branch-{branch} +cache: + - C:\ProgramData\chocolatey\bin -> appveyor.yml + - C:\ProgramData\chocolatey\lib -> appveyor.yml + clone_depth: 1 build_script: From 375ae890c76228c1f827bdc8f7684e81d5414466 Mon Sep 17 00:00:00 2001 From: forkfun <alice.sherepa@gmail.com> Date: Tue, 7 Dec 2021 15:25:43 +0100 Subject: [PATCH 02/21] enable rpl_semi_sync_after_sync and rpl_semi_sync_slave_compressed_protocol tests --- mysql-test/suite/rpl/disabled.def | 3 --- 1 file changed, 3 deletions(-) diff --git a/mysql-test/suite/rpl/disabled.def b/mysql-test/suite/rpl/disabled.def index 94073575606..5f5dc459ffa 100644 --- a/mysql-test/suite/rpl/disabled.def +++ b/mysql-test/suite/rpl/disabled.def @@ -10,9 +10,6 @@ # ############################################################################## -#rpl_get_master_version_and_clock : Bug#11766137 Jan 05 2011 joro Valgrind warnings rpl_partition_archive : MDEV-5077 2013-09-27 svoj Cannot exchange partition with archive table rpl_row_binlog_max_cache_size : MDEV-11092 rpl_row_index_choice : MDEV-11666 -rpl_semi_sync_after_sync : fails after MDEV-16172 -rpl_semi_sync_slave_compressed_protocol : MDEV-25580 2021-05-05 Sujatha From 8dd1f01d0933596f3dde87f32b39bebca5aa1bff Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub <wlad@mariadb.com> Date: Tue, 7 Dec 2021 17:42:47 +0100 Subject: [PATCH 03/21] MDEV-27191 MariaDB client - "system" command does not work on Windows - define USE_POPEN, like it is done elsewhere. - use Notepad as default editor on Windows for the "edit" command. --- client/mysql.cc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/client/mysql.cc b/client/mysql.cc index 177d0f60757..382c216f772 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -96,8 +96,8 @@ extern "C" { # include <readline.h> # endif #define HAVE_READLINE -#define USE_POPEN #endif +#define USE_POPEN } #ifdef HAVE_VIDATTR @@ -4210,11 +4210,6 @@ com_nopager(String *buffer __attribute__((unused)), } #endif - -/* - Sorry, you can't send the result to an editor in Win32 -*/ - #ifdef USE_POPEN static int com_edit(String *buffer,char *line __attribute__((unused))) @@ -4236,7 +4231,7 @@ com_edit(String *buffer,char *line __attribute__((unused))) if (!(editor = (char *)getenv("EDITOR")) && !(editor = (char *)getenv("VISUAL"))) - editor = "vi"; + editor = IF_WIN("notepad","vi"); strxmov(buff,editor," ",filename,NullS); if ((error= system(buff))) { @@ -4251,7 +4246,7 @@ com_edit(String *buffer,char *line __attribute__((unused))) if ((fd = my_open(filename,O_RDONLY, MYF(MY_WME))) < 0) goto err; (void) buffer->alloc((uint) stat_arg.st_size); - if ((tmp=read(fd,(char*) buffer->ptr(),buffer->alloced_length())) >= 0L) + if ((tmp=(int)my_read(fd,(uchar*) buffer->ptr(),buffer->alloced_length(),MYF(0))) >= 0) buffer->length((uint) tmp); else buffer->length(0); From f13c2107b39a6dd1e52bb582545631a4d93f2681 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub <wlad@mariadb.com> Date: Tue, 7 Dec 2021 17:46:29 +0100 Subject: [PATCH 04/21] Don't beep in mysql_upgrade_service.exe This beep looks especially strange, as mysqladmin output is redirected to the log file --- sql/mysql_upgrade_service.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/mysql_upgrade_service.cc b/sql/mysql_upgrade_service.cc index 5afe4ccbc52..78302520f82 100644 --- a/sql/mysql_upgrade_service.cc +++ b/sql/mysql_upgrade_service.cc @@ -464,8 +464,8 @@ int main(int argc, char **argv) if (WaitForSingleObject(mysqld_process, 0) != WAIT_TIMEOUT) die("mysqld.exe did not start"); - if (run_tool(P_WAIT, mysqladmin_path, "--protocol=pipe", - socket_param, "ping", NULL) == 0) + if (run_tool(P_WAIT, mysqladmin_path, "--protocol=pipe", socket_param, + "ping", "--no-beep", NULL) == 0) { break; } From f974062c519e95a00dbd5a1711e2bb95490eb4dc Mon Sep 17 00:00:00 2001 From: Sergei Krivonos <sergeikrivonos@gmail.com> Date: Tue, 7 Dec 2021 19:34:31 +0200 Subject: [PATCH 05/21] MDEV-19129: Fixed configure for Xcode, CMake generate CMake Error in wsrep-lib/CMakeLists.txt: The custom command generating /Users/name/build/mariadb-server/sql/lex_token.h is attached to multiple targets: GenServerSource sql but none of these is a common dependency of the other(s). This is not allowed by the Xcode "new build system". --- sql/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 17779adcff8..47716274966 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -351,6 +351,7 @@ ADD_CUSTOM_TARGET( GenServerSource DEPENDS ${GEN_SOURCES} ) +ADD_DEPENDENCIES(sql GenServerSource) #Need this only for embedded SET_TARGET_PROPERTIES(GenServerSource PROPERTIES EXCLUDE_FROM_ALL TRUE) From eafa2a1411d373fc21e82cc49cc1e6d7a33c127a Mon Sep 17 00:00:00 2001 From: forkfun <alice.sherepa@gmail.com> Date: Thu, 9 Dec 2021 16:29:22 +0100 Subject: [PATCH 06/21] enable partition_open_files_limit test --- mysql-test/disabled.def | 1 - mysql-test/r/partition_open_files_limit.result | 2 ++ mysql-test/suite/binlog/t/binlog_sf.test | 8 ++------ mysql-test/t/compound.test | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/mysql-test/disabled.def b/mysql-test/disabled.def index 8ba5b271c2b..6ae67e00eca 100644 --- a/mysql-test/disabled.def +++ b/mysql-test/disabled.def @@ -16,4 +16,3 @@ mysql_embedded : Bug#12561297 2011-05-14 Anitha Dependent on PB2 chang #show_explain : Psergey: random timeout in range-checked-for-each record query. file_contents : MDEV-6526 these files are not installed anymore max_statement_time : cannot possibly work, depends on timing -partition_open_files_limit : open_files_limit check broken by MDEV-18360 diff --git a/mysql-test/r/partition_open_files_limit.result b/mysql-test/r/partition_open_files_limit.result index fed32a69c44..327fe24d27d 100644 --- a/mysql-test/r/partition_open_files_limit.result +++ b/mysql-test/r/partition_open_files_limit.result @@ -1,4 +1,6 @@ DROP TABLE IF EXISTS `t1`; +call mtr.add_suppression("option 'table_open_cache'"); +call mtr.add_suppression("option 'max_connections'"); # Bug#46922: crash when adding partitions and open_files_limit is reached CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=MyISAM PARTITION BY KEY () PARTITIONS 1; diff --git a/mysql-test/suite/binlog/t/binlog_sf.test b/mysql-test/suite/binlog/t/binlog_sf.test index 05b31afcb58..fecc4736972 100644 --- a/mysql-test/suite/binlog/t/binlog_sf.test +++ b/mysql-test/suite/binlog/t/binlog_sf.test @@ -1,4 +1,3 @@ --- source include/have_log_bin.inc # We change binlog format inside the test, so no need to re-run with # more than one binlog_format. @@ -9,13 +8,9 @@ # save status -let $oblf=`select @@SESSION.BINLOG_FORMAT`; let $otfc=`select @@log_bin_trust_function_creators`; - set global log_bin_trust_function_creators=0; - - # fail *on definition* set binlog_format=STATEMENT; @@ -186,6 +181,7 @@ drop function fn16456; # restore status --disable_query_log -eval set binlog_format=$oblf; +set binlog_format=STATEMENT; eval set global log_bin_trust_function_creators=$otfc; +reset master; --enable_query_log diff --git a/mysql-test/t/compound.test b/mysql-test/t/compound.test index 7583bc1cca9..046a9b9767e 100644 --- a/mysql-test/t/compound.test +++ b/mysql-test/t/compound.test @@ -1,7 +1,7 @@ # # MDEV-5317 Compound statement / anonymous blocks # -source include/have_log_bin.inc; +source include/have_binlog_format_mixed_or_statement.inc; delimiter |; CREATE TABLE t1 (a INT PRIMARY KEY)| From 0745db71798ea7179140f4ffe85d6d69c9ba7da6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik <serg@mariadb.org> Date: Thu, 9 Dec 2021 19:14:17 +0100 Subject: [PATCH 07/21] don't use buffered_option_error_reporter without perfschema it's not printed, not cleaned up without perfschema, so isn't supposed to be written into either this fixes "Memory not freed" warnings when early command line options produce warnings in non-perfschema builds --- sql/mysqld.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index d7a44138780..6a7ea117c84 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1392,12 +1392,13 @@ static MYSQL_SOCKET unix_sock, base_ip_sock, extra_ip_sock; struct my_rnd_struct sql_rand; ///< used by sql_class.cc:THD::THD() #ifndef EMBEDDED_LIBRARY +C_MODE_START +#ifdef WITH_PERFSCHEMA_STORAGE_ENGINE /** Error reporter that buffer log messages. @param level log message level @param format log message format string */ -C_MODE_START static void buffered_option_error_reporter(enum loglevel level, const char *format, ...) { @@ -1409,6 +1410,7 @@ static void buffered_option_error_reporter(enum loglevel level, va_end(args); buffered_logs.buffer(level, buffer); } +#endif /** @@ -5809,7 +5811,7 @@ int mysqld_main(int argc, char **argv) Initialize the array of performance schema instrument configurations. */ init_pfs_instrument_array(); -#endif /* WITH_PERFSCHEMA_STORAGE_ENGINE */ + /* Logs generated while parsing the command line options are buffered and printed later. @@ -5817,7 +5819,7 @@ int mysqld_main(int argc, char **argv) buffered_logs.init(); my_getopt_error_reporter= buffered_option_error_reporter; my_charset_error_reporter= buffered_option_error_reporter; -#ifdef WITH_PERFSCHEMA_STORAGE_ENGINE + pfs_param.m_pfs_instrument= const_cast<char*>(""); #endif /* WITH_PERFSCHEMA_STORAGE_ENGINE */ my_timer_init(&sys_timer_info); From 6b066ec33285c089ce770c2d1eb50bc49787ee92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com> Date: Mon, 13 Dec 2021 08:04:45 +0200 Subject: [PATCH 08/21] MDEV-27235: Crash on SET GLOBAL innodb_encrypt_tables fil_crypt_set_encrypt_tables(): If no encryption threads have been initialized, do nothing. --- mysql-test/suite/innodb/r/temporary_table.result | 1 + mysql-test/suite/innodb/t/temporary_table.test | 2 ++ storage/innobase/fil/fil0crypt.cc | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/mysql-test/suite/innodb/r/temporary_table.result b/mysql-test/suite/innodb/r/temporary_table.result index 0307ba88200..9a11b7561af 100644 --- a/mysql-test/suite/innodb/r/temporary_table.result +++ b/mysql-test/suite/innodb/r/temporary_table.result @@ -136,6 +136,7 @@ show tables; Tables_in_test create temporary table t1 (keyc int, c1 char(100), c2 char(100)) engine = innodb; ERROR HY000: Can't create table `test`.`t1` (errno: 165 "Table is read only") +SET GLOBAL innodb_encrypt_tables=DEFAULT; # test various bad start-up parameters FOUND 2 /InnoDB: Unable to create temporary file/ in mysqld.1.err FOUND 1 /innodb_temporary and innodb_system file names seem to be the same/ in mysqld.1.err diff --git a/mysql-test/suite/innodb/t/temporary_table.test b/mysql-test/suite/innodb/t/temporary_table.test index d7fe66e2efb..46ac867c1c2 100644 --- a/mysql-test/suite/innodb/t/temporary_table.test +++ b/mysql-test/suite/innodb/t/temporary_table.test @@ -121,6 +121,8 @@ show tables; --error ER_CANT_CREATE_TABLE create temporary table t1 (keyc int, c1 char(100), c2 char(100)) engine = innodb; +SET GLOBAL innodb_encrypt_tables=DEFAULT; + --echo # test various bad start-up parameters let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err; diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 46fc12ee6b0..a8ffa3e2578 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2470,6 +2470,10 @@ void fil_crypt_set_encrypt_tables( uint val) { + if (!fil_crypt_threads_inited) { + return; + } + mutex_enter(&fil_system->mutex); srv_encrypt_tables = val; From 8bb55633699612279744c055e22eeca8d4058273 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky <julius.goryavsky@mariadb.com> Date: Mon, 13 Dec 2021 02:15:57 +0100 Subject: [PATCH 09/21] MDEV-27181: Galera SST scripts should use ssl_capath for CA directory 1. Galera SST scripts should use ssl_capath (not ssl_ca) for CA directory. The current implementation tries to automatically detect the path using the trailing slash in the ssl_ca variable value, but this approach is not compatible with the server configuration. Now, by analogy with the server, SST scripts also use a separate ssl_capath variable. In addition, a similar tcapath variable has been added for the old-style configuration (in the "sst" section). 2. Openssl utility detection made more reliable. 3. Removed extra spaces in automatically generated command lines - to simplify debugging of the SST scripts. 4. In general, the code for detecting the presence or absence of auxiliary utilities has been improved - it is made more reliable in some configurations (and for shells other than bash). --- mysql-test/std_data/capath/3106f582.0 | 1 + mysql-test/std_data/capath/cacert.pem | 79 ++++ mysql-test/std_data/capath/ed1f42db.0 | 1 + mysql-test/suite/galera/disabled.def | 1 - ...alera_sst_rsync_encrypt_with_capath.result | 398 ++++++++++++++++++ .../galera_sst_rsync_encrypt_with_key.result | 2 - ...alera_sst_rsync_encrypt_with_server.result | 2 - .../galera_sst_mariabackup_table_options.test | 1 + .../galera_sst_rsync_encrypt_with_capath.cnf | 20 + .../galera_sst_rsync_encrypt_with_capath.test | 26 ++ .../t/galera_sst_rsync_encrypt_with_key.test | 3 - .../galera_sst_rsync_encrypt_with_server.test | 3 - .../t/galera_ipv6_mariabackup.test | 1 + .../t/galera_ipv6_mariabackup_section.test | 1 + scripts/wsrep_sst_common.sh | 142 ++++--- scripts/wsrep_sst_mariabackup.sh | 292 +++++++------ scripts/wsrep_sst_mysqldump.sh | 11 +- scripts/wsrep_sst_rsync.sh | 117 ++--- scripts/wsrep_sst_xtrabackup-v2.sh | 321 +++++++------- scripts/wsrep_sst_xtrabackup.sh | 5 +- vio/viosslfactories.c | 17 + 21 files changed, 1033 insertions(+), 411 deletions(-) create mode 120000 mysql-test/std_data/capath/3106f582.0 create mode 100644 mysql-test/std_data/capath/cacert.pem create mode 120000 mysql-test/std_data/capath/ed1f42db.0 create mode 100644 mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_capath.result create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.test diff --git a/mysql-test/std_data/capath/3106f582.0 b/mysql-test/std_data/capath/3106f582.0 new file mode 120000 index 00000000000..1310cfcff20 --- /dev/null +++ b/mysql-test/std_data/capath/3106f582.0 @@ -0,0 +1 @@ +cacert.pem \ No newline at end of file diff --git a/mysql-test/std_data/capath/cacert.pem b/mysql-test/std_data/capath/cacert.pem new file mode 100644 index 00000000000..23dda2318e1 --- /dev/null +++ b/mysql-test/std_data/capath/cacert.pem @@ -0,0 +1,79 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + d0:4d:23:85:ee:59:b3:fa + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB + Validity + Not Before: Jan 27 10:11:10 2019 GMT + Not After : Jan 22 10:11:10 2039 GMT + Subject: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e8:0e:a7:84:d3:75:30:06:30:b2:10:b9:d1:88: + 36:2b:5e:f8:c8:44:57:cb:67:72:ab:96:95:33:d5: + 88:d1:8f:23:50:98:ba:6d:20:00:80:bd:35:d5:c1: + bf:98:49:c4:0a:15:4a:34:a6:21:9b:2e:8c:15:09: + f0:63:81:02:c2:7c:e2:53:e0:f7:a1:1a:40:5e:8f: + 41:4a:4c:56:d4:20:f1:d5:a7:c1:53:2e:ff:7e:37: + 17:cc:7e:74:bd:e2:22:33:ce:8c:77:62:a4:c5:3f: + 44:35:7b:7e:b9:f5:7d:8c:7a:27:58:fd:2c:42:86: + 2e:e7:6b:01:99:7b:fe:7d:a7:a1:4f:3e:39:39:54: + 1f:61:de:74:66:d1:77:4f:43:1b:66:70:29:85:de: + fc:8f:8e:1b:7b:a2:66:48:26:7f:9b:a6:fd:4a:e4: + dc:eb:ed:bd:f8:e3:f1:57:98:13:6f:f1:a3:2a:e3: + 73:bd:8d:7c:6f:4b:59:35:bc:b5:42:3e:99:a7:13: + 8d:be:2e:5c:9a:c6:5b:ab:ae:bf:00:e9:c8:ee:05: + 22:8e:d5:67:1a:47:9a:6d:9c:f9:42:3e:15:34:f8: + 31:ec:b4:7e:d3:92:95:b0:b8:f9:66:f3:bd:1d:31: + 2c:b1:90:62:a1:f8:4e:a6:5d:26:22:f0:e1:fe:16: + 2b:69 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 + X509v3 Authority Key Identifier: + keyid:CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha256WithRSAEncryption + df:fd:74:29:5b:5e:9a:8b:09:02:40:59:73:cb:71:47:3f:97: + 3d:a9:fd:c4:8c:01:29:c9:86:b8:71:55:ff:72:0e:50:dc:c8: + b5:e6:91:41:52:47:21:30:cc:4d:e7:3b:4b:db:55:ea:7d:46: + eb:53:e0:b7:1b:80:7c:b1:0c:d3:d1:bc:a0:73:ae:96:1f:fd: + 05:52:7e:54:d5:03:52:69:7b:34:5f:27:d7:98:da:98:76:73: + e6:bb:50:59:2a:94:90:67:03:1c:a4:76:2f:ee:ef:59:60:09: + 48:33:03:2b:52:ed:83:42:f8:71:19:7f:d8:be:40:ed:20:01: + 90:3c:7e:1c:8b:d2:9f:f3:2f:09:1f:50:c8:10:e1:8a:d9:a5: + 49:9c:0b:74:17:b9:2b:68:f6:1e:73:c2:73:10:38:b3:35:e2: + 87:91:1b:a1:d1:9b:81:9d:1b:32:cc:03:6e:4c:82:95:81:11: + 42:56:e2:16:2b:22:65:db:40:2c:ca:dc:03:f4:d5:07:cf:f5: + 13:b2:cf:51:5b:24:cd:c7:d1:9b:42:8e:f9:df:5d:1e:5a:09: + a3:4f:a9:0b:f4:21:c5:bb:ff:02:93:67:e8:2d:ee:ab:d9:59: + 76:03:2c:a1:bd:fb:dc:af:b6:82:94:71:85:53:a8:18:0d:3a: + 9e:42:eb:59 +-----BEGIN CERTIFICATE----- +MIIDfzCCAmegAwIBAgIJANBNI4XuWbP6MA0GCSqGSIb3DQEBCwUAMFYxDzANBgNV +BAMMBmNhY2VydDELMAkGA1UEBhMCRkkxETAPBgNVBAgMCEhlbHNpbmtpMREwDwYD +VQQHDAhIZWxzaW5raTEQMA4GA1UECgwHTWFyaWFEQjAeFw0xOTAxMjcxMDExMTBa +Fw0zOTAxMjIxMDExMTBaMFYxDzANBgNVBAMMBmNhY2VydDELMAkGA1UEBhMCRkkx +ETAPBgNVBAgMCEhlbHNpbmtpMREwDwYDVQQHDAhIZWxzaW5raTEQMA4GA1UECgwH +TWFyaWFEQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOgOp4TTdTAG +MLIQudGINite+MhEV8tncquWlTPViNGPI1CYum0gAIC9NdXBv5hJxAoVSjSmIZsu +jBUJ8GOBAsJ84lPg96EaQF6PQUpMVtQg8dWnwVMu/343F8x+dL3iIjPOjHdipMU/ +RDV7frn1fYx6J1j9LEKGLudrAZl7/n2noU8+OTlUH2HedGbRd09DG2ZwKYXe/I+O +G3uiZkgmf5um/Urk3Ovtvfjj8VeYE2/xoyrjc72NfG9LWTW8tUI+macTjb4uXJrG +W6uuvwDpyO4FIo7VZxpHmm2c+UI+FTT4Mey0ftOSlbC4+WbzvR0xLLGQYqH4TqZd +JiLw4f4WK2kCAwEAAaNQME4wHQYDVR0OBBYEFMpxmYnwcqt1ZrtlagMEcqV7laaT +MB8GA1UdIwQYMBaAFMpxmYnwcqt1ZrtlagMEcqV7laaTMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggEBAN/9dClbXpqLCQJAWXPLcUc/lz2p/cSMASnJhrhx +Vf9yDlDcyLXmkUFSRyEwzE3nO0vbVep9RutT4LcbgHyxDNPRvKBzrpYf/QVSflTV +A1JpezRfJ9eY2ph2c+a7UFkqlJBnAxykdi/u71lgCUgzAytS7YNC+HEZf9i+QO0g +AZA8fhyL0p/zLwkfUMgQ4YrZpUmcC3QXuSto9h5zwnMQOLM14oeRG6HRm4GdGzLM +A25MgpWBEUJW4hYrImXbQCzK3AP01QfP9ROyz1FbJM3H0ZtCjvnfXR5aCaNPqQv0 +IcW7/wKTZ+gt7qvZWXYDLKG9+9yvtoKUcYVTqBgNOp5C61k= +-----END CERTIFICATE----- diff --git a/mysql-test/std_data/capath/ed1f42db.0 b/mysql-test/std_data/capath/ed1f42db.0 new file mode 120000 index 00000000000..1310cfcff20 --- /dev/null +++ b/mysql-test/std_data/capath/ed1f42db.0 @@ -0,0 +1 @@ +cacert.pem \ No newline at end of file diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index d92f3f7d6b8..955a2c82ebc 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -29,5 +29,4 @@ versioning_trx_id: MDEV-18590: galera.versioning_trx_id: Test failure: mysqltest galera_wsrep_provider_unset_set: wsrep_provider is read-only for security reasons pxc-421: wsrep_provider is read-only for security reasons galera_sst_xtrabackup-v2: Test fails due to innodb issues -galera_sst_xtrabackup-v2_encrypt_with_key: Test fails due to innodb issues galera_sst_xtrabackup-v2_data_dir: Test fails due to innodb issues diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_capath.result b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_capath.result new file mode 100644 index 00000000000..170ba62dd12 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_capath.result @@ -0,0 +1,398 @@ +connection node_1; +connection node_2; +connection node_1; +Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_shutdown_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +Performing State Transfer on a server that starts from a clean var directory +This is accomplished by shutting down node #2 and removing its var directory before restarting it +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +Cleaning var directory ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_clean_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Killing server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Performing --wsrep-recover ... +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_kill_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +while a DDL was in progress on it +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +connection node_2; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; +connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +connection node_2; +SET wsrep_sync_wait = 0; +Killing server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Performing --wsrep-recover ... +connection node_2; +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_kill_slave_ddl; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +SET GLOBAL debug_dbug = $debug_orig; +include/assert_grep.inc [Using stunnel for SSL encryption] diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_key.result b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_key.result index 251c087412b..170ba62dd12 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_key.result +++ b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_key.result @@ -1,7 +1,5 @@ connection node_1; connection node_2; -connection node_2; -CALL mtr.add_suppression("\\[ERROR\\] .*ib_buffer_pool' for reading: No such file or directory"); connection node_1; Performing State Transfer on a server that has been shut down cleanly and restarted connection node_1; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_server.result b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_server.result index 251c087412b..170ba62dd12 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_server.result +++ b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_server.result @@ -1,7 +1,5 @@ connection node_1; connection node_2; -connection node_2; -CALL mtr.add_suppression("\\[ERROR\\] .*ib_buffer_pool' for reading: No such file or directory"); connection node_1; Performing State Transfer on a server that has been shut down cleanly and restarted connection node_1; diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test b/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test index 953a1635559..7a3a6c825c4 100644 --- a/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test @@ -66,6 +66,7 @@ COMMIT; --source include/wait_condition.inc --echo Cleaning var directory ... +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat --remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mtr --remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/performance_schema --remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/test diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.cnf new file mode 100644 index 00000000000..3ab762df013 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.cnf @@ -0,0 +1,20 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=rsync +ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem +ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem +ssl-capath=@ENV.MYSQL_TEST_DIR/std_data/capath +# We need to turn off the default setting for the duration +# of the test (to test working with a directory instead of +# a file): +ssl-ca= + +[sst] +ssl-mode=VERIFY_CA + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.test b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.test new file mode 100644 index 00000000000..a2d92723ec4 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.test @@ -0,0 +1,26 @@ +--source include/big_test.inc +--source include/galera_cluster.inc +--source include/have_debug.inc +--source include/have_stunnel.inc + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_1 +--source suite/galera/include/galera_st_shutdown_slave.inc +--source suite/galera/include/galera_st_clean_slave.inc + +--source suite/galera/include/galera_st_kill_slave.inc +--source suite/galera/include/galera_st_kill_slave_ddl.inc + +# Confirm that transfer was SSL-encrypted +--let $assert_text = Using stunnel for SSL encryption +--let $assert_select = Using stunnel for SSL encryption +--let $assert_count = 5 +--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err +--let $assert_only_after = CURRENT_TEST +--source include/assert_grep.inc + +--source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test index 838c473b9ce..a2d92723ec4 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test @@ -8,9 +8,6 @@ --let $node_2=node_2 --source include/auto_increment_offset_save.inc ---connection node_2 -CALL mtr.add_suppression("\\[ERROR\\] .*ib_buffer_pool' for reading: No such file or directory"); - --connection node_1 --source suite/galera/include/galera_st_shutdown_slave.inc --source suite/galera/include/galera_st_clean_slave.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test index 838c473b9ce..a2d92723ec4 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test @@ -8,9 +8,6 @@ --let $node_2=node_2 --source include/auto_increment_offset_save.inc ---connection node_2 -CALL mtr.add_suppression("\\[ERROR\\] .*ib_buffer_pool' for reading: No such file or directory"); - --connection node_1 --source suite/galera/include/galera_st_shutdown_slave.inc --source suite/galera/include/galera_st_clean_slave.inc diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test index dc3331d1be3..2bb2bb284e9 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source include/check_ipv6.inc --source include/have_innodb.inc diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test index c88ff99790c..86607e61542 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source include/check_ipv6.inc --source include/have_innodb.inc diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index dbd639595df..5c84aa7c17f 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -19,6 +19,9 @@ set -u +# Setting the path for some utilities on CentOS +export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" + WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" WSREP_SST_OPT_BINLOG_INDEX="" @@ -384,10 +387,8 @@ case "$1" in skip_mysqld_arg=1 ;; '--innodb-force-recovery') - if [ -n "$value" ]; then - if [ "$value" -ne 0 ]; then - INNODB_FORCE_RECOVERY="$value" - fi + if [ -n "$value" -a "$value" != "0" ]; then + INNODB_FORCE_RECOVERY="$value" fi skip_mysqld_arg=1 ;; @@ -584,7 +585,8 @@ get_binlog() if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then if [ -n "$WSREP_SST_OPT_PORT" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then - echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 + echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT" \ + "differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 exit 2 fi else @@ -616,8 +618,19 @@ fi readonly WSREP_SST_OPT_ADDR readonly WSREP_SST_OPT_ADDR_PORT -# try to use my_print_defaults, mysql and mysqldump that come with the sources -# (for MTR suite) +commandex() +{ + if [ -n "$BASH_VERSION" ]; then + command -v "$1" || : + elif [ -x "$1" ]; then + echo "$1" + else + which "$1" || : + fi +} + +# try to use my_print_defaults, mysql and mysqldump that come +# with the sources (for MTR suite): script_binary=$(dirname "$0") SCRIPTS_DIR=$(cd "$script_binary"; pwd -P) EXTRA_DIR="$SCRIPTS_DIR/../extra" @@ -626,13 +639,13 @@ CLIENT_DIR="$SCRIPTS_DIR/../client" if [ -x "$CLIENT_DIR/mysql" ]; then MYSQL_CLIENT="$CLIENT_DIR/mysql" else - MYSQL_CLIENT="$(command -v mysql)" + MYSQL_CLIENT=$(commandex 'mysql') fi if [ -x "$CLIENT_DIR/mysqldump" ]; then MYSQLDUMP="$CLIENT_DIR/mysqldump" else - MYSQLDUMP="$(command -v mysqldump)" + MYSQLDUMP=$(commandex 'mysqldump') fi wsrep_log() @@ -663,7 +676,7 @@ if [ -x "$SCRIPTS_DIR/my_print_defaults" ]; then elif [ -x "$EXTRA_DIR/my_print_defaults" ]; then MY_PRINT_DEFAULTS="$EXTRA_DIR/my_print_defaults" else - MY_PRINT_DEFAULTS="$(command -v my_print_defaults)" + MY_PRINT_DEFAULTS=$(commandex 'my_print_defaults') if [ -z "$MY_PRINT_DEFAULTS" ]; then wsrep_log_error "my_print_defaults not found in path" exit 2 @@ -673,16 +686,16 @@ fi readonly MY_PRINT_DEFAULTS wsrep_defaults="$WSREP_SST_OPT_DEFAULTS" -wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULTS" -wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" +wsrep_defaults="$wsrep_defaults${WSREP_SST_OPT_EXTRA_DEFAULTS:+ }$WSREP_SST_OPT_EXTRA_DEFAULTS" +wsrep_defaults="$wsrep_defaults${WSREP_SST_OPT_SUFFIX_DEFAULT:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" -readonly WSREP_SST_OPT_CONF="$wsrep_defaults" +readonly WSREP_SST_OPT_CONF="${wsrep_defaults:+ }$wsrep_defaults" wsrep_defaults="$WSREP_SST_OPT_DEFAULT" -wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULT" -wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" +wsrep_defaults="$wsrep_defaults${WSREP_SST_OPT_EXTRA_DEFAULT:+ }$WSREP_SST_OPT_EXTRA_DEFAULT" +wsrep_defaults="$wsrep_defaults${WSREP_SST_OPT_SUFFIX_DEFAULT:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" -readonly WSREP_SST_OPT_CONF_UNQUOTED="$wsrep_defaults" +readonly WSREP_SST_OPT_CONF_UNQUOTED="${wsrep_defaults:+ }$wsrep_defaults" # # User can specify mariabackup specific settings that will be used during sst @@ -819,8 +832,7 @@ if wsrep_auth_not_set; then fi # Splitting WSREP_SST_OPT_AUTH as "user:password" pair: -if ! wsrep_auth_not_set -then +if ! wsrep_auth_not_set; then # Extract username as shortest prefix up to first ':' character: WSREP_SST_OPT_AUTH_USER="${WSREP_SST_OPT_AUTH%%:*}" if [ -z "$WSREP_SST_OPT_USER" ]; then @@ -848,8 +860,7 @@ readonly WSREP_SST_OPT_USER readonly WSREP_SST_OPT_PSWD readonly WSREP_SST_OPT_AUTH -if [ -n "$WSREP_SST_OPT_REMOTE_AUTH" ] -then +if [ -n "$WSREP_SST_OPT_REMOTE_AUTH" ]; then # Split auth string at the last ':' readonly WSREP_SST_OPT_REMOTE_USER="${WSREP_SST_OPT_REMOTE_AUTH%%:*}" readonly WSREP_SST_OPT_REMOTE_PSWD="${WSREP_SST_OPT_REMOTE_AUTH#*:}" @@ -860,8 +871,7 @@ fi readonly WSREP_SST_OPT_REMOTE_AUTH -if [ -n "$WSREP_SST_OPT_DATA" ] -then +if [ -n "$WSREP_SST_OPT_DATA" ]; then SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress" else SST_PROGRESS_FILE="" @@ -870,13 +880,14 @@ fi wsrep_cleanup_progress_file() { [ -n "$SST_PROGRESS_FILE" -a \ - -f "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null || true + -f "$SST_PROGRESS_FILE" ] && \ + rm -f "$SST_PROGRESS_FILE" 2>/dev/null || : } wsrep_check_program() { local prog="$1" - local cmd=$(command -v "$prog") + local cmd=$(commandex "$prog") if [ -z "$cmd" ]; then echo "'$prog' not found in PATH" return 2 # no such file or directory @@ -898,9 +909,9 @@ wsrep_check_programs() wsrep_check_datadir() { - if [ -z "$WSREP_SST_OPT_DATA" ] - then - wsrep_log_error "The '--datadir' parameter must be passed to the SST script" + if [ -z "$WSREP_SST_OPT_DATA" ]; then + wsrep_log_error \ + "The '--datadir' parameter must be passed to the SST script" exit 2 fi } @@ -912,10 +923,10 @@ get_openssl() return fi # Let's look for openssl: - OPENSSL_BINARY="$(command -v openssl)" + OPENSSL_BINARY=$(commandex 'openssl') if [ -z "$OPENSSL_BINARY" ]; then OPENSSL_BINARY='/usr/bin/openssl' - if [ -z "$OPENSSL_BINARY" ]; then + if [ ! -x "$OPENSSL_BINARY" ]; then OPENSSL_BINARY="" fi fi @@ -928,13 +939,12 @@ get_openssl() wsrep_gen_secret() { get_openssl - if [ -n "$OPENSSL_BINARY" ] - then + if [ -n "$OPENSSL_BINARY" ]; then echo $("$OPENSSL_BINARY" rand -hex 16) else printf "%04x%04x%04x%04x%04x%04x%04x%04x" \ - $RANDOM $RANDOM $RANDOM $RANDOM \ - $RANDOM $RANDOM $RANDOM $RANDOM + $RANDOM $RANDOM $RANDOM $RANDOM \ + $RANDOM $RANDOM $RANDOM $RANDOM fi } @@ -968,14 +978,14 @@ is_local_ip() fi # Now let's check if the given address is assigned to # one of the network cards: - local ip_util="$(command -v ip)" + local ip_util=$(commandex 'ip') if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6] <address>/<mask>": "$ip_util" address show \ | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \ | grep -F " $1/" >/dev/null && return 0 else - local ifconfig_util="$(command -v ifconfig)" + local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6] <address> ...": "$ifconfig_util" \ @@ -992,15 +1002,15 @@ check_sockets_utils() sockstat_available=0 ss_available=0 - [ -n "$(command -v lsof)" ] && lsof_available=1 - [ -n "$(command -v sockstat)" ] && sockstat_available=1 - [ -n "$(command -v ss)" ] && ss_available=1 + [ -n "$(commandex lsof)" ] && lsof_available=1 + [ -n "$(commandex sockstat)" ] && sockstat_available=1 + [ -n "$(commandex ss)" ] && ss_available=1 if [ $lsof_available -eq 0 -a \ $sockstat_available -eq 0 -a \ $ss_available -eq 0 ] then - wsrep_log_error "Neither lsof, nor sockstat or ss tool was found in " \ + wsrep_log_error "Neither lsof, nor sockstat or ss tool was found in" \ "the PATH. Make sure you have it installed." exit 2 # ENOENT fi @@ -1085,26 +1095,38 @@ check_for_dhparams() # verify_ca_matches_cert() { - local ca="$1" - local cert="$2" - local path=${3:-0} + local cert="$1" + local ca="$2" + local cap="$3" # If the openssl utility is not installed, then # we will not do this certificate check: get_openssl if [ -z "$OPENSSL_BINARY" ]; then + wsrep_log_info "openssl utility not found" return fi - local not_match=0 + local readable=1; [ ! -r "$cert" ] && readable=0 + [ -n "$ca" ] && [ ! -r "$ca" ] && readable=0 + [ -n "$cap" ] && [ ! -r "$cap" ] && readable=0 - if [ $path -eq 0 ]; then - "$OPENSSL_BINARY" verify -verbose -CAfile "$ca" "$cert" >/dev/null 2>&1 || not_match=1 - else - "$OPENSSL_BINARY" verify -verbose -CApath "$ca" "$cert" >/dev/null 2>&1 || not_match=1 + if [ readable -eq 0 ]; then + wsrep_log_error \ + "Both PEM file and CA file (or path) must be readable" + exit 22 fi + local not_match=0 + local errmsg + errmsg=$("$OPENSSL_BINARY" verify -verbose \ + ${ca:+ -CAfile} ${ca:+ "$ca"} \ + ${cap:+ -CApath} ${cap:+ "$cap"} \ + "$cert" 2>&1) || not_match=1 + if [ $not_match -eq 1 ]; then + wsrep_log_info "run: \"$OPENSSL_BINARY\" verify -verbose${ca:+ -CAfile \"$ca\"}${cap:+ -CApath \"$cap\"} \"$cert\"" + wsrep_log_info "output: $errmsg" wsrep_log_error "******** FATAL ERROR ********************************************" wsrep_log_error "* The certifcate and CA (certificate authority) do not match. *" wsrep_log_error "* It does not appear that the certificate was issued by the CA. *" @@ -1124,12 +1146,18 @@ verify_ca_matches_cert() # verify_cert_matches_key() { - local cert_path="$1" - local key_path="$2" + local cert="$1" + local key="$2" + + if [ ! -r "$key" -o ! -r "$cert" ]; then + wsrep_log_error "Both the certificate file and the key file" \ + "must be readable" + exit 22 + fi # If the diff utility is not installed, then # we will not do this certificate check: - if [ -z "$(command -v diff)" ]; then + if [ -z "$(commandex diff)" ]; then return fi @@ -1142,13 +1170,13 @@ verify_cert_matches_key() # Generate the public key from the cert and the key. # They should match (otherwise we can't create an SSL connection). - if ! diff <("$OPENSSL_BINARY" x509 -in "$cert_path" -pubkey -noout 2>/dev/null) \ - <("$OPENSSL_BINARY" pkey -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1 + if ! diff <("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null) \ + <("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null) >/dev/null 2>&1 then - wsrep_log_error "******************* FATAL ERROR ****************" - wsrep_log_error "* The certifcate and private key do not match. *" - wsrep_log_error "* Please check your certificate and key files. *" - wsrep_log_error "************************************************" + wsrep_log_error "******************* FATAL ERROR *****************" + wsrep_log_error "* The certificate and private key do not match. *" + wsrep_log_error "* Please check your certificate and key files. *" + wsrep_log_error "*************************************************" exit 22 fi } @@ -1305,9 +1333,9 @@ get_proc() elif [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ]; then nproc=$(sysctl -n hw.ncpu) fi + set -e if [ -z "$nproc" ] || [ $nproc -eq 0 ]; then nproc=1 fi - set -e fi } diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index b429a9effd5..4bca785fcad 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -20,10 +20,11 @@ # https://mariadb.com/kb/en/mariabackup-overview/ # Make sure to read that before proceeding! +OS="$(uname)" + . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir -OS="$(uname)" ealgo="" eformat="" ekey="" @@ -34,7 +35,7 @@ ssyslog="" ssystag="" BACKUP_PID="" tcert="" -tpath=0 +tcap="" tpem="" tkey="" tmode="DISABLED" @@ -88,14 +89,14 @@ readonly SECRET_TAG="secret" # For backup locks it is 1 sent by joiner sst_ver=1 -if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" declare -a RC -BACKUP_BIN="$(command -v mariabackup)" -if [ ! -x "$BACKUP_BIN" ]; then +BACKUP_BIN=$(commandex 'mariabackup') +if [ -z "$BACKUP_BIN" ]; then wsrep_log_error 'mariabackup binary not found in path' exit 42 fi @@ -145,14 +146,14 @@ get_keys() if [ $encrypt -eq 0 ]; then if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then - wsrep_log_error "Options for encryption are specified, " \ + wsrep_log_error "Options for encryption are specified," \ "but encryption itself is disabled. SST may fail." fi return fi if [ $sfmt = 'tar' ]; then - wsrep_log_info "NOTE: key-based encryption (encrypt=1) " \ + wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 return @@ -165,16 +166,18 @@ get_keys() exit 3 fi - if [ -z "$ekey" -a ! -r "$ekeyfile" ]; then - wsrep_log_error "FATAL: Either key must be specified " \ - "or keyfile must be readable" - exit 3 + if [ -z "$ekey" ]; then + if [ ! -r "$ekeyfile" ]; then + wsrep_log_error "FATAL: Either key must be specified" \ + "or keyfile must be readable" + exit 3 + fi fi if [ "$eformat" = 'openssl' ]; then get_openssl if [ -z "$OPENSSL_BINARY" ]; then - wsrep_log_error "If encryption using the openssl is enabled, " \ + wsrep_log_error "If encryption using the openssl is enabled," \ "then you need to install openssl" exit 2 fi @@ -192,12 +195,12 @@ get_keys() ecmd="$ecmd -k '$ekey'" fi elif [ "$eformat" = 'xbcrypt' ]; then - if [ -z "$(command -v xbcrypt)" ]; then - wsrep_log_error "If encryption using the xbcrypt is enabled, " \ + if [ -z "$(commandex xbcrypt)" ]; then + wsrep_log_error "If encryption using the xbcrypt is enabled," \ "then you need to install xbcrypt" exit 2 fi - wsrep_log_info "NOTE: xbcrypt-based encryption, " \ + wsrep_log_info "NOTE: xbcrypt-based encryption," \ "supported only from Xtrabackup 2.1.4" if [ -z "$ekey" ]; then ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key-file='$ekeyfile'" @@ -342,40 +345,34 @@ get_transfer() CN_option=",commonname=''" if [ $encrypt -eq 2 ]; then - wsrep_log_info "Using openssl based encryption with socat: with crt and pem" - if [ -z "$tpem" -o -z "$tcert" ]; then + wsrep_log_info \ + "Using openssl based encryption with socat: with crt and pem" + if [ -z "$tpem" -o -z "$tcert$tcap" ]; then wsrep_log_error \ "Both PEM file and CRT file (or path) are required" exit 22 fi - if [ ! -r "$tpem" -o ! -r "$tcert" ]; then - wsrep_log_error \ - "Both PEM file and CRT file (or path) must be readable" - exit 22 + verify_ca_matches_cert "$tpem" "$tcert" "$tcap" + tcmd="$tcmd,cert='$tpem'" + if [ -n "$tcert" ]; then + tcmd="$tcmd,cafile='$tcert'" fi - verify_ca_matches_cert "$tcert" "$tpem" $tpath - if [ $tpath -eq 0 ]; then - tcmd="$tcmd,cert='$tpem',cafile='$tcert'" - else - tcmd="$tcmd,cert='$tpem',capath='$tcert'" + if [ -n "$tcap" ]; then + tcmd="$tcmd,capath='$tcap'" fi stagemsg="$stagemsg-OpenSSL-Encrypted-2" - wsrep_log_info "$action with cert=$tpem, ca=$tcert" + wsrep_log_info "$action with cert='$tpem', ca='$tcert', capath='$tcap'" elif [ $encrypt -eq 3 -o $encrypt -eq 4 ]; then - wsrep_log_info "Using openssl based encryption with socat: with key and crt" + wsrep_log_info \ + "Using openssl based encryption with socat: with key and crt" if [ -z "$tpem" -o -z "$tkey" ]; then - wsrep_log_error "Both certificate file (or path) " \ - "and key file are required" - exit 22 - fi - if [ ! -r "$tpem" -o ! -r "$tkey" ]; then - wsrep_log_error "Both certificate file (or path) " \ - "and key file must be readable" + wsrep_log_error "Both the certificate file (or path) and" \ + "the key file are required" exit 22 fi verify_cert_matches_key "$tpem" "$tkey" stagemsg="$stagemsg-OpenSSL-Encrypted-3" - if [ -z "$tcert" ]; then + if [ -z "$tcert$tcap" ]; then if [ $encrypt -eq 4 ]; then wsrep_log_error \ "Peer certificate file (or path) required if encrypt=4" @@ -384,14 +381,11 @@ get_transfer() # no verification CN_option="" tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0" - wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0" + wsrep_log_info \ + "$action with cert='$tpem', key='$tkey', verify=0" else # CA verification - if [ ! -r "$tcert" ]; then - wsrep_log_error "Certificate file or path must be readable" - exit 22 - fi - verify_ca_matches_cert "$tcert" "$tpem" $tpath + verify_ca_matches_cert "$tpem" "$tcert" "$tcap" if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' -o $encrypt -eq 4 ] @@ -402,12 +396,15 @@ get_transfer() else CN_option=",commonname='$WSREP_SST_OPT_HOST_UNESCAPED'" fi - if [ $tpath -eq 0 ]; then - tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'" - else - tcmd="$tcmd,cert='$tpem',key='$tkey',capath='$tcert'" + tcmd="$tcmd,cert='$tpem',key='$tkey'" + if [ -n "$tcert" ]; then + tcmd="$tcmd,cafile='$tcert'" fi - wsrep_log_info "$action with cert=$tpem, key=$tkey, ca=$tcert" + if [ -n "$tcap" ]; then + tcmd="$tcmd,capath='$tcap'" + fi + wsrep_log_info "$action with cert='$tpem', key='$tkey'," \ + "ca='$tcert', capath='$tcap'" fi else wsrep_log_info "Unknown encryption mode: encrypt=$encrypt" @@ -425,7 +422,9 @@ get_transfer() get_footprint() { pushd "$WSREP_SST_OPT_DATA" 1>/dev/null - payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }') + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' \ + -type f -print0 | du --files0-from=- --block-size=1 -c -s | \ + awk 'END { print $1 }') if [ "$compress" != 'none' ]; then # QuickLZ has around 50% compression ratio # When compression/compaction used, the progress is only an approximate. @@ -438,7 +437,7 @@ get_footprint() adjust_progress() { - if [ -z "$(command -v pv)" ]; then + if [ -z "$(commandex pv)" ]; then wsrep_log_error "pv not found in path: $PATH" wsrep_log_error "Disabling all progress/rate-limiting" pcmd="" @@ -470,33 +469,42 @@ check_server_ssl_config() { # backward-compatible behavior: tcert=$(parse_cnf 'sst' 'tca') + tcap=$(parse_cnf 'sst' 'tcapath') tpem=$(parse_cnf 'sst' 'tcert') tkey=$(parse_cnf 'sst' 'tkey') # reading new ssl configuration options: local tcert2=$(parse_cnf "$encgroups" 'ssl-ca') + local tcap2=$(parse_cnf "$encgroups" 'ssl-capath') local tpem2=$(parse_cnf "$encgroups" 'ssl-cert') local tkey2=$(parse_cnf "$encgroups" 'ssl-key') # if there are no old options, then we take new ones: - if [ -z "$tcert" -a -z "$tpem" -a -z "$tkey" ]; then + if [ -z "$tcert" -a -z "$tcap" -a -z "$tpem" -a -z "$tkey" ]; then tcert="$tcert2" + tcap="$tcap2" tpem="$tpem2" tkey="$tkey2" # checking for presence of the new-style SSL configuration: - elif [ -n "$tcert2" -o -n "$tpem2" -o -n "$tkey2" ]; then + elif [ -n "$tcert2" -o -n "$tcap2" -o -n "$tpem2" -o -n "$tkey2" ]; then if [ "$tcert" != "$tcert2" -o \ + "$tcap" != "$tcap2" -o \ "$tpem" != "$tpem2" -o \ "$tkey" != "$tkey2" ] then - wsrep_log_info "new ssl configuration options (ssl-ca, ssl-cert " \ - "and ssl-key) are ignored by SST due to presence " \ - "of the tca, tcert and/or tkey in the [sst] section" + wsrep_log_info \ + "new ssl configuration options (ssl-ca[path], ssl-cert" \ + "and ssl-key) are ignored by SST due to presence" \ + "of the tca[path], tcert and/or tkey in the [sst] section" fi fi if [ -n "$tcert" ]; then - tcert=$(trim_string "$tcert") - if [ "${tcert%/}" != "$tcert" ]; then - tpath=1 - fi + tcert=$(trim_string "$tcert") + if [ "${tcert%/}" != "$tcert" ] || [ -d "$tcert" ]; then + tcap="$tcert" + tcert="" + fi + fi + if [ -n "$tcap" ]; then + tcap=$(trim_string "$tcap") fi } @@ -506,10 +514,10 @@ read_cnf() tfmt=$(parse_cnf sst transferfmt 'socat') encrypt=$(parse_cnf "$encgroups" 'encrypt' 0) - tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | tr [:lower:] [:upper:]) + tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | \ + tr [:lower:] [:upper:]) - if [ $encrypt -eq 0 -o $encrypt -ge 2 ] - then + if [ $encrypt -eq 0 -o $encrypt -ge 2 ]; then if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then check_server_ssl_config fi @@ -517,11 +525,13 @@ read_cnf() if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ] then encrypt=3 # enable cert/key SSL encyption - # avoid CA verification if not set explicitly: - # nodes may happen to have different CA if self-generated - # zeroing up tcert does the trick - [ "${tmode#VERIFY}" != "$tmode" ] || tcert="" + # nodes may happen to have different CA if self-generated, + # zeroing up tcert and tcap does the trick: + if [ "${tmode#VERIFY}" = "$tmode" ]; then + tcert="" + tcap="" + fi fi fi elif [ $encrypt -eq 1 ]; then @@ -535,8 +545,9 @@ read_cnf() fi fi - wsrep_log_info "SSL configuration: CA='$tcert', CERT='$tpem'," \ - "KEY='$tkey', MODE='$tmode', encrypt='$encrypt'" + wsrep_log_info "SSL configuration: CA='$tcert', CAPATH='$tcap'," \ + "CERT='$tpem', KEY='$tkey', MODE='$tmode'," \ + "encrypt='$encrypt'" sockopt=$(parse_cnf sst sockopt "") progress=$(parse_cnf sst progress "") @@ -561,7 +572,8 @@ read_cnf() sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir '/tmp/sst_log_archive') if [ $speciald -eq 0 ]; then - wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1" + wsrep_log_error \ + "sst-special-dirs equal to 0 is not supported, falling back to 1" speciald=1 fi @@ -589,7 +601,7 @@ get_stream() { if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then sfmt='mbstream' - STREAM_BIN="$(command -v mbstream)" + local STREAM_BIN=$(commandex "$sfmt") if [ -z "$STREAM_BIN" ]; then wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path" exit 42 @@ -621,7 +633,7 @@ cleanup_at_exit() # Since this is invoked just after exit NNN local estatus=$? if [ $estatus -ne 0 ]; then - wsrep_log_error "Cleanup after exit with status:$estatus" + wsrep_log_error "Cleanup after exit with status: $estatus" fi if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then @@ -630,7 +642,8 @@ cleanup_at_exit() else if [ -n "$BACKUP_PID" ]; then if check_pid "$BACKUP_PID" 1; then - wsrep_log_error "mariabackup process is still running. Killing..." + wsrep_log_error \ + "mariabackup process is still running. Killing..." cleanup_pid $CHECK_PID "$BACKUP_PID" fi fi @@ -638,8 +651,8 @@ cleanup_at_exit() fi if [ -n "$progress" -a -p "$progress" ]; then - wsrep_log_info "Cleaning up fifo file $progress" - rm -f "$progress" || true + wsrep_log_info "Cleaning up fifo file: $progress" + rm -f "$progress" || : fi wsrep_log_info "Cleaning up temporary directories" @@ -649,8 +662,8 @@ cleanup_at_exit() [ -d "$STATDIR" ] && rm -rf "$STATDIR" fi else - [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true - [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true + [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || : + [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || : fi # Final cleanup @@ -662,7 +675,7 @@ cleanup_at_exit() # This means a signal was delivered to the process. # So, more cleanup. if [ $estatus -ge 128 ]; then - kill -KILL -- -$$ || true + kill -KILL -- -$$ || : fi fi @@ -738,7 +751,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then - if [ -n "$(command -v timeout)" ]; then + if [ -n "$(commandex timeout)" ]; then if timeout --help | grep -qw -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else @@ -760,14 +773,14 @@ recv_joiner() popd 1>/dev/null if [ ${RC[0]} -eq 124 ]; then - wsrep_log_error "Possible timeout in receiving first data from " \ + wsrep_log_error "Possible timeout in receiving first data from" \ "donor in gtid stage: exit codes: ${RC[@]}" exit 32 fi for ecode in "${RC[@]}"; do if [ $ecode -ne 0 ]; then - wsrep_log_error "Error while getting data from donor node: " \ + wsrep_log_error "Error while getting data from donor node:" \ "exit codes: ${RC[@]}" exit 32 fi @@ -776,7 +789,7 @@ recv_joiner() if [ $checkf -eq 1 ]; then if [ ! -r "$MAGIC_FILE" ]; then # this message should cause joiner to abort - wsrep_log_error "receiving process ended without creating " \ + wsrep_log_error "receiving process ended without creating" \ "'$MAGIC_FILE'" wsrep_log_info "Contents of datadir" wsrep_log_info $(ls -l "$dir/"*) @@ -784,10 +797,11 @@ recv_joiner() fi # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2) + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" exit 32 fi @@ -810,7 +824,7 @@ send_donor() for ecode in "${RC[@]}"; do if [ $ecode -ne 0 ]; then - wsrep_log_error "Error while sending data to joiner node: " \ + wsrep_log_error "Error while sending data to joiner node:" \ "exit codes: ${RC[@]}" exit 32 fi @@ -823,7 +837,9 @@ monitor_process() while true ; do if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then - wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." + wsrep_log_error \ + "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ + "terminated unexpectedly." kill -- -"$WSREP_SST_OPT_PARENT" exit 32 fi @@ -845,7 +861,7 @@ read_cnf setup_ports if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then - disver='--no-version-check' + disver=' --no-version-check' fi # if no command line argument and INNODB_DATA_HOME_DIR environment variable @@ -867,7 +883,7 @@ INNODB_DATA_HOME_DIR=$(pwd -P) cd "$OLD_PWD" if [ $ssyslog -eq 1 ]; then - if [ -n "$(command -v logger)" ]; then + if [ -n "$(commandex logger)" ]; then wsrep_log_info "Logging all stderr of SST/mariabackup to syslog" exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) @@ -898,10 +914,8 @@ else fi fi - if [ -e "$INNOAPPLYLOG" ] - then - if [ -n "$sstlogarchivedir" ] - then + if [ -e "$INNOAPPLYLOG" ]; then + if [ -n "$sstlogarchivedir" ]; then newfile=$(basename "$INNOAPPLYLOG") newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" else @@ -912,10 +926,8 @@ else gzip "$newfile" fi - if [ -e "$INNOMOVELOG" ] - then - if [ -n "$sstlogarchivedir" ] - then + if [ -e "$INNOMOVELOG" ]; then + if [ -n "$sstlogarchivedir" ]; then newfile=$(basename "$INNOMOVELOG") newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" else @@ -926,10 +938,8 @@ else gzip "$newfile" fi - if [ -e "$INNOBACKUPLOG" ] - then - if [ -n "$sstlogarchivedir" ] - then + if [ -e "$INNOBACKUPLOG" ]; then + if [ -n "$sstlogarchivedir" ]; then newfile=$(basename "$INNOBACKUPLOG") newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" else @@ -949,15 +959,15 @@ setup_commands() { local mysqld_args="" if [ -n "$WSREP_SST_OPT_MYSQLD" ]; then - mysqld_args="--mysqld-args $WSREP_SST_OPT_MYSQLD" + mysqld_args=" --mysqld-args $WSREP_SST_OPT_MYSQLD" fi - if [ -z "$INNODB_FORCE_RECOVERY" ]; then - INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY" - else - INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --innodb-force-recovery=$INNODB_FORCE_RECOVERY --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY" + local recovery="" + if [ -n "$INNODB_FORCE_RECOVERY" ]; then + recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi - INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" - INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP" + INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" + INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts $tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP" } get_stream @@ -1055,27 +1065,28 @@ then tcmd="$ecmd | $tcmd" fi - iopts="--databases-exclude='lost+found' $iopts" + iopts="--databases-exclude='lost+found'${iopts:+ }$iopts" if [ ${FORCE_FTWRL:-0} -eq 1 ]; then - wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" - iopts="--no-backup-locks $iopts" + wsrep_log_info "Forcing FTWRL due to environment variable" \ + "FORCE_FTWRL equal to $FORCE_FTWRL" + iopts="--no-backup-locks${iopts:+ }$iopts" fi # if compression is enabled for backup files, then add the # appropriate options to the mariabackup command line: if [ "$compress" != 'none' ]; then - iopts="--compress${compress:+=$compress} $iopts" + iopts="--compress${compress:+=$compress}${iopts:+ }$iopts" if [ -n "$compress_threads" ]; then - iopts="--compress-threads=$compress_threads $iopts" + iopts="--compress-threads=$compress_threads${iopts:+ }$iopts" fi if [ -n "$compress_chunk" ]; then - iopts="--compress-chunk-size=$compress_chunk $iopts" + iopts="--compress-chunk-size=$compress_chunk${iopts:+ }$iopts" fi fi if [ -n "$backup_threads" ]; then - iopts="--parallel=$backup_threads $iopts" + iopts="--parallel=$backup_threads${iopts:+ }$iopts" fi setup_commands @@ -1084,7 +1095,7 @@ then set -e if [ ${RC[0]} -ne 0 ]; then - wsrep_log_error "mariabackup finished with error: ${RC[0]}. " \ + wsrep_log_error "mariabackup finished with error: ${RC[0]}." \ "Check syslog or '$INNOBACKUPLOG' for details" exit 22 elif [ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]; then @@ -1125,7 +1136,8 @@ then elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then - [ -e "$SST_PROGRESS_FILE" ] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + [ -e "$SST_PROGRESS_FILE" ] && \ + wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" ib_home_dir="$INNODB_DATA_HOME_DIR" @@ -1146,7 +1158,7 @@ then ib_undo_dir="$INNODB_UNDO_DIR" if [ -n "$backup_threads" ]; then - impts="--parallel=$backup_threads $impts" + impts="--parallel=$backup_threads${impts:+ }$impts" fi stagemsg='Joiner-Recv' @@ -1165,15 +1177,15 @@ then ADDR="$WSREP_SST_OPT_ADDR" - if [ "${tmode#VERIFY}" != "$tmode" ] - then # backward-incompatible behavior + if [ "${tmode#VERIFY}" != "$tmode" ]; then + # backward-incompatible behavior: CN="" - if [ -n "$tpem" ] - then + if [ -n "$tpem" ]; then # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then - wsrep_log_error 'openssl not found but it is required for authentication' + wsrep_log_error \ + 'openssl not found but it is required for authentication' exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ @@ -1213,15 +1225,17 @@ then if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1 then - wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." + wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ + "terminated unexpectedly." exit 32 fi - if [ ! -r "$STATDIR/$IST_FILE" ] - then + if [ ! -r "$STATDIR/$IST_FILE" ]; then if [ -d "$DATA/.sst" ]; then - wsrep_log_info "WARNING: Stale temporary SST directory: '$DATA/.sst' from previous state transfer. Removing" + wsrep_log_info \ + "WARNING: Stale temporary SST directory:" \ + "'$DATA/.sst' from previous state transfer, removing..." rm -rf "$DATA/.sst" fi mkdir -p "$DATA/.sst" @@ -1229,17 +1243,20 @@ then jpid=$! wsrep_log_info "Proceeding with SST" - wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" + wsrep_log_info \ + "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then find -E ${ib_home_dir:+"$ib_home_dir"} \ ${ib_undo_dir:+"$ib_undo_dir"} \ ${ib_log_dir:+"$ib_log_dir"} \ - "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+ + "$DATA" -mindepth 1 -prune -regex "$cpat" \ + -o -exec rm -rfv {} 1>&2 \+ else find ${ib_home_dir:+"$ib_home_dir"} \ ${ib_undo_dir:+"$ib_undo_dir"} \ ${ib_log_dir:+"$ib_log_dir"} \ - "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+ + "$DATA" -mindepth 1 -prune -regex "$cpat" \ + -o -exec rm -rfv {} 1>&2 \+ fi get_binlog @@ -1248,9 +1265,9 @@ then binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") cd "$binlog_dir" wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || true + rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ || true + rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ || : cd "$OLD_PWD" fi @@ -1262,7 +1279,8 @@ then monitor_process $jpid if [ ! -s "$DATA/xtrabackup_checkpoints" ]; then - wsrep_log_error "xtrabackup_checkpoints missing, failed mariabackup/SST on donor" + wsrep_log_error "xtrabackup_checkpoints missing," \ + "failed mariabackup/SST on donor" exit 2 fi @@ -1277,7 +1295,7 @@ then if [ -n "$qpfiles" ]; then wsrep_log_info "Compressed qpress files found" - if [ -z "$(command -v qpress)" ]; then + if [ -z "$(commandex qpress)" ]; then wsrep_log_error "qpress utility not found in the path" exit 22 fi @@ -1300,14 +1318,17 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit "Joiner-Decompression" \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" extcode=$? if [ $extcode -eq 0 ]; then wsrep_log_info "Removing qpress files after decompression" find "$DATA" -type f -name '*.qp' -delete if [ $? -ne 0 ]; then - wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + wsrep_log_error \ + "Something went wrong with deletion of qpress files." \ + "Investigate" fi else wsrep_log_error "Decompression failed. Exit code: $extcode" @@ -1321,7 +1342,7 @@ then BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || true + mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : cd "$BINLOG_DIRNAME" for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do @@ -1336,7 +1357,8 @@ then timeit "mariabackup prepare stage" "$INNOAPPLY" if [ $? -ne 0 ]; then - wsrep_log_error "mariabackup apply finished with errors. Check syslog or '$INNOAPPLYLOG' for details" + wsrep_log_error "mariabackup apply finished with errors." \ + "Check syslog or '$INNOAPPLYLOG' for details." exit 22 fi diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 798bee1ac10..bed2cac0a9a 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -19,7 +19,6 @@ # This is a reference script for mysqldump-based state snapshot tansfer . $(dirname "$0")/wsrep_sst_common -PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin EINVAL=22 @@ -93,8 +92,7 @@ DROP PREPARE stmt;" SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" SET_WSREP_GTID_DOMAIN_ID="" -if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ] -then +if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ]; then SET_WSREP_GTID_DOMAIN_ID=" SET @val = (SELECT GLOBAL_VALUE FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES WHERE VARIABLE_NAME = 'WSREP_GTID_STRICT_MODE' AND GLOBAL_VALUE > 0); SET @stmt = IF (@val IS NOT NULL, 'SET GLOBAL WSREP_GTID_DOMAIN_ID=$WSREP_SST_OPT_GTID_DOMAIN_ID', 'SET @dummy = 0'); @@ -103,7 +101,7 @@ then DROP PREPARE stmt;" fi -MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF_UNQUOTED "\ +MYSQL="$MYSQL_CLIENT$WSREP_SST_OPT_CONF_UNQUOTED "\ "$AUTH -h$WSREP_SST_OPT_HOST_UNESCAPED "\ "-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10" @@ -125,8 +123,7 @@ SET_GTID_BINLOG_STATE="" SQL_LOG_BIN_OFF="" # Safety check -if [ ${SERVER_VERSION%%.*} -gt 5 ] -then +if [ ${SERVER_VERSION%%.*} -gt 5 ]; then # If binary logging is enabled on the joiner node, we need to copy donor's # gtid_binlog_state to joiner. In order to do that, a RESET MASTER must be # executed to erase binary logs (if any). Binary logging should also be @@ -140,7 +137,7 @@ then fi # NOTE: we don't use --routines here because we're dumping mysql.proc table -MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF_UNQUOTED $AUTH -S$WSREP_SST_OPT_SOCKET \ +MYSQLDUMP="$MYSQLDUMP$WSREP_SST_OPT_CONF_UNQUOTED $AUTH -S$WSREP_SST_OPT_SOCKET \ --add-drop-database --add-drop-table --skip-add-locks --create-options \ --disable-keys --extended-insert --skip-lock-tables --quick --set-charset \ --skip-comments --flush-privileges --all-databases --events" diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 29c9cd43470..b0cc8cb3066 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -25,9 +25,6 @@ STUNNEL_REAL_PID=0 # stunnel process id OS="$(uname)" [ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH -# Setting the path for lsof on CentOS -export PATH="/usr/sbin:/sbin:$PATH" - . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir @@ -37,7 +34,8 @@ cleanup_joiner() { local failure=0 - wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID, stunnel PID=$STUNNEL_REAL_PID" + wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID," \ + "stunnel PID=$STUNNEL_REAL_PID" if [ -n "$STUNNEL" ]; then if cleanup_pid $STUNNEL_REAL_PID "$STUNNEL_PID" "$STUNNEL_CONF"; then @@ -87,7 +85,7 @@ check_pid_and_port() if [ $lsof_available -ne 0 ]; then port_info=$(lsof -Pnl -i ":$port" 2>/dev/null | \ - grep -F '(LISTEN)') + grep -F '(LISTEN)') echo "$port_info" | \ grep -q -E "[[:space:]](\\*|\\[?::\\]?):$port[[:space:]]" && busy=1 else @@ -124,7 +122,7 @@ check_pid_and_port() fi if ! check_port "$pid" "$port" "$utils"; then - wsrep_log_error "rsync or stunnel daemon port '$port' " \ + wsrep_log_error "rsync or stunnel daemon port '$port'" \ "has been taken by another program" exit 16 # EBUSY fi @@ -223,6 +221,7 @@ FILTER="-f '- /lost+found' SSTKEY=$(parse_cnf 'sst' 'tkey') SSTCERT=$(parse_cnf 'sst' 'tcert') SSTCA=$(parse_cnf 'sst' 'tca') +SSTCAP=$(parse_cnf 'sst' 'tcapath') SST_SECTIONS="--mysqld|sst" @@ -231,28 +230,34 @@ check_server_ssl_config() SSTKEY=$(parse_cnf "$SST_SECTIONS" 'ssl-key') SSTCERT=$(parse_cnf "$SST_SECTIONS" 'ssl-cert') SSTCA=$(parse_cnf "$SST_SECTIONS" 'ssl-ca') + SSTCAP=$(parse_cnf "$SST_SECTIONS" 'ssl-capath') } SSLMODE=$(parse_cnf "$SST_SECTIONS" 'ssl-mode' | tr [:lower:] [:upper:]) # no old-style SSL config in [sst], check for new one: -if [ -z "$SSTKEY" -a -z "$SSTCERT" -a -z "$SSTCA" ]; then +if [ -z "$SSTKEY" -a -z "$SSTCERT" -a -z "$SSTCA" -a -z "$SSTCAP" ]; then check_server_ssl_config fi -SSTPATH=0 if [ -n "$SSTCA" ]; then - SSTCA=$(trim_string "$SSTCA") - if [ "${SSTCA%/}" != "$SSTCA" ]; then - SSTPATH=1 - fi + SSTCA=$(trim_string "$SSTCA") + if [ "${SSTCA%/}" != "$SSTCA" ] || [ -d "$SSTCA" ]; then + SSTCAP="$SSTCA" + SSTCA="" + fi +fi + +if [ -n "$SSTCAP" ]; then + SSTCAP=$(trim_string "$SSTCAP") fi if [ -z "$SSLMODE" ]; then # Implicit verification if CA is set and the SSL mode # is not specified by user: - if [ -n "$SSTCA" ]; then - if [ -n "$(command -v stunnel)" ]; then + if [ -n "$SSTCA$SSTCAP" ]; then + STUNNEL_BIN=$(commandex 'stunnel') + if [ -n "$STUNNEL_BIN" ]; then SSLMODE='VERIFY_CA' fi # Require SSL by default if SSL key and cert are present: @@ -265,17 +270,18 @@ if [ -n "$SSTCERT" -a -n "$SSTKEY" ]; then verify_cert_matches_key "$SSTCERT" "$SSTKEY" fi -if [ -n "$SSTCA" ]; then - if [ $SSTPATH -eq 0 ]; then +CAFILE_OPT="" +CAPATH_OPT="" +if [ -n "$SSTCA$SSTCAP" ]; then + if [ -n "$SSTCA" ]; then CAFILE_OPT="CAfile = $SSTCA" - else - CAFILE_OPT="CApath = $SSTCA" + fi + if [ -n "$SSTCAP" ]; then + CAPATH_OPT="CApath = $SSTCAP" fi if [ -n "$SSTCERT" ]; then - verify_ca_matches_cert "$SSTCA" "$SSTCERT" $SSTPATH + verify_ca_matches_cert "$SSTCERT" "$SSTCA" "$SSTCAP" fi -else - CAFILE_OPT="" fi VERIFY_OPT="" @@ -295,7 +301,7 @@ then exit 22 # EINVAL ;; esac - if [ -z "$SSTCA" ]; then + if [ -z "$SSTCA$SSTCAP" ]; then wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file or path" exit 22 # EINVAL fi @@ -318,9 +324,12 @@ fi STUNNEL="" if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then - STUNNEL_BIN="$(command -v stunnel)" + if [ -z "${STUNNEL_BIN+x}" ]; then + STUNNEL_BIN=$(commandex 'stunnel') + fi if [ -n "$STUNNEL_BIN" ]; then - wsrep_log_info "Using stunnel for SSL encryption: CA: '$SSTCA', ssl-mode='$SSLMODE'" + wsrep_log_info "Using stunnel for SSL encryption: CA: '$SSTCA'," \ + "CAPATH='$SSTCAP', ssl-mode='$SSLMODE'" STUNNEL="$STUNNEL_BIN $STUNNEL_CONF" fi fi @@ -340,6 +349,7 @@ then key = $SSTKEY cert = $SSTCERT ${CAFILE_OPT} +${CAPATH_OPT} foreground = yes pid = $STUNNEL_PID debug = warning @@ -392,8 +402,8 @@ EOF # Prepare binlog files cd "$BINLOG_DIRNAME" - binlog_files_full=$(tail -n $BINLOG_N_FILES "$WSREP_SST_OPT_BINLOG_INDEX") - + binlog_files_full=$(tail -n $BINLOG_N_FILES \ + "$WSREP_SST_OPT_BINLOG_INDEX") binlog_files="" for ii in $binlog_files_full do @@ -417,9 +427,10 @@ EOF WHOLE_FILE_OPT="--whole-file" fi - # first, the normal directories, so that we can detect incompatible protocol + # first, the normal directories, so that we can detect + # incompatible protocol: RC=0 - eval rsync ${STUNNEL:+"'--rsh=$STUNNEL'"} \ + eval rsync ${STUNNEL:+"--rsh='$STUNNEL'"} \ --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT $FILTER "'$WSREP_SST_OPT_DATA/'" \ @@ -430,8 +441,9 @@ EOF case $RC in 12) RC=71 # EPROTO wsrep_log_error \ - "rsync server on the other end has incompatible protocol. " \ - "Make sure you have the same version of rsync on all nodes." + "rsync server on the other end has incompatible" \ + "protocol. Make sure you have the same version of" \ + "rsync on all nodes." ;; 22) RC=12 # ENOMEM ;; @@ -481,9 +493,9 @@ EOF find . -maxdepth 1 -mindepth 1 -type d -not -name 'lost+found' \ -not -name '.zfs' -print0 | xargs -I{} -0 -P $backup_threads \ rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --recursive --delete --quiet \ - $WHOLE_FILE_OPT --exclude '*/ib_logfile*' --exclude '*/aria_log.*' \ + --owner --group --perms --links --specials --ignore-times \ + --inplace --recursive --delete --quiet $WHOLE_FILE_OPT \ + --exclude '*/ib_logfile*' --exclude '*/aria_log.*' \ --exclude '*/aria_log_control' "$WSREP_SST_OPT_DATA/{}/" \ "rsync://$WSREP_SST_OPT_ADDR/{}" >&2 || RC=$? @@ -514,7 +526,8 @@ EOF fi rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --archive --quiet --checksum "$MAGIC_FILE" "rsync://$WSREP_SST_OPT_ADDR" + --archive --quiet --checksum "$MAGIC_FILE" \ + "rsync://$WSREP_SST_OPT_ADDR" echo "done $STATE" @@ -546,7 +559,8 @@ then check_round=0 while check_pid "$STUNNEL_PID" 1 do - wsrep_log_info "lingering stunnel daemon found at startup, waiting for it to exit" + wsrep_log_info "Lingering stunnel daemon found at startup," \ + "waiting for it to exit" check_round=$(( check_round + 1 )) if [ $check_round -eq 10 ]; then wsrep_log_error "stunnel daemon already running." @@ -563,7 +577,8 @@ then check_round=0 while check_pid "$RSYNC_PID" 1 do - wsrep_log_info "lingering rsync daemon found at startup, waiting for it to exit" + wsrep_log_info "Lingering rsync daemon found at startup," \ + "waiting for it to exit" check_round=$(( check_round + 1 )) if [ $check_round -eq 10 ]; then wsrep_log_error "rsync daemon already running." @@ -575,9 +590,7 @@ then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - if [ -z "$STUNNEL" ]; then - [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" - fi + [ -z "$STUNNEL" ] && [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" ADDR="$WSREP_SST_OPT_ADDR" RSYNC_PORT="$WSREP_SST_OPT_PORT" @@ -628,19 +641,21 @@ EOF echo $$ > "$SST_PID" - if [ -z "$STUNNEL" ] - then - rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS & + if [ -z "$STUNNEL" ]; then + rsync --daemon --no-detach --port "$RSYNC_PORT" \ + --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS & RSYNC_REAL_PID=$! TRANSFER_REAL_PID=$RSYNC_REAL_PID TRANSFER_PID="$RSYNC_PID" else # Let's check if the path to the config file contains a space? + RSYNC_BIN=$(commandex 'rsync') if [ "${RSYNC_CONF#* }" = "$RSYNC_CONF" ]; then cat << EOF > "$STUNNEL_CONF" key = $SSTKEY cert = $SSTCERT ${CAFILE_OPT} +${CAPATH_OPT} foreground = yes pid = $STUNNEL_PID debug = warning @@ -650,17 +665,18 @@ ${CHECK_OPT} ${CHECK_OPT_LOCAL} [rsync] accept = $STUNNEL_ACCEPT -exec = $(command -v rsync) +exec = $RSYNC_BIN execargs = rsync --server --daemon --config=$RSYNC_CONF . EOF else # The path contains a space, so we will run it via # shell with "eval" command: - export RSYNC_CMD="eval $(command -v rsync) --server --daemon --config='$RSYNC_CONF' ." + export RSYNC_CMD="eval '$RSYNC_BIN' --server --daemon --config='$RSYNC_CONF' ." cat << EOF > "$STUNNEL_CONF" key = $SSTKEY cert = $SSTCERT ${CAFILE_OPT} +${CAPATH_OPT} foreground = yes pid = $STUNNEL_PID debug = warning @@ -688,7 +704,8 @@ EOF # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then - wsrep_log_error 'openssl not found but it is required for authentication' + wsrep_log_error \ + 'openssl not found but it is required for authentication' exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ @@ -703,7 +720,8 @@ EOF ADDR="$WSREP_SST_OPT_HOST" fi - until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT" + until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID \ + "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT" do sleep 0.2 done @@ -722,7 +740,7 @@ EOF if ! ps -p $MYSQLD_PID >/dev/null 2>&1 then wsrep_log_error \ - "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly." + "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly." kill -- -$MYSQLD_PID sleep 1 exit 32 @@ -768,10 +786,11 @@ EOF if [ -r "$MAGIC_FILE" ]; then if [ -n "$MY_SECRET" ]; then # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2) + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" exit 32 fi # remove secret from the magic file, and output diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 73f15e79a4c..d5c978c4147 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -17,13 +17,15 @@ # MA 02110-1335 USA. # Documentation: -# http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html +# https://mariadb.com/kb/en/mariabackup-overview/ +# https://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html # Make sure to read that before proceeding! +OS="$(uname)" + . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir -OS="$(uname)" ealgo="" eformat="" ekey="" @@ -34,7 +36,7 @@ ssyslog="" ssystag="" BACKUP_PID="" tcert="" -tpath=0 +tcap="" tpem="" tkey="" tmode="DISABLED" @@ -88,14 +90,14 @@ readonly SECRET_TAG="secret" sst_ver=-1 -if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" declare -a RC -BACKUP_BIN="$(command -v innobackupex)" -if [ ! -x "$BACKUP_BIN" ]; then +BACKUP_BIN=$(commandex 'innobackupex') +if [ -z "$BACKUP_BIN" ]; then wsrep_log_error 'innobackupex binary not found in path' exit 42 fi @@ -145,14 +147,14 @@ get_keys() if [ $encrypt -eq 0 ]; then if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then - wsrep_log_error "Options for encryption are specified, " \ + wsrep_log_error "Options for encryption are specified," \ "but encryption itself is disabled. SST may fail." fi return fi if [ $sfmt = 'tar' ]; then - wsrep_log_info "NOTE: key-based encryption (encrypt=1) " \ + wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 return @@ -165,16 +167,18 @@ get_keys() exit 3 fi - if [ -z "$ekey" -a ! -r "$ekeyfile" ]; then - wsrep_log_error "FATAL: Either key must be specified " \ - "or keyfile must be readable" - exit 3 + if [ -z "$ekey" ]; then + if [ ! -r "$ekeyfile" ]; then + wsrep_log_error "FATAL: Either key must be specified" \ + "or keyfile must be readable" + exit 3 + fi fi if [ "$eformat" = 'openssl' ]; then get_openssl if [ -z "$OPENSSL_BINARY" ]; then - wsrep_log_error "If encryption using the openssl is enabled, " \ + wsrep_log_error "If encryption using the openssl is enabled," \ "then you need to install openssl" exit 2 fi @@ -192,19 +196,19 @@ get_keys() ecmd="$ecmd -k '$ekey'" fi elif [ "$eformat" = 'xbcrypt' ]; then - if [ -z "$(command -v xbcrypt)" ]; then - wsrep_log_error "If encryption using the xbcrypt is enabled, " \ + if [ -z "$(commandex xbcrypt)" ]; then + wsrep_log_error "If encryption using the xbcrypt is enabled," \ "then you need to install xbcrypt" exit 2 fi - wsrep_log_info "NOTE: xbcrypt-based encryption, " \ + wsrep_log_info "NOTE: xbcrypt-based encryption," \ "supported only from Xtrabackup 2.1.4" if [ -z "$ekey" ]; then ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key-file='$ekeyfile'" else - wsrep_log_warning \ - "Using the 'encrypt-key' option causes the encryption key " \ - "to be set via the command-line and is considered insecure. " \ + wsrep_log_info \ + "Using the 'encrypt-key' option causes the encryption key" \ + "to be set via the command-line and is considered insecure." \ "It is recommended to use the 'encrypt-key-file' option instead." ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key='$ekey'" fi @@ -346,40 +350,34 @@ get_transfer() CN_option=",commonname=''" if [ $encrypt -eq 2 ]; then - wsrep_log_info "Using openssl based encryption with socat: with crt and pem" - if [ -z "$tpem" -o -z "$tcert" ]; then + wsrep_log_info \ + "Using openssl based encryption with socat: with crt and pem" + if [ -z "$tpem" -o -z "$tcert$tcap" ]; then wsrep_log_error \ "Both PEM file and CRT file (or path) are required" exit 22 fi - if [ ! -r "$tpem" -o ! -r "$tcert" ]; then - wsrep_log_error \ - "Both PEM file and CRT file (or path) must be readable" - exit 22 + verify_ca_matches_cert "$tpem" "$tcert" "$tcap" + tcmd="$tcmd,cert='$tpem'" + if [ -n "$tcert" ]; then + tcmd="$tcmd,cafile='$tcert'" fi - verify_ca_matches_cert "$tcert" "$tpem" $tpath - if [ $tpath -eq 0 ]; then - tcmd="$tcmd,cert='$tpem',cafile='$tcert'" - else - tcmd="$tcmd,cert='$tpem',capath='$tcert'" + if [ -n "$tcap" ]; then + tcmd="$tcmd,capath='$tcap'" fi stagemsg="$stagemsg-OpenSSL-Encrypted-2" - wsrep_log_info "$action with cert=$tpem, ca=$tcert" + wsrep_log_info "$action with cert='$tpem', ca='$tcert', capath='$tcap'" elif [ $encrypt -eq 3 -o $encrypt -eq 4 ]; then - wsrep_log_info "Using openssl based encryption with socat: with key and crt" + wsrep_log_info \ + "Using openssl based encryption with socat: with key and crt" if [ -z "$tpem" -o -z "$tkey" ]; then - wsrep_log_error "Both certificate file (or path) " \ - "and key file are required" - exit 22 - fi - if [ ! -r "$tpem" -o ! -r "$tkey" ]; then - wsrep_log_error "Both certificate file (or path) " \ - "and key file must be readable" + wsrep_log_error "Both the certificate file (or path) and" \ + "the key file are required" exit 22 fi verify_cert_matches_key "$tpem" "$tkey" stagemsg="$stagemsg-OpenSSL-Encrypted-3" - if [ -z "$tcert" ]; then + if [ -z "$tcert$tcap" ]; then if [ $encrypt -eq 4 ]; then wsrep_log_error \ "Peer certificate file (or path) required if encrypt=4" @@ -388,14 +386,11 @@ get_transfer() # no verification CN_option="" tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0" - wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0" + wsrep_log_info \ + "$action with cert='$tpem', key='$tkey', verify=0" else # CA verification - if [ ! -r "$tcert" ]; then - wsrep_log_error "Certificate file or path must be readable" - exit 22 - fi - verify_ca_matches_cert "$tcert" "$tpem" $tpath + verify_ca_matches_cert "$tpem" "$tcert" "$tcap" if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' -o $encrypt -eq 4 ] @@ -406,12 +401,15 @@ get_transfer() else CN_option=",commonname='$WSREP_SST_OPT_HOST_UNESCAPED'" fi - if [ $tpath -eq 0 ]; then - tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'" - else - tcmd="$tcmd,cert='$tpem',key='$tkey',capath='$tcert'" + tcmd="$tcmd,cert='$tpem',key='$tkey'" + if [ -n "$tcert" ]; then + tcmd="$tcmd,cafile='$tcert'" fi - wsrep_log_info "$action with cert=$tpem, key=$tkey, ca=$tcert" + if [ -n "$tcap" ]; then + tcmd="$tcmd,capath='$tcap'" + fi + wsrep_log_info "$action with cert='$tpem', key='$tkey'," \ + "ca='$tcert', capath='$tcap'" fi else wsrep_log_info "Unknown encryption mode: encrypt=$encrypt" @@ -429,7 +427,9 @@ get_transfer() get_footprint() { pushd "$WSREP_SST_OPT_DATA" 1>/dev/null - payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }') + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' \ + -type f -print0 | du --files0-from=- --block-size=1 -c -s | \ + awk 'END { print $1 }') if [ "$compress" != 'none' ]; then # QuickLZ has around 50% compression ratio # When compression/compaction used, the progress is only an approximate. @@ -442,7 +442,7 @@ get_footprint() adjust_progress() { - if [ -z "$(command -v pv)" ]; then + if [ -z "$(commandex pv)" ]; then wsrep_log_error "pv not found in path: $PATH" wsrep_log_error "Disabling all progress/rate-limiting" pcmd="" @@ -474,33 +474,42 @@ check_server_ssl_config() { # backward-compatible behavior: tcert=$(parse_cnf 'sst' 'tca') + tcap=$(parse_cnf 'sst' 'tcapath') tpem=$(parse_cnf 'sst' 'tcert') tkey=$(parse_cnf 'sst' 'tkey') # reading new ssl configuration options: local tcert2=$(parse_cnf "$encgroups" 'ssl-ca') + local tcap2=$(parse_cnf "$encgroups" 'ssl-capath') local tpem2=$(parse_cnf "$encgroups" 'ssl-cert') local tkey2=$(parse_cnf "$encgroups" 'ssl-key') # if there are no old options, then we take new ones: - if [ -z "$tcert" -a -z "$tpem" -a -z "$tkey" ]; then + if [ -z "$tcert" -a -z "$tcap" -a -z "$tpem" -a -z "$tkey" ]; then tcert="$tcert2" + tcap="$tcap2" tpem="$tpem2" tkey="$tkey2" # checking for presence of the new-style SSL configuration: - elif [ -n "$tcert2" -o -n "$tpem2" -o -n "$tkey2" ]; then + elif [ -n "$tcert2" -o -n "$tcap2" -o -n "$tpem2" -o -n "$tkey2" ]; then if [ "$tcert" != "$tcert2" -o \ + "$tcap" != "$tcap2" -o \ "$tpem" != "$tpem2" -o \ "$tkey" != "$tkey2" ] then - wsrep_log_info "new ssl configuration options (ssl-ca, ssl-cert " \ - "and ssl-key) are ignored by SST due to presence " \ - "of the tca, tcert and/or tkey in the [sst] section" + wsrep_log_info \ + "new ssl configuration options (ssl-ca[path], ssl-cert" \ + "and ssl-key) are ignored by SST due to presence" \ + "of the tca[path], tcert and/or tkey in the [sst] section" fi fi if [ -n "$tcert" ]; then - tcert=$(trim_string "$tcert") - if [ "${tcert%/}" != "$tcert" ]; then - tpath=1 - fi + tcert=$(trim_string "$tcert") + if [ "${tcert%/}" != "$tcert" ] || [ -d "$tcert" ]; then + tcap="$tcert" + tcert="" + fi + fi + if [ -n "$tcap" ]; then + tcap=$(trim_string "$tcap") fi } @@ -510,10 +519,10 @@ read_cnf() tfmt=$(parse_cnf sst transferfmt 'socat') encrypt=$(parse_cnf "$encgroups" 'encrypt' 0) - tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | tr [:lower:] [:upper:]) + tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | \ + tr [:lower:] [:upper:]) - if [ $encrypt -eq 0 -o $encrypt -ge 2 ] - then + if [ $encrypt -eq 0 -o $encrypt -ge 2 ]; then if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then check_server_ssl_config fi @@ -521,11 +530,13 @@ read_cnf() if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ] then encrypt=3 # enable cert/key SSL encyption - # avoid CA verification if not set explicitly: - # nodes may happen to have different CA if self-generated - # zeroing up tcert does the trick - [ "${tmode#VERIFY}" != "$tmode" ] || tcert="" + # nodes may happen to have different CA if self-generated, + # zeroing up tcert and tcap does the trick: + if [ "${tmode#VERIFY}" = "$tmode" ]; then + tcert="" + tcap="" + fi fi fi elif [ $encrypt -eq 1 ]; then @@ -539,8 +550,9 @@ read_cnf() fi fi - wsrep_log_info "SSL configuration: CA='$tcert', CERT='$tpem'," \ - "KEY='$tkey', MODE='$tmode', encrypt='$encrypt'" + wsrep_log_info "SSL configuration: CA='$tcert', CAPATH='$tcap'," \ + "CERT='$tpem', KEY='$tkey', MODE='$tmode'," \ + "encrypt='$encrypt'" sockopt=$(parse_cnf sst sockopt "") progress=$(parse_cnf sst progress "") @@ -566,7 +578,8 @@ read_cnf() sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir '/tmp/sst_log_archive') if [ $speciald -eq 0 ]; then - wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1" + wsrep_log_error \ + "sst-special-dirs equal to 0 is not supported, falling back to 1" speciald=1 fi @@ -593,12 +606,12 @@ read_cnf() get_stream() { if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then - STREAM_BIN=$(command -v "$sfmt") + local STREAM_BIN=$(commandex "$sfmt") if [ -z "$STREAM_BIN" ]; then if [ "$sfmt" = 'xbstream' ]; then - STREAM_BIN="$(command -v mbstream)" + STREAM_BIN=$(commandex 'mbstream') else - STREAM_BIN="$(command -v xbstream)" + STREAM_BIN=$(commandex 'xbstream') fi fi if [ -z "$STREAM_BIN" ]; then @@ -632,7 +645,7 @@ cleanup_at_exit() # Since this is invoked just after exit NNN local estatus=$? if [ $estatus -ne 0 ]; then - wsrep_log_error "Cleanup after exit with status:$estatus" + wsrep_log_error "Cleanup after exit with status: $estatus" fi if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then @@ -641,7 +654,8 @@ cleanup_at_exit() else if [ -n "$BACKUP_PID" ]; then if check_pid "$BACKUP_PID" 1; then - wsrep_log_error "xtrabackup process is still running. Killing..." + wsrep_log_error \ + "innobackupex process is still running. Killing..." cleanup_pid $CHECK_PID "$BACKUP_PID" fi fi @@ -649,8 +663,8 @@ cleanup_at_exit() fi if [ -n "$progress" -a -p "$progress" ]; then - wsrep_log_info "Cleaning up fifo file $progress" - rm -f "$progress" || true + wsrep_log_info "Cleaning up fifo file: $progress" + rm -f "$progress" || : fi wsrep_log_info "Cleaning up temporary directories" @@ -660,8 +674,8 @@ cleanup_at_exit() [ -d "$STATDIR" ] && rm -rf "$STATDIR" fi else - [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true - [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true + [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || : + [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || : fi # Final cleanup @@ -673,7 +687,7 @@ cleanup_at_exit() # This means a signal was delivered to the process. # So, more cleanup. if [ $estatus -ge 128 ]; then - kill -KILL -- -$$ || true + kill -KILL -- -$$ || : fi fi @@ -749,7 +763,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then - if [ -n "$(command -v timeout)" ]; then + if [ -n "$(commandex timeout)" ]; then if timeout --help | grep -qw -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else @@ -771,14 +785,14 @@ recv_joiner() popd 1>/dev/null if [ ${RC[0]} -eq 124 ]; then - wsrep_log_error "Possible timeout in receiving first data from " \ + wsrep_log_error "Possible timeout in receiving first data from" \ "donor in gtid stage: exit codes: ${RC[@]}" exit 32 fi for ecode in "${RC[@]}"; do if [ $ecode -ne 0 ]; then - wsrep_log_error "Error while getting data from donor node: " \ + wsrep_log_error "Error while getting data from donor node:" \ "exit codes: ${RC[@]}" exit 32 fi @@ -787,7 +801,7 @@ recv_joiner() if [ $checkf -eq 1 ]; then if [ ! -r "$MAGIC_FILE" ]; then # this message should cause joiner to abort - wsrep_log_error "receiving process ended without creating " \ + wsrep_log_error "receiving process ended without creating" \ "'$MAGIC_FILE'" wsrep_log_info "Contents of datadir" wsrep_log_info $(ls -l "$dir/"*) @@ -795,10 +809,11 @@ recv_joiner() fi # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2) + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" exit 32 fi @@ -821,7 +836,7 @@ send_donor() for ecode in "${RC[@]}"; do if [ $ecode -ne 0 ]; then - wsrep_log_error "Error while sending data to joiner node: " \ + wsrep_log_error "Error while sending data to joiner node:" \ "exit codes: ${RC[@]}" exit 32 fi @@ -834,7 +849,9 @@ monitor_process() while true ; do if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then - wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." + wsrep_log_error \ + "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ + "terminated unexpectedly." kill -- -"$WSREP_SST_OPT_PARENT" exit 32 fi @@ -850,13 +867,17 @@ monitor_process() XB_REQUIRED_VERSION="2.3.5" XB_VERSION=`$BACKUP_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` -if [[ -z "$XB_VERSION" ]]; then - wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" +if [ -z "$XB_VERSION" ]; then + wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version." \ + "Needs xtrabackup-$XB_REQUIRED_VERSION or higher to" \ + "perform SST" exit 2 fi if ! check_for_version "$XB_VERSION" "$XB_REQUIRED_VERSION"; then - wsrep_log_error "FATAL: The $BACKUP_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" + wsrep_log_error "FATAL: The $BACKUP_BIN version is $XB_VERSION." \ + "Needs xtrabackup-$XB_REQUIRED_VERSION or higher to" \ + "perform SST" exit 2 fi @@ -871,7 +892,7 @@ read_cnf setup_ports if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then - disver='--no-version-check' + disver=' --no-version-check' fi # if no command line argument and INNODB_DATA_HOME_DIR environment variable @@ -893,19 +914,19 @@ INNODB_DATA_HOME_DIR=$(pwd -P) cd "$OLD_PWD" if [ $ssyslog -eq 1 ]; then - if [ -n "$(command -v logger)" ]; then + if [ -n "$(commandex logger)" ]; then wsrep_log_info "Logging all stderr of SST/xtrabackup to syslog" exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) wsrep_log_error() { - logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" + logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" } wsrep_log_info() { - logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" + logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" } else wsrep_log_error "logger not in path: $PATH. Ignoring" @@ -924,10 +945,8 @@ else fi fi - if [ -e "$INNOAPPLYLOG" ] - then - if [ -n "$sstlogarchivedir" ] - then + if [ -e "$INNOAPPLYLOG" ]; then + if [ -n "$sstlogarchivedir" ]; then newfile=$(basename "$INNOAPPLYLOG") newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" else @@ -938,10 +957,8 @@ else gzip "$newfile" fi - if [ -e "$INNOMOVELOG" ] - then - if [ -n "$sstlogarchivedir" ] - then + if [ -e "$INNOMOVELOG" ]; then + if [ -n "$sstlogarchivedir" ]; then newfile=$(basename "$INNOMOVELOG") newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" else @@ -952,10 +969,8 @@ else gzip "$newfile" fi - if [ -e "$INNOBACKUPLOG" ] - then - if [ -n "$sstlogarchivedir" ] - then + if [ -e "$INNOBACKUPLOG" ]; then + if [ -n "$sstlogarchivedir" ]; then newfile=$(basename "$INNOBACKUPLOG") newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" else @@ -973,17 +988,17 @@ fi setup_commands() { - if [ -z "$INNODB_FORCE_RECOVERY" ]; then - INNOAPPLY="$BACKUP_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY" - else - INNOAPPLY="$BACKUP_BIN $disver $iapts $INNOEXTRA --innodb-force-recovery=$INNODB_FORCE_RECOVERY --apply-log $rebuildcmd '$DATA' $INNOAPPLY" + local recovery="" + if [ -n "$INNODB_FORCE_RECOVERY" ]; then + recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi - INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories '$DATA' $INNOMOVE" - sfmt_work="$sfmt" + INNOAPPLY="$BACKUP_BIN$disver$recovery${iapts:+ }$iapts$INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories '$DATA' $INNOMOVE" + local sfmt_work="$sfmt" if [ "$sfmt" = 'mbstream' ]; then sfmt_work='xbstream' fi - INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP" + INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF$disver${iopts:+ }$iopts $tmpopts$INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP" } get_stream @@ -1081,27 +1096,28 @@ then tcmd="$ecmd | $tcmd" fi - iopts="--databases-exclude='lost+found' $iopts" + iopts="--databases-exclude='lost+found'${iopts:+ }$iopts" if [ ${FORCE_FTWRL:-0} -eq 1 ]; then - wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" - iopts="--no-backup-locks $iopts" + wsrep_log_info "Forcing FTWRL due to environment variable" \ + "FORCE_FTWRL equal to $FORCE_FTWRL" + iopts="--no-backup-locks${iopts:+ }$iopts" fi # if compression is enabled for backup files, then add the # appropriate options to the innobackupex command line: if [ "$compress" != 'none' ]; then - iopts="--compress${compress:+=$compress} $iopts" + iopts="--compress${compress:+=$compress}${iopts:+ }$iopts" if [ -n "$compress_threads" ]; then - iopts="--compress-threads=$compress_threads $iopts" + iopts="--compress-threads=$compress_threads${iopts:+ }$iopts" fi if [ -n "$compress_chunk" ]; then - iopts="--compress-chunk-size=$compress_chunk $iopts" + iopts="--compress-chunk-size=$compress_chunk${iopts:+ }$iopts" fi fi if [ -n "$backup_threads" ]; then - iopts="--parallel=$backup_threads $iopts" + iopts="--parallel=$backup_threads${iopts:+ }$iopts" fi setup_commands @@ -1110,7 +1126,7 @@ then set -e if [ ${RC[0]} -ne 0 ]; then - wsrep_log_error "innobackupex finished with error: ${RC[0]}. " \ + wsrep_log_error "innobackupex finished with error: ${RC[0]}." \ "Check syslog or '$INNOBACKUPLOG' for details" exit 22 elif [ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]; then @@ -1151,7 +1167,8 @@ then elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then - [ -e "$SST_PROGRESS_FILE" ] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + [ -e "$SST_PROGRESS_FILE" ] && \ + wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" ib_home_dir="$INNODB_DATA_HOME_DIR" @@ -1172,7 +1189,7 @@ then ib_undo_dir="$INNODB_UNDO_DIR" if [ -n "$backup_threads" ]; then - impts="--parallel=$backup_threads $impts" + impts="--parallel=$backup_threads${impts:+ }$impts" fi stagemsg='Joiner-Recv' @@ -1191,15 +1208,15 @@ then ADDR="$WSREP_SST_OPT_ADDR" - if [ "${tmode#VERIFY}" != "$tmode" ] - then # backward-incompatible behavior + if [ "${tmode#VERIFY}" != "$tmode" ]; then + # backward-incompatible behavior: CN="" - if [ -n "$tpem" ] - then + if [ -n "$tpem" ]; then # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then - wsrep_log_error 'openssl not found but it is required for authentication' + wsrep_log_error \ + 'openssl not found but it is required for authentication' exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ @@ -1239,15 +1256,17 @@ then if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1 then - wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." + wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ + "terminated unexpectedly." exit 32 fi - if [ ! -r "$STATDIR/$IST_FILE" ] - then + if [ ! -r "$STATDIR/$IST_FILE" ]; then if [ -d "$DATA/.sst" ]; then - wsrep_log_info "WARNING: Stale temporary SST directory: '$DATA/.sst' from previous state transfer. Removing" + wsrep_log_info \ + "WARNING: Stale temporary SST directory:" \ + "'$DATA/.sst' from previous state transfer, removing..." rm -rf "$DATA/.sst" fi mkdir -p "$DATA/.sst" @@ -1255,17 +1274,20 @@ then jpid=$! wsrep_log_info "Proceeding with SST" - wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" + wsrep_log_info \ + "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then find -E ${ib_home_dir:+"$ib_home_dir"} \ ${ib_undo_dir:+"$ib_undo_dir"} \ ${ib_log_dir:+"$ib_log_dir"} \ - "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+ + "$DATA" -mindepth 1 -prune -regex "$cpat" \ + -o -exec rm -rfv {} 1>&2 \+ else find ${ib_home_dir:+"$ib_home_dir"} \ ${ib_undo_dir:+"$ib_undo_dir"} \ ${ib_log_dir:+"$ib_log_dir"} \ - "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+ + "$DATA" -mindepth 1 -prune -regex "$cpat" \ + -o -exec rm -rfv {} 1>&2 \+ fi get_binlog @@ -1274,9 +1296,9 @@ then binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") cd "$binlog_dir" wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || true + rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ || true + rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ || : cd "$OLD_PWD" fi @@ -1288,7 +1310,8 @@ then monitor_process $jpid if [ ! -s "$DATA/xtrabackup_checkpoints" ]; then - wsrep_log_error "xtrabackup_checkpoints missing, failed xtrabackup/SST on donor" + wsrep_log_error "xtrabackup_checkpoints missing," \ + "failed xtrabackup/SST on donor" exit 2 fi @@ -1305,7 +1328,7 @@ then if [ -n "$qpfiles" ]; then wsrep_log_info "Compressed qpress files found" - if [ -z "$(command -v qpress)" ]; then + if [ -z "$(commandex qpress)" ]; then wsrep_log_error "qpress utility not found in the path" exit 22 fi @@ -1328,14 +1351,17 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit "Joiner-Decompression" \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" extcode=$? if [ $extcode -eq 0 ]; then wsrep_log_info "Removing qpress files after decompression" find "$DATA" -type f -name '*.qp' -delete if [ $? -ne 0 ]; then - wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + wsrep_log_error \ + "Something went wrong with deletion of qpress files." \ + "Investigate" fi else wsrep_log_error "Decompression failed. Exit code: $extcode" @@ -1349,7 +1375,7 @@ then BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || true + mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : cd "$BINLOG_DIRNAME" for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do @@ -1364,12 +1390,11 @@ then timeit "Xtrabackup prepare stage" "$INNOAPPLY" if [ $? -ne 0 ]; then - wsrep_log_error "xtrabackup apply finished with errors. Check '$INNOAPPLYLOG' for details" + wsrep_log_error "xtrabackup apply finished with errors." \ + "Check syslog or '$INNOAPPLYLOG' for details." exit 22 fi - # [ -f "$INNOAPPLYLOG" ] && rm "$INNOAPPLYLOG" - MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh index b84aca4866e..de7b771bd2d 100644 --- a/scripts/wsrep_sst_xtrabackup.sh +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -17,7 +17,7 @@ # MA 02110-1335 USA. # Optional dependencies and options documented here: -# http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html +# https://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html # Make sure to read that before proceeding! . $(dirname "$0")/wsrep_sst_common @@ -64,9 +64,6 @@ INFO_FILE="xtrabackup_galera_info" IST_FILE="xtrabackup_ist" MAGIC_FILE="${DATA}/${INFO_FILE}" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit(){ local stage="$1" shift diff --git a/vio/viosslfactories.c b/vio/viosslfactories.c index 8ab7565a666..08f0905e044 100644 --- a/vio/viosslfactories.c +++ b/vio/viosslfactories.c @@ -178,6 +178,12 @@ new_VioSSLFd(const char *key_file, const char *cert_file, struct st_VioSSLFd *ssl_fd; long ssl_ctx_options= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3; DBUG_ENTER("new_VioSSLFd"); + + if (ca_file && ! ca_file[0]) ca_file = NULL; + if (ca_path && ! ca_path[0]) ca_path = NULL; + if (crl_file && ! crl_file[0]) crl_file = NULL; + if (crl_path && ! crl_path[0]) crl_path = NULL; + DBUG_PRINT("enter", ("key_file: '%s' cert_file: '%s' ca_file: '%s' ca_path: '%s' " "cipher: '%s' crl_file: '%s' crl_path: '%s' ", @@ -308,6 +314,11 @@ new_VioSSLConnectorFd(const char *key_file, const char *cert_file, struct st_VioSSLFd *ssl_fd; int verify= SSL_VERIFY_PEER; + if (ca_file && ! ca_file[0]) ca_file = NULL; + if (ca_path && ! ca_path[0]) ca_path = NULL; + if (crl_file && ! crl_file[0]) crl_file = NULL; + if (crl_path && ! crl_path[0]) crl_path = NULL; + /* Turn off verification of servers certificate if both ca_file and ca_path is set to NULL @@ -339,6 +350,12 @@ new_VioSSLAcceptorFd(const char *key_file, const char *cert_file, { struct st_VioSSLFd *ssl_fd; int verify= SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE; + + if (ca_file && ! ca_file[0]) ca_file = NULL; + if (ca_path && ! ca_path[0]) ca_path = NULL; + if (crl_file && ! crl_file[0]) crl_file = NULL; + if (crl_path && ! crl_path[0]) crl_path = NULL; + if (!(ssl_fd= new_VioSSLFd(key_file, cert_file, ca_file, ca_path, cipher, FALSE, error, crl_file, crl_path))) From 7bc629a5ce9e1edf2c27ddfba2a55c4341d55b4f Mon Sep 17 00:00:00 2001 From: Julius Goryavsky <julius.goryavsky@mariadb.com> Date: Mon, 13 Dec 2021 02:15:57 +0100 Subject: [PATCH 10/21] MDEV-27181: Galera SST scripts should use ssl_capath for CA directory 1. Galera SST scripts should use ssl_capath (not ssl_ca) for CA directory. The current implementation tries to automatically detect the path using the trailing slash in the ssl_ca variable value, but this approach is not compatible with the server configuration. Now, by analogy with the server, SST scripts also use a separate ssl_capath variable. In addition, a similar tcapath variable has been added for the old-style configuration (in the "sst" section). 2. Openssl utility detection made more reliable. 3. Removed extra spaces in automatically generated command lines - to simplify debugging of the SST scripts. 4. In general, the code for detecting the presence or absence of auxiliary utilities has been improved - it is made more reliable in some configurations (and for shells other than bash). --- mysql-test/std_data/capath/3106f582.0 | 1 + mysql-test/std_data/capath/cacert.pem | 79 ++++ mysql-test/std_data/capath/ed1f42db.0 | 1 + mysql-test/suite/galera/disabled.def | 1 - ...alera_sst_rsync_encrypt_with_capath.result | 398 ++++++++++++++++++ .../galera_sst_rsync_encrypt_with_key.result | 2 - ...alera_sst_rsync_encrypt_with_server.result | 2 - .../galera_sst_mariabackup_table_options.test | 1 + .../galera_sst_rsync_encrypt_with_capath.cnf | 20 + .../galera_sst_rsync_encrypt_with_capath.test | 26 ++ .../t/galera_sst_rsync_encrypt_with_key.test | 3 - .../galera_sst_rsync_encrypt_with_server.test | 3 - .../t/galera_ipv6_mariabackup.test | 1 + .../t/galera_ipv6_mariabackup_section.test | 1 + scripts/wsrep_sst_common.sh | 142 ++++--- scripts/wsrep_sst_mariabackup.sh | 292 +++++++------ scripts/wsrep_sst_mysqldump.sh | 11 +- scripts/wsrep_sst_rsync.sh | 117 ++--- vio/viosslfactories.c | 17 + 19 files changed, 859 insertions(+), 259 deletions(-) create mode 120000 mysql-test/std_data/capath/3106f582.0 create mode 100644 mysql-test/std_data/capath/cacert.pem create mode 120000 mysql-test/std_data/capath/ed1f42db.0 create mode 100644 mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_capath.result create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.test diff --git a/mysql-test/std_data/capath/3106f582.0 b/mysql-test/std_data/capath/3106f582.0 new file mode 120000 index 00000000000..1310cfcff20 --- /dev/null +++ b/mysql-test/std_data/capath/3106f582.0 @@ -0,0 +1 @@ +cacert.pem \ No newline at end of file diff --git a/mysql-test/std_data/capath/cacert.pem b/mysql-test/std_data/capath/cacert.pem new file mode 100644 index 00000000000..23dda2318e1 --- /dev/null +++ b/mysql-test/std_data/capath/cacert.pem @@ -0,0 +1,79 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + d0:4d:23:85:ee:59:b3:fa + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB + Validity + Not Before: Jan 27 10:11:10 2019 GMT + Not After : Jan 22 10:11:10 2039 GMT + Subject: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e8:0e:a7:84:d3:75:30:06:30:b2:10:b9:d1:88: + 36:2b:5e:f8:c8:44:57:cb:67:72:ab:96:95:33:d5: + 88:d1:8f:23:50:98:ba:6d:20:00:80:bd:35:d5:c1: + bf:98:49:c4:0a:15:4a:34:a6:21:9b:2e:8c:15:09: + f0:63:81:02:c2:7c:e2:53:e0:f7:a1:1a:40:5e:8f: + 41:4a:4c:56:d4:20:f1:d5:a7:c1:53:2e:ff:7e:37: + 17:cc:7e:74:bd:e2:22:33:ce:8c:77:62:a4:c5:3f: + 44:35:7b:7e:b9:f5:7d:8c:7a:27:58:fd:2c:42:86: + 2e:e7:6b:01:99:7b:fe:7d:a7:a1:4f:3e:39:39:54: + 1f:61:de:74:66:d1:77:4f:43:1b:66:70:29:85:de: + fc:8f:8e:1b:7b:a2:66:48:26:7f:9b:a6:fd:4a:e4: + dc:eb:ed:bd:f8:e3:f1:57:98:13:6f:f1:a3:2a:e3: + 73:bd:8d:7c:6f:4b:59:35:bc:b5:42:3e:99:a7:13: + 8d:be:2e:5c:9a:c6:5b:ab:ae:bf:00:e9:c8:ee:05: + 22:8e:d5:67:1a:47:9a:6d:9c:f9:42:3e:15:34:f8: + 31:ec:b4:7e:d3:92:95:b0:b8:f9:66:f3:bd:1d:31: + 2c:b1:90:62:a1:f8:4e:a6:5d:26:22:f0:e1:fe:16: + 2b:69 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 + X509v3 Authority Key Identifier: + keyid:CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha256WithRSAEncryption + df:fd:74:29:5b:5e:9a:8b:09:02:40:59:73:cb:71:47:3f:97: + 3d:a9:fd:c4:8c:01:29:c9:86:b8:71:55:ff:72:0e:50:dc:c8: + b5:e6:91:41:52:47:21:30:cc:4d:e7:3b:4b:db:55:ea:7d:46: + eb:53:e0:b7:1b:80:7c:b1:0c:d3:d1:bc:a0:73:ae:96:1f:fd: + 05:52:7e:54:d5:03:52:69:7b:34:5f:27:d7:98:da:98:76:73: + e6:bb:50:59:2a:94:90:67:03:1c:a4:76:2f:ee:ef:59:60:09: + 48:33:03:2b:52:ed:83:42:f8:71:19:7f:d8:be:40:ed:20:01: + 90:3c:7e:1c:8b:d2:9f:f3:2f:09:1f:50:c8:10:e1:8a:d9:a5: + 49:9c:0b:74:17:b9:2b:68:f6:1e:73:c2:73:10:38:b3:35:e2: + 87:91:1b:a1:d1:9b:81:9d:1b:32:cc:03:6e:4c:82:95:81:11: + 42:56:e2:16:2b:22:65:db:40:2c:ca:dc:03:f4:d5:07:cf:f5: + 13:b2:cf:51:5b:24:cd:c7:d1:9b:42:8e:f9:df:5d:1e:5a:09: + a3:4f:a9:0b:f4:21:c5:bb:ff:02:93:67:e8:2d:ee:ab:d9:59: + 76:03:2c:a1:bd:fb:dc:af:b6:82:94:71:85:53:a8:18:0d:3a: + 9e:42:eb:59 +-----BEGIN CERTIFICATE----- +MIIDfzCCAmegAwIBAgIJANBNI4XuWbP6MA0GCSqGSIb3DQEBCwUAMFYxDzANBgNV +BAMMBmNhY2VydDELMAkGA1UEBhMCRkkxETAPBgNVBAgMCEhlbHNpbmtpMREwDwYD +VQQHDAhIZWxzaW5raTEQMA4GA1UECgwHTWFyaWFEQjAeFw0xOTAxMjcxMDExMTBa +Fw0zOTAxMjIxMDExMTBaMFYxDzANBgNVBAMMBmNhY2VydDELMAkGA1UEBhMCRkkx +ETAPBgNVBAgMCEhlbHNpbmtpMREwDwYDVQQHDAhIZWxzaW5raTEQMA4GA1UECgwH +TWFyaWFEQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOgOp4TTdTAG +MLIQudGINite+MhEV8tncquWlTPViNGPI1CYum0gAIC9NdXBv5hJxAoVSjSmIZsu +jBUJ8GOBAsJ84lPg96EaQF6PQUpMVtQg8dWnwVMu/343F8x+dL3iIjPOjHdipMU/ +RDV7frn1fYx6J1j9LEKGLudrAZl7/n2noU8+OTlUH2HedGbRd09DG2ZwKYXe/I+O +G3uiZkgmf5um/Urk3Ovtvfjj8VeYE2/xoyrjc72NfG9LWTW8tUI+macTjb4uXJrG +W6uuvwDpyO4FIo7VZxpHmm2c+UI+FTT4Mey0ftOSlbC4+WbzvR0xLLGQYqH4TqZd +JiLw4f4WK2kCAwEAAaNQME4wHQYDVR0OBBYEFMpxmYnwcqt1ZrtlagMEcqV7laaT +MB8GA1UdIwQYMBaAFMpxmYnwcqt1ZrtlagMEcqV7laaTMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggEBAN/9dClbXpqLCQJAWXPLcUc/lz2p/cSMASnJhrhx +Vf9yDlDcyLXmkUFSRyEwzE3nO0vbVep9RutT4LcbgHyxDNPRvKBzrpYf/QVSflTV +A1JpezRfJ9eY2ph2c+a7UFkqlJBnAxykdi/u71lgCUgzAytS7YNC+HEZf9i+QO0g +AZA8fhyL0p/zLwkfUMgQ4YrZpUmcC3QXuSto9h5zwnMQOLM14oeRG6HRm4GdGzLM +A25MgpWBEUJW4hYrImXbQCzK3AP01QfP9ROyz1FbJM3H0ZtCjvnfXR5aCaNPqQv0 +IcW7/wKTZ+gt7qvZWXYDLKG9+9yvtoKUcYVTqBgNOp5C61k= +-----END CERTIFICATE----- diff --git a/mysql-test/std_data/capath/ed1f42db.0 b/mysql-test/std_data/capath/ed1f42db.0 new file mode 120000 index 00000000000..1310cfcff20 --- /dev/null +++ b/mysql-test/std_data/capath/ed1f42db.0 @@ -0,0 +1 @@ +cacert.pem \ No newline at end of file diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index d92f3f7d6b8..955a2c82ebc 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -29,5 +29,4 @@ versioning_trx_id: MDEV-18590: galera.versioning_trx_id: Test failure: mysqltest galera_wsrep_provider_unset_set: wsrep_provider is read-only for security reasons pxc-421: wsrep_provider is read-only for security reasons galera_sst_xtrabackup-v2: Test fails due to innodb issues -galera_sst_xtrabackup-v2_encrypt_with_key: Test fails due to innodb issues galera_sst_xtrabackup-v2_data_dir: Test fails due to innodb issues diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_capath.result b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_capath.result new file mode 100644 index 00000000000..170ba62dd12 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_capath.result @@ -0,0 +1,398 @@ +connection node_1; +connection node_2; +connection node_1; +Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_shutdown_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +Performing State Transfer on a server that starts from a clean var directory +This is accomplished by shutting down node #2 and removing its var directory before restarting it +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +Cleaning var directory ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_clean_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +Killing server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +INSERT INTO t1 VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Performing --wsrep-recover ... +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +INSERT INTO t1 VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +INSERT INTO t1 VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_kill_slave; +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +while a DDL was in progress on it +connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +connection node_2; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; +connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +connection node_2; +SET wsrep_sync_wait = 0; +Killing server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +connection node_2; +Performing --wsrep-recover ... +connection node_2; +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +COMMIT; +connection node_1a_galera_st_kill_slave_ddl; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +ROLLBACK; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +SET AUTOCOMMIT=ON; +connection node_1; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 +SELECT COUNT(*) = 35 FROM t1; +COUNT(*) = 35 +1 +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +SET AUTOCOMMIT=ON; +SET GLOBAL debug_dbug = $debug_orig; +include/assert_grep.inc [Using stunnel for SSL encryption] diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_key.result b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_key.result index 251c087412b..170ba62dd12 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_key.result +++ b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_key.result @@ -1,7 +1,5 @@ connection node_1; connection node_2; -connection node_2; -CALL mtr.add_suppression("\\[ERROR\\] .*ib_buffer_pool' for reading: No such file or directory"); connection node_1; Performing State Transfer on a server that has been shut down cleanly and restarted connection node_1; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_server.result b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_server.result index 251c087412b..170ba62dd12 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_server.result +++ b/mysql-test/suite/galera/r/galera_sst_rsync_encrypt_with_server.result @@ -1,7 +1,5 @@ connection node_1; connection node_2; -connection node_2; -CALL mtr.add_suppression("\\[ERROR\\] .*ib_buffer_pool' for reading: No such file or directory"); connection node_1; Performing State Transfer on a server that has been shut down cleanly and restarted connection node_1; diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test b/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test index 953a1635559..7a3a6c825c4 100644 --- a/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test @@ -66,6 +66,7 @@ COMMIT; --source include/wait_condition.inc --echo Cleaning var directory ... +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat --remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mtr --remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/performance_schema --remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/test diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.cnf new file mode 100644 index 00000000000..3ab762df013 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.cnf @@ -0,0 +1,20 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=rsync +ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem +ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem +ssl-capath=@ENV.MYSQL_TEST_DIR/std_data/capath +# We need to turn off the default setting for the duration +# of the test (to test working with a directory instead of +# a file): +ssl-ca= + +[sst] +ssl-mode=VERIFY_CA + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.test b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.test new file mode 100644 index 00000000000..a2d92723ec4 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_capath.test @@ -0,0 +1,26 @@ +--source include/big_test.inc +--source include/galera_cluster.inc +--source include/have_debug.inc +--source include/have_stunnel.inc + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_1 +--source suite/galera/include/galera_st_shutdown_slave.inc +--source suite/galera/include/galera_st_clean_slave.inc + +--source suite/galera/include/galera_st_kill_slave.inc +--source suite/galera/include/galera_st_kill_slave_ddl.inc + +# Confirm that transfer was SSL-encrypted +--let $assert_text = Using stunnel for SSL encryption +--let $assert_select = Using stunnel for SSL encryption +--let $assert_count = 5 +--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err +--let $assert_only_after = CURRENT_TEST +--source include/assert_grep.inc + +--source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test index 838c473b9ce..a2d92723ec4 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test @@ -8,9 +8,6 @@ --let $node_2=node_2 --source include/auto_increment_offset_save.inc ---connection node_2 -CALL mtr.add_suppression("\\[ERROR\\] .*ib_buffer_pool' for reading: No such file or directory"); - --connection node_1 --source suite/galera/include/galera_st_shutdown_slave.inc --source suite/galera/include/galera_st_clean_slave.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test index 838c473b9ce..a2d92723ec4 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test @@ -8,9 +8,6 @@ --let $node_2=node_2 --source include/auto_increment_offset_save.inc ---connection node_2 -CALL mtr.add_suppression("\\[ERROR\\] .*ib_buffer_pool' for reading: No such file or directory"); - --connection node_1 --source suite/galera/include/galera_st_shutdown_slave.inc --source suite/galera/include/galera_st_clean_slave.inc diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test index dc3331d1be3..2bb2bb284e9 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source include/check_ipv6.inc --source include/have_innodb.inc diff --git a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test index c88ff99790c..86607e61542 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test +++ b/mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup_section.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source include/check_ipv6.inc --source include/have_innodb.inc diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index dbd639595df..5c84aa7c17f 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -19,6 +19,9 @@ set -u +# Setting the path for some utilities on CentOS +export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" + WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" WSREP_SST_OPT_BINLOG_INDEX="" @@ -384,10 +387,8 @@ case "$1" in skip_mysqld_arg=1 ;; '--innodb-force-recovery') - if [ -n "$value" ]; then - if [ "$value" -ne 0 ]; then - INNODB_FORCE_RECOVERY="$value" - fi + if [ -n "$value" -a "$value" != "0" ]; then + INNODB_FORCE_RECOVERY="$value" fi skip_mysqld_arg=1 ;; @@ -584,7 +585,8 @@ get_binlog() if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then if [ -n "$WSREP_SST_OPT_PORT" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then - echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 + echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT" \ + "differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 exit 2 fi else @@ -616,8 +618,19 @@ fi readonly WSREP_SST_OPT_ADDR readonly WSREP_SST_OPT_ADDR_PORT -# try to use my_print_defaults, mysql and mysqldump that come with the sources -# (for MTR suite) +commandex() +{ + if [ -n "$BASH_VERSION" ]; then + command -v "$1" || : + elif [ -x "$1" ]; then + echo "$1" + else + which "$1" || : + fi +} + +# try to use my_print_defaults, mysql and mysqldump that come +# with the sources (for MTR suite): script_binary=$(dirname "$0") SCRIPTS_DIR=$(cd "$script_binary"; pwd -P) EXTRA_DIR="$SCRIPTS_DIR/../extra" @@ -626,13 +639,13 @@ CLIENT_DIR="$SCRIPTS_DIR/../client" if [ -x "$CLIENT_DIR/mysql" ]; then MYSQL_CLIENT="$CLIENT_DIR/mysql" else - MYSQL_CLIENT="$(command -v mysql)" + MYSQL_CLIENT=$(commandex 'mysql') fi if [ -x "$CLIENT_DIR/mysqldump" ]; then MYSQLDUMP="$CLIENT_DIR/mysqldump" else - MYSQLDUMP="$(command -v mysqldump)" + MYSQLDUMP=$(commandex 'mysqldump') fi wsrep_log() @@ -663,7 +676,7 @@ if [ -x "$SCRIPTS_DIR/my_print_defaults" ]; then elif [ -x "$EXTRA_DIR/my_print_defaults" ]; then MY_PRINT_DEFAULTS="$EXTRA_DIR/my_print_defaults" else - MY_PRINT_DEFAULTS="$(command -v my_print_defaults)" + MY_PRINT_DEFAULTS=$(commandex 'my_print_defaults') if [ -z "$MY_PRINT_DEFAULTS" ]; then wsrep_log_error "my_print_defaults not found in path" exit 2 @@ -673,16 +686,16 @@ fi readonly MY_PRINT_DEFAULTS wsrep_defaults="$WSREP_SST_OPT_DEFAULTS" -wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULTS" -wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" +wsrep_defaults="$wsrep_defaults${WSREP_SST_OPT_EXTRA_DEFAULTS:+ }$WSREP_SST_OPT_EXTRA_DEFAULTS" +wsrep_defaults="$wsrep_defaults${WSREP_SST_OPT_SUFFIX_DEFAULT:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" -readonly WSREP_SST_OPT_CONF="$wsrep_defaults" +readonly WSREP_SST_OPT_CONF="${wsrep_defaults:+ }$wsrep_defaults" wsrep_defaults="$WSREP_SST_OPT_DEFAULT" -wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULT" -wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" +wsrep_defaults="$wsrep_defaults${WSREP_SST_OPT_EXTRA_DEFAULT:+ }$WSREP_SST_OPT_EXTRA_DEFAULT" +wsrep_defaults="$wsrep_defaults${WSREP_SST_OPT_SUFFIX_DEFAULT:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT" -readonly WSREP_SST_OPT_CONF_UNQUOTED="$wsrep_defaults" +readonly WSREP_SST_OPT_CONF_UNQUOTED="${wsrep_defaults:+ }$wsrep_defaults" # # User can specify mariabackup specific settings that will be used during sst @@ -819,8 +832,7 @@ if wsrep_auth_not_set; then fi # Splitting WSREP_SST_OPT_AUTH as "user:password" pair: -if ! wsrep_auth_not_set -then +if ! wsrep_auth_not_set; then # Extract username as shortest prefix up to first ':' character: WSREP_SST_OPT_AUTH_USER="${WSREP_SST_OPT_AUTH%%:*}" if [ -z "$WSREP_SST_OPT_USER" ]; then @@ -848,8 +860,7 @@ readonly WSREP_SST_OPT_USER readonly WSREP_SST_OPT_PSWD readonly WSREP_SST_OPT_AUTH -if [ -n "$WSREP_SST_OPT_REMOTE_AUTH" ] -then +if [ -n "$WSREP_SST_OPT_REMOTE_AUTH" ]; then # Split auth string at the last ':' readonly WSREP_SST_OPT_REMOTE_USER="${WSREP_SST_OPT_REMOTE_AUTH%%:*}" readonly WSREP_SST_OPT_REMOTE_PSWD="${WSREP_SST_OPT_REMOTE_AUTH#*:}" @@ -860,8 +871,7 @@ fi readonly WSREP_SST_OPT_REMOTE_AUTH -if [ -n "$WSREP_SST_OPT_DATA" ] -then +if [ -n "$WSREP_SST_OPT_DATA" ]; then SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress" else SST_PROGRESS_FILE="" @@ -870,13 +880,14 @@ fi wsrep_cleanup_progress_file() { [ -n "$SST_PROGRESS_FILE" -a \ - -f "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null || true + -f "$SST_PROGRESS_FILE" ] && \ + rm -f "$SST_PROGRESS_FILE" 2>/dev/null || : } wsrep_check_program() { local prog="$1" - local cmd=$(command -v "$prog") + local cmd=$(commandex "$prog") if [ -z "$cmd" ]; then echo "'$prog' not found in PATH" return 2 # no such file or directory @@ -898,9 +909,9 @@ wsrep_check_programs() wsrep_check_datadir() { - if [ -z "$WSREP_SST_OPT_DATA" ] - then - wsrep_log_error "The '--datadir' parameter must be passed to the SST script" + if [ -z "$WSREP_SST_OPT_DATA" ]; then + wsrep_log_error \ + "The '--datadir' parameter must be passed to the SST script" exit 2 fi } @@ -912,10 +923,10 @@ get_openssl() return fi # Let's look for openssl: - OPENSSL_BINARY="$(command -v openssl)" + OPENSSL_BINARY=$(commandex 'openssl') if [ -z "$OPENSSL_BINARY" ]; then OPENSSL_BINARY='/usr/bin/openssl' - if [ -z "$OPENSSL_BINARY" ]; then + if [ ! -x "$OPENSSL_BINARY" ]; then OPENSSL_BINARY="" fi fi @@ -928,13 +939,12 @@ get_openssl() wsrep_gen_secret() { get_openssl - if [ -n "$OPENSSL_BINARY" ] - then + if [ -n "$OPENSSL_BINARY" ]; then echo $("$OPENSSL_BINARY" rand -hex 16) else printf "%04x%04x%04x%04x%04x%04x%04x%04x" \ - $RANDOM $RANDOM $RANDOM $RANDOM \ - $RANDOM $RANDOM $RANDOM $RANDOM + $RANDOM $RANDOM $RANDOM $RANDOM \ + $RANDOM $RANDOM $RANDOM $RANDOM fi } @@ -968,14 +978,14 @@ is_local_ip() fi # Now let's check if the given address is assigned to # one of the network cards: - local ip_util="$(command -v ip)" + local ip_util=$(commandex 'ip') if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6] <address>/<mask>": "$ip_util" address show \ | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \ | grep -F " $1/" >/dev/null && return 0 else - local ifconfig_util="$(command -v ifconfig)" + local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6] <address> ...": "$ifconfig_util" \ @@ -992,15 +1002,15 @@ check_sockets_utils() sockstat_available=0 ss_available=0 - [ -n "$(command -v lsof)" ] && lsof_available=1 - [ -n "$(command -v sockstat)" ] && sockstat_available=1 - [ -n "$(command -v ss)" ] && ss_available=1 + [ -n "$(commandex lsof)" ] && lsof_available=1 + [ -n "$(commandex sockstat)" ] && sockstat_available=1 + [ -n "$(commandex ss)" ] && ss_available=1 if [ $lsof_available -eq 0 -a \ $sockstat_available -eq 0 -a \ $ss_available -eq 0 ] then - wsrep_log_error "Neither lsof, nor sockstat or ss tool was found in " \ + wsrep_log_error "Neither lsof, nor sockstat or ss tool was found in" \ "the PATH. Make sure you have it installed." exit 2 # ENOENT fi @@ -1085,26 +1095,38 @@ check_for_dhparams() # verify_ca_matches_cert() { - local ca="$1" - local cert="$2" - local path=${3:-0} + local cert="$1" + local ca="$2" + local cap="$3" # If the openssl utility is not installed, then # we will not do this certificate check: get_openssl if [ -z "$OPENSSL_BINARY" ]; then + wsrep_log_info "openssl utility not found" return fi - local not_match=0 + local readable=1; [ ! -r "$cert" ] && readable=0 + [ -n "$ca" ] && [ ! -r "$ca" ] && readable=0 + [ -n "$cap" ] && [ ! -r "$cap" ] && readable=0 - if [ $path -eq 0 ]; then - "$OPENSSL_BINARY" verify -verbose -CAfile "$ca" "$cert" >/dev/null 2>&1 || not_match=1 - else - "$OPENSSL_BINARY" verify -verbose -CApath "$ca" "$cert" >/dev/null 2>&1 || not_match=1 + if [ readable -eq 0 ]; then + wsrep_log_error \ + "Both PEM file and CA file (or path) must be readable" + exit 22 fi + local not_match=0 + local errmsg + errmsg=$("$OPENSSL_BINARY" verify -verbose \ + ${ca:+ -CAfile} ${ca:+ "$ca"} \ + ${cap:+ -CApath} ${cap:+ "$cap"} \ + "$cert" 2>&1) || not_match=1 + if [ $not_match -eq 1 ]; then + wsrep_log_info "run: \"$OPENSSL_BINARY\" verify -verbose${ca:+ -CAfile \"$ca\"}${cap:+ -CApath \"$cap\"} \"$cert\"" + wsrep_log_info "output: $errmsg" wsrep_log_error "******** FATAL ERROR ********************************************" wsrep_log_error "* The certifcate and CA (certificate authority) do not match. *" wsrep_log_error "* It does not appear that the certificate was issued by the CA. *" @@ -1124,12 +1146,18 @@ verify_ca_matches_cert() # verify_cert_matches_key() { - local cert_path="$1" - local key_path="$2" + local cert="$1" + local key="$2" + + if [ ! -r "$key" -o ! -r "$cert" ]; then + wsrep_log_error "Both the certificate file and the key file" \ + "must be readable" + exit 22 + fi # If the diff utility is not installed, then # we will not do this certificate check: - if [ -z "$(command -v diff)" ]; then + if [ -z "$(commandex diff)" ]; then return fi @@ -1142,13 +1170,13 @@ verify_cert_matches_key() # Generate the public key from the cert and the key. # They should match (otherwise we can't create an SSL connection). - if ! diff <("$OPENSSL_BINARY" x509 -in "$cert_path" -pubkey -noout 2>/dev/null) \ - <("$OPENSSL_BINARY" pkey -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1 + if ! diff <("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null) \ + <("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null) >/dev/null 2>&1 then - wsrep_log_error "******************* FATAL ERROR ****************" - wsrep_log_error "* The certifcate and private key do not match. *" - wsrep_log_error "* Please check your certificate and key files. *" - wsrep_log_error "************************************************" + wsrep_log_error "******************* FATAL ERROR *****************" + wsrep_log_error "* The certificate and private key do not match. *" + wsrep_log_error "* Please check your certificate and key files. *" + wsrep_log_error "*************************************************" exit 22 fi } @@ -1305,9 +1333,9 @@ get_proc() elif [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ]; then nproc=$(sysctl -n hw.ncpu) fi + set -e if [ -z "$nproc" ] || [ $nproc -eq 0 ]; then nproc=1 fi - set -e fi } diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index b429a9effd5..4bca785fcad 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -20,10 +20,11 @@ # https://mariadb.com/kb/en/mariabackup-overview/ # Make sure to read that before proceeding! +OS="$(uname)" + . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir -OS="$(uname)" ealgo="" eformat="" ekey="" @@ -34,7 +35,7 @@ ssyslog="" ssystag="" BACKUP_PID="" tcert="" -tpath=0 +tcap="" tpem="" tkey="" tmode="DISABLED" @@ -88,14 +89,14 @@ readonly SECRET_TAG="secret" # For backup locks it is 1 sent by joiner sst_ver=1 -if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" declare -a RC -BACKUP_BIN="$(command -v mariabackup)" -if [ ! -x "$BACKUP_BIN" ]; then +BACKUP_BIN=$(commandex 'mariabackup') +if [ -z "$BACKUP_BIN" ]; then wsrep_log_error 'mariabackup binary not found in path' exit 42 fi @@ -145,14 +146,14 @@ get_keys() if [ $encrypt -eq 0 ]; then if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then - wsrep_log_error "Options for encryption are specified, " \ + wsrep_log_error "Options for encryption are specified," \ "but encryption itself is disabled. SST may fail." fi return fi if [ $sfmt = 'tar' ]; then - wsrep_log_info "NOTE: key-based encryption (encrypt=1) " \ + wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 return @@ -165,16 +166,18 @@ get_keys() exit 3 fi - if [ -z "$ekey" -a ! -r "$ekeyfile" ]; then - wsrep_log_error "FATAL: Either key must be specified " \ - "or keyfile must be readable" - exit 3 + if [ -z "$ekey" ]; then + if [ ! -r "$ekeyfile" ]; then + wsrep_log_error "FATAL: Either key must be specified" \ + "or keyfile must be readable" + exit 3 + fi fi if [ "$eformat" = 'openssl' ]; then get_openssl if [ -z "$OPENSSL_BINARY" ]; then - wsrep_log_error "If encryption using the openssl is enabled, " \ + wsrep_log_error "If encryption using the openssl is enabled," \ "then you need to install openssl" exit 2 fi @@ -192,12 +195,12 @@ get_keys() ecmd="$ecmd -k '$ekey'" fi elif [ "$eformat" = 'xbcrypt' ]; then - if [ -z "$(command -v xbcrypt)" ]; then - wsrep_log_error "If encryption using the xbcrypt is enabled, " \ + if [ -z "$(commandex xbcrypt)" ]; then + wsrep_log_error "If encryption using the xbcrypt is enabled," \ "then you need to install xbcrypt" exit 2 fi - wsrep_log_info "NOTE: xbcrypt-based encryption, " \ + wsrep_log_info "NOTE: xbcrypt-based encryption," \ "supported only from Xtrabackup 2.1.4" if [ -z "$ekey" ]; then ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key-file='$ekeyfile'" @@ -342,40 +345,34 @@ get_transfer() CN_option=",commonname=''" if [ $encrypt -eq 2 ]; then - wsrep_log_info "Using openssl based encryption with socat: with crt and pem" - if [ -z "$tpem" -o -z "$tcert" ]; then + wsrep_log_info \ + "Using openssl based encryption with socat: with crt and pem" + if [ -z "$tpem" -o -z "$tcert$tcap" ]; then wsrep_log_error \ "Both PEM file and CRT file (or path) are required" exit 22 fi - if [ ! -r "$tpem" -o ! -r "$tcert" ]; then - wsrep_log_error \ - "Both PEM file and CRT file (or path) must be readable" - exit 22 + verify_ca_matches_cert "$tpem" "$tcert" "$tcap" + tcmd="$tcmd,cert='$tpem'" + if [ -n "$tcert" ]; then + tcmd="$tcmd,cafile='$tcert'" fi - verify_ca_matches_cert "$tcert" "$tpem" $tpath - if [ $tpath -eq 0 ]; then - tcmd="$tcmd,cert='$tpem',cafile='$tcert'" - else - tcmd="$tcmd,cert='$tpem',capath='$tcert'" + if [ -n "$tcap" ]; then + tcmd="$tcmd,capath='$tcap'" fi stagemsg="$stagemsg-OpenSSL-Encrypted-2" - wsrep_log_info "$action with cert=$tpem, ca=$tcert" + wsrep_log_info "$action with cert='$tpem', ca='$tcert', capath='$tcap'" elif [ $encrypt -eq 3 -o $encrypt -eq 4 ]; then - wsrep_log_info "Using openssl based encryption with socat: with key and crt" + wsrep_log_info \ + "Using openssl based encryption with socat: with key and crt" if [ -z "$tpem" -o -z "$tkey" ]; then - wsrep_log_error "Both certificate file (or path) " \ - "and key file are required" - exit 22 - fi - if [ ! -r "$tpem" -o ! -r "$tkey" ]; then - wsrep_log_error "Both certificate file (or path) " \ - "and key file must be readable" + wsrep_log_error "Both the certificate file (or path) and" \ + "the key file are required" exit 22 fi verify_cert_matches_key "$tpem" "$tkey" stagemsg="$stagemsg-OpenSSL-Encrypted-3" - if [ -z "$tcert" ]; then + if [ -z "$tcert$tcap" ]; then if [ $encrypt -eq 4 ]; then wsrep_log_error \ "Peer certificate file (or path) required if encrypt=4" @@ -384,14 +381,11 @@ get_transfer() # no verification CN_option="" tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0" - wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0" + wsrep_log_info \ + "$action with cert='$tpem', key='$tkey', verify=0" else # CA verification - if [ ! -r "$tcert" ]; then - wsrep_log_error "Certificate file or path must be readable" - exit 22 - fi - verify_ca_matches_cert "$tcert" "$tpem" $tpath + verify_ca_matches_cert "$tpem" "$tcert" "$tcap" if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' -o $encrypt -eq 4 ] @@ -402,12 +396,15 @@ get_transfer() else CN_option=",commonname='$WSREP_SST_OPT_HOST_UNESCAPED'" fi - if [ $tpath -eq 0 ]; then - tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'" - else - tcmd="$tcmd,cert='$tpem',key='$tkey',capath='$tcert'" + tcmd="$tcmd,cert='$tpem',key='$tkey'" + if [ -n "$tcert" ]; then + tcmd="$tcmd,cafile='$tcert'" fi - wsrep_log_info "$action with cert=$tpem, key=$tkey, ca=$tcert" + if [ -n "$tcap" ]; then + tcmd="$tcmd,capath='$tcap'" + fi + wsrep_log_info "$action with cert='$tpem', key='$tkey'," \ + "ca='$tcert', capath='$tcap'" fi else wsrep_log_info "Unknown encryption mode: encrypt=$encrypt" @@ -425,7 +422,9 @@ get_transfer() get_footprint() { pushd "$WSREP_SST_OPT_DATA" 1>/dev/null - payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }') + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' \ + -type f -print0 | du --files0-from=- --block-size=1 -c -s | \ + awk 'END { print $1 }') if [ "$compress" != 'none' ]; then # QuickLZ has around 50% compression ratio # When compression/compaction used, the progress is only an approximate. @@ -438,7 +437,7 @@ get_footprint() adjust_progress() { - if [ -z "$(command -v pv)" ]; then + if [ -z "$(commandex pv)" ]; then wsrep_log_error "pv not found in path: $PATH" wsrep_log_error "Disabling all progress/rate-limiting" pcmd="" @@ -470,33 +469,42 @@ check_server_ssl_config() { # backward-compatible behavior: tcert=$(parse_cnf 'sst' 'tca') + tcap=$(parse_cnf 'sst' 'tcapath') tpem=$(parse_cnf 'sst' 'tcert') tkey=$(parse_cnf 'sst' 'tkey') # reading new ssl configuration options: local tcert2=$(parse_cnf "$encgroups" 'ssl-ca') + local tcap2=$(parse_cnf "$encgroups" 'ssl-capath') local tpem2=$(parse_cnf "$encgroups" 'ssl-cert') local tkey2=$(parse_cnf "$encgroups" 'ssl-key') # if there are no old options, then we take new ones: - if [ -z "$tcert" -a -z "$tpem" -a -z "$tkey" ]; then + if [ -z "$tcert" -a -z "$tcap" -a -z "$tpem" -a -z "$tkey" ]; then tcert="$tcert2" + tcap="$tcap2" tpem="$tpem2" tkey="$tkey2" # checking for presence of the new-style SSL configuration: - elif [ -n "$tcert2" -o -n "$tpem2" -o -n "$tkey2" ]; then + elif [ -n "$tcert2" -o -n "$tcap2" -o -n "$tpem2" -o -n "$tkey2" ]; then if [ "$tcert" != "$tcert2" -o \ + "$tcap" != "$tcap2" -o \ "$tpem" != "$tpem2" -o \ "$tkey" != "$tkey2" ] then - wsrep_log_info "new ssl configuration options (ssl-ca, ssl-cert " \ - "and ssl-key) are ignored by SST due to presence " \ - "of the tca, tcert and/or tkey in the [sst] section" + wsrep_log_info \ + "new ssl configuration options (ssl-ca[path], ssl-cert" \ + "and ssl-key) are ignored by SST due to presence" \ + "of the tca[path], tcert and/or tkey in the [sst] section" fi fi if [ -n "$tcert" ]; then - tcert=$(trim_string "$tcert") - if [ "${tcert%/}" != "$tcert" ]; then - tpath=1 - fi + tcert=$(trim_string "$tcert") + if [ "${tcert%/}" != "$tcert" ] || [ -d "$tcert" ]; then + tcap="$tcert" + tcert="" + fi + fi + if [ -n "$tcap" ]; then + tcap=$(trim_string "$tcap") fi } @@ -506,10 +514,10 @@ read_cnf() tfmt=$(parse_cnf sst transferfmt 'socat') encrypt=$(parse_cnf "$encgroups" 'encrypt' 0) - tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | tr [:lower:] [:upper:]) + tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | \ + tr [:lower:] [:upper:]) - if [ $encrypt -eq 0 -o $encrypt -ge 2 ] - then + if [ $encrypt -eq 0 -o $encrypt -ge 2 ]; then if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then check_server_ssl_config fi @@ -517,11 +525,13 @@ read_cnf() if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ] then encrypt=3 # enable cert/key SSL encyption - # avoid CA verification if not set explicitly: - # nodes may happen to have different CA if self-generated - # zeroing up tcert does the trick - [ "${tmode#VERIFY}" != "$tmode" ] || tcert="" + # nodes may happen to have different CA if self-generated, + # zeroing up tcert and tcap does the trick: + if [ "${tmode#VERIFY}" = "$tmode" ]; then + tcert="" + tcap="" + fi fi fi elif [ $encrypt -eq 1 ]; then @@ -535,8 +545,9 @@ read_cnf() fi fi - wsrep_log_info "SSL configuration: CA='$tcert', CERT='$tpem'," \ - "KEY='$tkey', MODE='$tmode', encrypt='$encrypt'" + wsrep_log_info "SSL configuration: CA='$tcert', CAPATH='$tcap'," \ + "CERT='$tpem', KEY='$tkey', MODE='$tmode'," \ + "encrypt='$encrypt'" sockopt=$(parse_cnf sst sockopt "") progress=$(parse_cnf sst progress "") @@ -561,7 +572,8 @@ read_cnf() sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir '/tmp/sst_log_archive') if [ $speciald -eq 0 ]; then - wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1" + wsrep_log_error \ + "sst-special-dirs equal to 0 is not supported, falling back to 1" speciald=1 fi @@ -589,7 +601,7 @@ get_stream() { if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then sfmt='mbstream' - STREAM_BIN="$(command -v mbstream)" + local STREAM_BIN=$(commandex "$sfmt") if [ -z "$STREAM_BIN" ]; then wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path" exit 42 @@ -621,7 +633,7 @@ cleanup_at_exit() # Since this is invoked just after exit NNN local estatus=$? if [ $estatus -ne 0 ]; then - wsrep_log_error "Cleanup after exit with status:$estatus" + wsrep_log_error "Cleanup after exit with status: $estatus" fi if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then @@ -630,7 +642,8 @@ cleanup_at_exit() else if [ -n "$BACKUP_PID" ]; then if check_pid "$BACKUP_PID" 1; then - wsrep_log_error "mariabackup process is still running. Killing..." + wsrep_log_error \ + "mariabackup process is still running. Killing..." cleanup_pid $CHECK_PID "$BACKUP_PID" fi fi @@ -638,8 +651,8 @@ cleanup_at_exit() fi if [ -n "$progress" -a -p "$progress" ]; then - wsrep_log_info "Cleaning up fifo file $progress" - rm -f "$progress" || true + wsrep_log_info "Cleaning up fifo file: $progress" + rm -f "$progress" || : fi wsrep_log_info "Cleaning up temporary directories" @@ -649,8 +662,8 @@ cleanup_at_exit() [ -d "$STATDIR" ] && rm -rf "$STATDIR" fi else - [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true - [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true + [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || : + [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || : fi # Final cleanup @@ -662,7 +675,7 @@ cleanup_at_exit() # This means a signal was delivered to the process. # So, more cleanup. if [ $estatus -ge 128 ]; then - kill -KILL -- -$$ || true + kill -KILL -- -$$ || : fi fi @@ -738,7 +751,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then - if [ -n "$(command -v timeout)" ]; then + if [ -n "$(commandex timeout)" ]; then if timeout --help | grep -qw -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else @@ -760,14 +773,14 @@ recv_joiner() popd 1>/dev/null if [ ${RC[0]} -eq 124 ]; then - wsrep_log_error "Possible timeout in receiving first data from " \ + wsrep_log_error "Possible timeout in receiving first data from" \ "donor in gtid stage: exit codes: ${RC[@]}" exit 32 fi for ecode in "${RC[@]}"; do if [ $ecode -ne 0 ]; then - wsrep_log_error "Error while getting data from donor node: " \ + wsrep_log_error "Error while getting data from donor node:" \ "exit codes: ${RC[@]}" exit 32 fi @@ -776,7 +789,7 @@ recv_joiner() if [ $checkf -eq 1 ]; then if [ ! -r "$MAGIC_FILE" ]; then # this message should cause joiner to abort - wsrep_log_error "receiving process ended without creating " \ + wsrep_log_error "receiving process ended without creating" \ "'$MAGIC_FILE'" wsrep_log_info "Contents of datadir" wsrep_log_info $(ls -l "$dir/"*) @@ -784,10 +797,11 @@ recv_joiner() fi # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2) + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" exit 32 fi @@ -810,7 +824,7 @@ send_donor() for ecode in "${RC[@]}"; do if [ $ecode -ne 0 ]; then - wsrep_log_error "Error while sending data to joiner node: " \ + wsrep_log_error "Error while sending data to joiner node:" \ "exit codes: ${RC[@]}" exit 32 fi @@ -823,7 +837,9 @@ monitor_process() while true ; do if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then - wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." + wsrep_log_error \ + "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ + "terminated unexpectedly." kill -- -"$WSREP_SST_OPT_PARENT" exit 32 fi @@ -845,7 +861,7 @@ read_cnf setup_ports if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then - disver='--no-version-check' + disver=' --no-version-check' fi # if no command line argument and INNODB_DATA_HOME_DIR environment variable @@ -867,7 +883,7 @@ INNODB_DATA_HOME_DIR=$(pwd -P) cd "$OLD_PWD" if [ $ssyslog -eq 1 ]; then - if [ -n "$(command -v logger)" ]; then + if [ -n "$(commandex logger)" ]; then wsrep_log_info "Logging all stderr of SST/mariabackup to syslog" exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) @@ -898,10 +914,8 @@ else fi fi - if [ -e "$INNOAPPLYLOG" ] - then - if [ -n "$sstlogarchivedir" ] - then + if [ -e "$INNOAPPLYLOG" ]; then + if [ -n "$sstlogarchivedir" ]; then newfile=$(basename "$INNOAPPLYLOG") newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" else @@ -912,10 +926,8 @@ else gzip "$newfile" fi - if [ -e "$INNOMOVELOG" ] - then - if [ -n "$sstlogarchivedir" ] - then + if [ -e "$INNOMOVELOG" ]; then + if [ -n "$sstlogarchivedir" ]; then newfile=$(basename "$INNOMOVELOG") newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" else @@ -926,10 +938,8 @@ else gzip "$newfile" fi - if [ -e "$INNOBACKUPLOG" ] - then - if [ -n "$sstlogarchivedir" ] - then + if [ -e "$INNOBACKUPLOG" ]; then + if [ -n "$sstlogarchivedir" ]; then newfile=$(basename "$INNOBACKUPLOG") newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP" else @@ -949,15 +959,15 @@ setup_commands() { local mysqld_args="" if [ -n "$WSREP_SST_OPT_MYSQLD" ]; then - mysqld_args="--mysqld-args $WSREP_SST_OPT_MYSQLD" + mysqld_args=" --mysqld-args $WSREP_SST_OPT_MYSQLD" fi - if [ -z "$INNODB_FORCE_RECOVERY" ]; then - INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY" - else - INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --innodb-force-recovery=$INNODB_FORCE_RECOVERY --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY" + local recovery="" + if [ -n "$INNODB_FORCE_RECOVERY" ]; then + recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi - INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" - INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP" + INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" + INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts $tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP" } get_stream @@ -1055,27 +1065,28 @@ then tcmd="$ecmd | $tcmd" fi - iopts="--databases-exclude='lost+found' $iopts" + iopts="--databases-exclude='lost+found'${iopts:+ }$iopts" if [ ${FORCE_FTWRL:-0} -eq 1 ]; then - wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" - iopts="--no-backup-locks $iopts" + wsrep_log_info "Forcing FTWRL due to environment variable" \ + "FORCE_FTWRL equal to $FORCE_FTWRL" + iopts="--no-backup-locks${iopts:+ }$iopts" fi # if compression is enabled for backup files, then add the # appropriate options to the mariabackup command line: if [ "$compress" != 'none' ]; then - iopts="--compress${compress:+=$compress} $iopts" + iopts="--compress${compress:+=$compress}${iopts:+ }$iopts" if [ -n "$compress_threads" ]; then - iopts="--compress-threads=$compress_threads $iopts" + iopts="--compress-threads=$compress_threads${iopts:+ }$iopts" fi if [ -n "$compress_chunk" ]; then - iopts="--compress-chunk-size=$compress_chunk $iopts" + iopts="--compress-chunk-size=$compress_chunk${iopts:+ }$iopts" fi fi if [ -n "$backup_threads" ]; then - iopts="--parallel=$backup_threads $iopts" + iopts="--parallel=$backup_threads${iopts:+ }$iopts" fi setup_commands @@ -1084,7 +1095,7 @@ then set -e if [ ${RC[0]} -ne 0 ]; then - wsrep_log_error "mariabackup finished with error: ${RC[0]}. " \ + wsrep_log_error "mariabackup finished with error: ${RC[0]}." \ "Check syslog or '$INNOBACKUPLOG' for details" exit 22 elif [ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]; then @@ -1125,7 +1136,8 @@ then elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then - [ -e "$SST_PROGRESS_FILE" ] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + [ -e "$SST_PROGRESS_FILE" ] && \ + wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" ib_home_dir="$INNODB_DATA_HOME_DIR" @@ -1146,7 +1158,7 @@ then ib_undo_dir="$INNODB_UNDO_DIR" if [ -n "$backup_threads" ]; then - impts="--parallel=$backup_threads $impts" + impts="--parallel=$backup_threads${impts:+ }$impts" fi stagemsg='Joiner-Recv' @@ -1165,15 +1177,15 @@ then ADDR="$WSREP_SST_OPT_ADDR" - if [ "${tmode#VERIFY}" != "$tmode" ] - then # backward-incompatible behavior + if [ "${tmode#VERIFY}" != "$tmode" ]; then + # backward-incompatible behavior: CN="" - if [ -n "$tpem" ] - then + if [ -n "$tpem" ]; then # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then - wsrep_log_error 'openssl not found but it is required for authentication' + wsrep_log_error \ + 'openssl not found but it is required for authentication' exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ @@ -1213,15 +1225,17 @@ then if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1 then - wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly." + wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ + "terminated unexpectedly." exit 32 fi - if [ ! -r "$STATDIR/$IST_FILE" ] - then + if [ ! -r "$STATDIR/$IST_FILE" ]; then if [ -d "$DATA/.sst" ]; then - wsrep_log_info "WARNING: Stale temporary SST directory: '$DATA/.sst' from previous state transfer. Removing" + wsrep_log_info \ + "WARNING: Stale temporary SST directory:" \ + "'$DATA/.sst' from previous state transfer, removing..." rm -rf "$DATA/.sst" fi mkdir -p "$DATA/.sst" @@ -1229,17 +1243,20 @@ then jpid=$! wsrep_log_info "Proceeding with SST" - wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" + wsrep_log_info \ + "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then find -E ${ib_home_dir:+"$ib_home_dir"} \ ${ib_undo_dir:+"$ib_undo_dir"} \ ${ib_log_dir:+"$ib_log_dir"} \ - "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+ + "$DATA" -mindepth 1 -prune -regex "$cpat" \ + -o -exec rm -rfv {} 1>&2 \+ else find ${ib_home_dir:+"$ib_home_dir"} \ ${ib_undo_dir:+"$ib_undo_dir"} \ ${ib_log_dir:+"$ib_log_dir"} \ - "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+ + "$DATA" -mindepth 1 -prune -regex "$cpat" \ + -o -exec rm -rfv {} 1>&2 \+ fi get_binlog @@ -1248,9 +1265,9 @@ then binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") cd "$binlog_dir" wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || true + rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ || true + rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ || : cd "$OLD_PWD" fi @@ -1262,7 +1279,8 @@ then monitor_process $jpid if [ ! -s "$DATA/xtrabackup_checkpoints" ]; then - wsrep_log_error "xtrabackup_checkpoints missing, failed mariabackup/SST on donor" + wsrep_log_error "xtrabackup_checkpoints missing," \ + "failed mariabackup/SST on donor" exit 2 fi @@ -1277,7 +1295,7 @@ then if [ -n "$qpfiles" ]; then wsrep_log_info "Compressed qpress files found" - if [ -z "$(command -v qpress)" ]; then + if [ -z "$(commandex qpress)" ]; then wsrep_log_error "qpress utility not found in the path" exit 22 fi @@ -1300,14 +1318,17 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit "Joiner-Decompression" \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" extcode=$? if [ $extcode -eq 0 ]; then wsrep_log_info "Removing qpress files after decompression" find "$DATA" -type f -name '*.qp' -delete if [ $? -ne 0 ]; then - wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + wsrep_log_error \ + "Something went wrong with deletion of qpress files." \ + "Investigate" fi else wsrep_log_error "Decompression failed. Exit code: $extcode" @@ -1321,7 +1342,7 @@ then BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || true + mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : cd "$BINLOG_DIRNAME" for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do @@ -1336,7 +1357,8 @@ then timeit "mariabackup prepare stage" "$INNOAPPLY" if [ $? -ne 0 ]; then - wsrep_log_error "mariabackup apply finished with errors. Check syslog or '$INNOAPPLYLOG' for details" + wsrep_log_error "mariabackup apply finished with errors." \ + "Check syslog or '$INNOAPPLYLOG' for details." exit 22 fi diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 798bee1ac10..bed2cac0a9a 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -19,7 +19,6 @@ # This is a reference script for mysqldump-based state snapshot tansfer . $(dirname "$0")/wsrep_sst_common -PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin EINVAL=22 @@ -93,8 +92,7 @@ DROP PREPARE stmt;" SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" SET_WSREP_GTID_DOMAIN_ID="" -if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ] -then +if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ]; then SET_WSREP_GTID_DOMAIN_ID=" SET @val = (SELECT GLOBAL_VALUE FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES WHERE VARIABLE_NAME = 'WSREP_GTID_STRICT_MODE' AND GLOBAL_VALUE > 0); SET @stmt = IF (@val IS NOT NULL, 'SET GLOBAL WSREP_GTID_DOMAIN_ID=$WSREP_SST_OPT_GTID_DOMAIN_ID', 'SET @dummy = 0'); @@ -103,7 +101,7 @@ then DROP PREPARE stmt;" fi -MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF_UNQUOTED "\ +MYSQL="$MYSQL_CLIENT$WSREP_SST_OPT_CONF_UNQUOTED "\ "$AUTH -h$WSREP_SST_OPT_HOST_UNESCAPED "\ "-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10" @@ -125,8 +123,7 @@ SET_GTID_BINLOG_STATE="" SQL_LOG_BIN_OFF="" # Safety check -if [ ${SERVER_VERSION%%.*} -gt 5 ] -then +if [ ${SERVER_VERSION%%.*} -gt 5 ]; then # If binary logging is enabled on the joiner node, we need to copy donor's # gtid_binlog_state to joiner. In order to do that, a RESET MASTER must be # executed to erase binary logs (if any). Binary logging should also be @@ -140,7 +137,7 @@ then fi # NOTE: we don't use --routines here because we're dumping mysql.proc table -MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF_UNQUOTED $AUTH -S$WSREP_SST_OPT_SOCKET \ +MYSQLDUMP="$MYSQLDUMP$WSREP_SST_OPT_CONF_UNQUOTED $AUTH -S$WSREP_SST_OPT_SOCKET \ --add-drop-database --add-drop-table --skip-add-locks --create-options \ --disable-keys --extended-insert --skip-lock-tables --quick --set-charset \ --skip-comments --flush-privileges --all-databases --events" diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 29c9cd43470..b0cc8cb3066 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -25,9 +25,6 @@ STUNNEL_REAL_PID=0 # stunnel process id OS="$(uname)" [ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH -# Setting the path for lsof on CentOS -export PATH="/usr/sbin:/sbin:$PATH" - . $(dirname "$0")/wsrep_sst_common wsrep_check_datadir @@ -37,7 +34,8 @@ cleanup_joiner() { local failure=0 - wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID, stunnel PID=$STUNNEL_REAL_PID" + wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID," \ + "stunnel PID=$STUNNEL_REAL_PID" if [ -n "$STUNNEL" ]; then if cleanup_pid $STUNNEL_REAL_PID "$STUNNEL_PID" "$STUNNEL_CONF"; then @@ -87,7 +85,7 @@ check_pid_and_port() if [ $lsof_available -ne 0 ]; then port_info=$(lsof -Pnl -i ":$port" 2>/dev/null | \ - grep -F '(LISTEN)') + grep -F '(LISTEN)') echo "$port_info" | \ grep -q -E "[[:space:]](\\*|\\[?::\\]?):$port[[:space:]]" && busy=1 else @@ -124,7 +122,7 @@ check_pid_and_port() fi if ! check_port "$pid" "$port" "$utils"; then - wsrep_log_error "rsync or stunnel daemon port '$port' " \ + wsrep_log_error "rsync or stunnel daemon port '$port'" \ "has been taken by another program" exit 16 # EBUSY fi @@ -223,6 +221,7 @@ FILTER="-f '- /lost+found' SSTKEY=$(parse_cnf 'sst' 'tkey') SSTCERT=$(parse_cnf 'sst' 'tcert') SSTCA=$(parse_cnf 'sst' 'tca') +SSTCAP=$(parse_cnf 'sst' 'tcapath') SST_SECTIONS="--mysqld|sst" @@ -231,28 +230,34 @@ check_server_ssl_config() SSTKEY=$(parse_cnf "$SST_SECTIONS" 'ssl-key') SSTCERT=$(parse_cnf "$SST_SECTIONS" 'ssl-cert') SSTCA=$(parse_cnf "$SST_SECTIONS" 'ssl-ca') + SSTCAP=$(parse_cnf "$SST_SECTIONS" 'ssl-capath') } SSLMODE=$(parse_cnf "$SST_SECTIONS" 'ssl-mode' | tr [:lower:] [:upper:]) # no old-style SSL config in [sst], check for new one: -if [ -z "$SSTKEY" -a -z "$SSTCERT" -a -z "$SSTCA" ]; then +if [ -z "$SSTKEY" -a -z "$SSTCERT" -a -z "$SSTCA" -a -z "$SSTCAP" ]; then check_server_ssl_config fi -SSTPATH=0 if [ -n "$SSTCA" ]; then - SSTCA=$(trim_string "$SSTCA") - if [ "${SSTCA%/}" != "$SSTCA" ]; then - SSTPATH=1 - fi + SSTCA=$(trim_string "$SSTCA") + if [ "${SSTCA%/}" != "$SSTCA" ] || [ -d "$SSTCA" ]; then + SSTCAP="$SSTCA" + SSTCA="" + fi +fi + +if [ -n "$SSTCAP" ]; then + SSTCAP=$(trim_string "$SSTCAP") fi if [ -z "$SSLMODE" ]; then # Implicit verification if CA is set and the SSL mode # is not specified by user: - if [ -n "$SSTCA" ]; then - if [ -n "$(command -v stunnel)" ]; then + if [ -n "$SSTCA$SSTCAP" ]; then + STUNNEL_BIN=$(commandex 'stunnel') + if [ -n "$STUNNEL_BIN" ]; then SSLMODE='VERIFY_CA' fi # Require SSL by default if SSL key and cert are present: @@ -265,17 +270,18 @@ if [ -n "$SSTCERT" -a -n "$SSTKEY" ]; then verify_cert_matches_key "$SSTCERT" "$SSTKEY" fi -if [ -n "$SSTCA" ]; then - if [ $SSTPATH -eq 0 ]; then +CAFILE_OPT="" +CAPATH_OPT="" +if [ -n "$SSTCA$SSTCAP" ]; then + if [ -n "$SSTCA" ]; then CAFILE_OPT="CAfile = $SSTCA" - else - CAFILE_OPT="CApath = $SSTCA" + fi + if [ -n "$SSTCAP" ]; then + CAPATH_OPT="CApath = $SSTCAP" fi if [ -n "$SSTCERT" ]; then - verify_ca_matches_cert "$SSTCA" "$SSTCERT" $SSTPATH + verify_ca_matches_cert "$SSTCERT" "$SSTCA" "$SSTCAP" fi -else - CAFILE_OPT="" fi VERIFY_OPT="" @@ -295,7 +301,7 @@ then exit 22 # EINVAL ;; esac - if [ -z "$SSTCA" ]; then + if [ -z "$SSTCA$SSTCAP" ]; then wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file or path" exit 22 # EINVAL fi @@ -318,9 +324,12 @@ fi STUNNEL="" if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then - STUNNEL_BIN="$(command -v stunnel)" + if [ -z "${STUNNEL_BIN+x}" ]; then + STUNNEL_BIN=$(commandex 'stunnel') + fi if [ -n "$STUNNEL_BIN" ]; then - wsrep_log_info "Using stunnel for SSL encryption: CA: '$SSTCA', ssl-mode='$SSLMODE'" + wsrep_log_info "Using stunnel for SSL encryption: CA: '$SSTCA'," \ + "CAPATH='$SSTCAP', ssl-mode='$SSLMODE'" STUNNEL="$STUNNEL_BIN $STUNNEL_CONF" fi fi @@ -340,6 +349,7 @@ then key = $SSTKEY cert = $SSTCERT ${CAFILE_OPT} +${CAPATH_OPT} foreground = yes pid = $STUNNEL_PID debug = warning @@ -392,8 +402,8 @@ EOF # Prepare binlog files cd "$BINLOG_DIRNAME" - binlog_files_full=$(tail -n $BINLOG_N_FILES "$WSREP_SST_OPT_BINLOG_INDEX") - + binlog_files_full=$(tail -n $BINLOG_N_FILES \ + "$WSREP_SST_OPT_BINLOG_INDEX") binlog_files="" for ii in $binlog_files_full do @@ -417,9 +427,10 @@ EOF WHOLE_FILE_OPT="--whole-file" fi - # first, the normal directories, so that we can detect incompatible protocol + # first, the normal directories, so that we can detect + # incompatible protocol: RC=0 - eval rsync ${STUNNEL:+"'--rsh=$STUNNEL'"} \ + eval rsync ${STUNNEL:+"--rsh='$STUNNEL'"} \ --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT $FILTER "'$WSREP_SST_OPT_DATA/'" \ @@ -430,8 +441,9 @@ EOF case $RC in 12) RC=71 # EPROTO wsrep_log_error \ - "rsync server on the other end has incompatible protocol. " \ - "Make sure you have the same version of rsync on all nodes." + "rsync server on the other end has incompatible" \ + "protocol. Make sure you have the same version of" \ + "rsync on all nodes." ;; 22) RC=12 # ENOMEM ;; @@ -481,9 +493,9 @@ EOF find . -maxdepth 1 -mindepth 1 -type d -not -name 'lost+found' \ -not -name '.zfs' -print0 | xargs -I{} -0 -P $backup_threads \ rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --owner --group --perms --links --specials \ - --ignore-times --inplace --recursive --delete --quiet \ - $WHOLE_FILE_OPT --exclude '*/ib_logfile*' --exclude '*/aria_log.*' \ + --owner --group --perms --links --specials --ignore-times \ + --inplace --recursive --delete --quiet $WHOLE_FILE_OPT \ + --exclude '*/ib_logfile*' --exclude '*/aria_log.*' \ --exclude '*/aria_log_control' "$WSREP_SST_OPT_DATA/{}/" \ "rsync://$WSREP_SST_OPT_ADDR/{}" >&2 || RC=$? @@ -514,7 +526,8 @@ EOF fi rsync ${STUNNEL:+--rsh="$STUNNEL"} \ - --archive --quiet --checksum "$MAGIC_FILE" "rsync://$WSREP_SST_OPT_ADDR" + --archive --quiet --checksum "$MAGIC_FILE" \ + "rsync://$WSREP_SST_OPT_ADDR" echo "done $STATE" @@ -546,7 +559,8 @@ then check_round=0 while check_pid "$STUNNEL_PID" 1 do - wsrep_log_info "lingering stunnel daemon found at startup, waiting for it to exit" + wsrep_log_info "Lingering stunnel daemon found at startup," \ + "waiting for it to exit" check_round=$(( check_round + 1 )) if [ $check_round -eq 10 ]; then wsrep_log_error "stunnel daemon already running." @@ -563,7 +577,8 @@ then check_round=0 while check_pid "$RSYNC_PID" 1 do - wsrep_log_info "lingering rsync daemon found at startup, waiting for it to exit" + wsrep_log_info "Lingering rsync daemon found at startup," \ + "waiting for it to exit" check_round=$(( check_round + 1 )) if [ $check_round -eq 10 ]; then wsrep_log_error "rsync daemon already running." @@ -575,9 +590,7 @@ then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - if [ -z "$STUNNEL" ]; then - [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" - fi + [ -z "$STUNNEL" ] && [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" ADDR="$WSREP_SST_OPT_ADDR" RSYNC_PORT="$WSREP_SST_OPT_PORT" @@ -628,19 +641,21 @@ EOF echo $$ > "$SST_PID" - if [ -z "$STUNNEL" ] - then - rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS & + if [ -z "$STUNNEL" ]; then + rsync --daemon --no-detach --port "$RSYNC_PORT" \ + --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS & RSYNC_REAL_PID=$! TRANSFER_REAL_PID=$RSYNC_REAL_PID TRANSFER_PID="$RSYNC_PID" else # Let's check if the path to the config file contains a space? + RSYNC_BIN=$(commandex 'rsync') if [ "${RSYNC_CONF#* }" = "$RSYNC_CONF" ]; then cat << EOF > "$STUNNEL_CONF" key = $SSTKEY cert = $SSTCERT ${CAFILE_OPT} +${CAPATH_OPT} foreground = yes pid = $STUNNEL_PID debug = warning @@ -650,17 +665,18 @@ ${CHECK_OPT} ${CHECK_OPT_LOCAL} [rsync] accept = $STUNNEL_ACCEPT -exec = $(command -v rsync) +exec = $RSYNC_BIN execargs = rsync --server --daemon --config=$RSYNC_CONF . EOF else # The path contains a space, so we will run it via # shell with "eval" command: - export RSYNC_CMD="eval $(command -v rsync) --server --daemon --config='$RSYNC_CONF' ." + export RSYNC_CMD="eval '$RSYNC_BIN' --server --daemon --config='$RSYNC_CONF' ." cat << EOF > "$STUNNEL_CONF" key = $SSTKEY cert = $SSTCERT ${CAFILE_OPT} +${CAPATH_OPT} foreground = yes pid = $STUNNEL_PID debug = warning @@ -688,7 +704,8 @@ EOF # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then - wsrep_log_error 'openssl not found but it is required for authentication' + wsrep_log_error \ + 'openssl not found but it is required for authentication' exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ @@ -703,7 +720,8 @@ EOF ADDR="$WSREP_SST_OPT_HOST" fi - until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT" + until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID \ + "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT" do sleep 0.2 done @@ -722,7 +740,7 @@ EOF if ! ps -p $MYSQLD_PID >/dev/null 2>&1 then wsrep_log_error \ - "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly." + "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly." kill -- -$MYSQLD_PID sleep 1 exit 32 @@ -768,10 +786,11 @@ EOF if [ -r "$MAGIC_FILE" ]; then if [ -n "$MY_SECRET" ]; then # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2) + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" exit 32 fi # remove secret from the magic file, and output diff --git a/vio/viosslfactories.c b/vio/viosslfactories.c index 8ab7565a666..08f0905e044 100644 --- a/vio/viosslfactories.c +++ b/vio/viosslfactories.c @@ -178,6 +178,12 @@ new_VioSSLFd(const char *key_file, const char *cert_file, struct st_VioSSLFd *ssl_fd; long ssl_ctx_options= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3; DBUG_ENTER("new_VioSSLFd"); + + if (ca_file && ! ca_file[0]) ca_file = NULL; + if (ca_path && ! ca_path[0]) ca_path = NULL; + if (crl_file && ! crl_file[0]) crl_file = NULL; + if (crl_path && ! crl_path[0]) crl_path = NULL; + DBUG_PRINT("enter", ("key_file: '%s' cert_file: '%s' ca_file: '%s' ca_path: '%s' " "cipher: '%s' crl_file: '%s' crl_path: '%s' ", @@ -308,6 +314,11 @@ new_VioSSLConnectorFd(const char *key_file, const char *cert_file, struct st_VioSSLFd *ssl_fd; int verify= SSL_VERIFY_PEER; + if (ca_file && ! ca_file[0]) ca_file = NULL; + if (ca_path && ! ca_path[0]) ca_path = NULL; + if (crl_file && ! crl_file[0]) crl_file = NULL; + if (crl_path && ! crl_path[0]) crl_path = NULL; + /* Turn off verification of servers certificate if both ca_file and ca_path is set to NULL @@ -339,6 +350,12 @@ new_VioSSLAcceptorFd(const char *key_file, const char *cert_file, { struct st_VioSSLFd *ssl_fd; int verify= SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE; + + if (ca_file && ! ca_file[0]) ca_file = NULL; + if (ca_path && ! ca_path[0]) ca_path = NULL; + if (crl_file && ! crl_file[0]) crl_file = NULL; + if (crl_path && ! crl_path[0]) crl_path = NULL; + if (!(ssl_fd= new_VioSSLFd(key_file, cert_file, ca_file, ca_path, cipher, FALSE, error, crl_file, crl_path))) From ef9517eb81cf6c278e88622b3fa4a668a8bcb13f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com> Date: Wed, 15 Dec 2021 14:17:55 +0200 Subject: [PATCH 11/21] MDEV-27268 Failed InnoDB initialization leaves garbage files behind create_log_files(): Check log_set_capacity() before modifying or creating any log files. innobase_start_or_create_for_mysql(): If create_log_files() fails and we were initializing a new database, delete the system tablespace files before exiting. --- mysql-test/suite/innodb/r/log_file.result | 1 - mysql-test/suite/innodb/t/log_file.test | 3 --- storage/innobase/srv/srv0start.cc | 15 +++++++++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/innodb/r/log_file.result b/mysql-test/suite/innodb/r/log_file.result index b93bfc0d02b..952b9f3a3af 100644 --- a/mysql-test/suite/innodb/r/log_file.result +++ b/mysql-test/suite/innodb/r/log_file.result @@ -14,7 +14,6 @@ WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /File .path.to.non-existent.*ib_logfile101: 'create' returned OS error \d+/ in mysqld.1.err -# Remove ibdata1 & ibdata2 # Successfully let InnoDB create tablespaces SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb' diff --git a/mysql-test/suite/innodb/t/log_file.test b/mysql-test/suite/innodb/t/log_file.test index 8a82ab7f29f..7aca2336de6 100644 --- a/mysql-test/suite/innodb/t/log_file.test +++ b/mysql-test/suite/innodb/t/log_file.test @@ -67,9 +67,6 @@ eval $check_no_innodb; let SEARCH_PATTERN=File .path.to.non-existent.*ib_logfile101: 'create' returned OS error \d+; --source include/search_pattern_in_file.inc ---echo # Remove ibdata1 & ibdata2 ---remove_file $bugdir/ibdata1 ---remove_file $bugdir/ibdata2 --list_files $bugdir --echo # Successfully let InnoDB create tablespaces diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 9548730b359..9fc18272a2a 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2020, MariaDB Corporation. +Copyright (c) 2013, 2021, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -426,6 +426,10 @@ create_log_files( return(DB_READ_ONLY); } + if (!log_set_capacity(srv_log_file_size_requested)) { + return(DB_ERROR); + } + /* Crashing after deleting the first file should be recoverable. The buffer pool was clean, and we can simply create all log files from the scratch. */ @@ -482,9 +486,6 @@ create_log_files( } log_init(srv_n_log_files); - if (!log_set_capacity(srv_log_file_size_requested)) { - return(DB_ERROR); - } fil_open_log_and_system_tablespace_files(); @@ -1936,6 +1937,12 @@ innobase_start_or_create_for_mysql() logfilename, dirnamelen, flushed_lsn, logfile0); if (err != DB_SUCCESS) { + for (Tablespace::const_iterator + i = srv_sys_space.begin(); + i != srv_sys_space.end(); i++) { + os_file_delete(innodb_data_file_key, + i->filepath()); + } return(srv_init_abort(err)); } } else { From f1ca949f2bcb3a0bd32fbb2c0eed7ba7dd1e099f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com> Date: Wed, 15 Dec 2021 13:16:29 +0200 Subject: [PATCH 12/21] Disable following tests from galera_3nodes suite * galera_pc_bootstrap * galera_ipv6_mariabackup * galera_ipv6_mariabackup_section * galera_ipv6_rsync * galera_ipv6_rsync_section * galera_ssl_reload * galera_toi_vote * galera_wsrep_schema_init because MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed --- mysql-test/suite/galera_3nodes/disabled.def | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mysql-test/suite/galera_3nodes/disabled.def b/mysql-test/suite/galera_3nodes/disabled.def index 9100db4d441..bbeddc8dcd8 100644 --- a/mysql-test/suite/galera_3nodes/disabled.def +++ b/mysql-test/suite/galera_3nodes/disabled.def @@ -15,6 +15,14 @@ galera_gtid_2_cluster : MDEV-23775 Galera test failure on galera_3nodes.galera_g galera_ist_gcache_rollover : MDEV-23578 WSREP: exception caused by message: {v=0,t=1,ut=255,o=4,s=0,sr=0,as=1,f=6,src=50524cfe,srcvid=view_id(REG,50524cfe,4),insvid=view_id(UNKNOWN,00000000,0),ru=00000000,r=[-1,-1],fs=75,nl=(} galera_load_data_ist : MDEV-24639 galera_3nodes.galera_load_data_ist MTR failed with SIGABRT: query 'reap' failed: 2013: Lost connection to MySQL server during query galera_load_data_ist : MDEV-24639 galera_3nodes.galera_load_data_ist MTR failed with SIGABRT: query 'reap' failed: 2013: Lost connection to MySQL server during query +galera_pc_bootstrap : MDEV-24097 MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed galera_safe_to_bootstrap : MDEV-24097 galera_3nodes.galera_safe_to_bootstrap MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed galera_slave_options_do : MDEV-8798 galera_slave_options_ignore : MDEV-8798 +galera_ipv6_mariabackup : MDEV-24097 MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed +galera_ipv6_mariabackup_section : MDEV-24097 MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed +galera_ipv6_rsync : MDEV-24097 MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed +galera_ipv6_rsync_section : MDEV-24097 MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed +galera_ssl_reload : MDEV-24097 MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed +galera_toi_vote : MDEV-24097 MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed +galera_wsrep_schema_init : MDEV-24097 MTR sporadaically fails: Failed to start mysqld or mysql_shutdown failed From 136bcfdf7504092e7e49eba95e8be010da44c594 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia <psergey@askmonty.org> Date: Wed, 15 Dec 2021 15:12:06 +0300 Subject: [PATCH 13/21] MDEV-27270: Wrong query plan with Range Checked for Each Record and ORDER BY ... LIMIT Followup to fix for MDEV-25858: When test_if_skip_sort_order() decides to use an index to satisfy ORDER BY ... LIMIT clause, it should disable "Range Checked for Each Record" optimization. Do this in all cases. --- mysql-test/r/order_by_innodb.result | 23 +++++++++++++++++++++++ mysql-test/t/order_by_innodb.test | 22 ++++++++++++++++++++++ sql/sql_select.cc | 8 ++++++++ 3 files changed, 53 insertions(+) diff --git a/mysql-test/r/order_by_innodb.result b/mysql-test/r/order_by_innodb.result index 14b9b861a14..28922ef65f2 100644 --- a/mysql-test/r/order_by_innodb.result +++ b/mysql-test/r/order_by_innodb.result @@ -198,5 +198,28 @@ id id 1 NULL 2 1 3 3 +# +# MDEV-27270: Wrong query plan with Range Checked for Each Record and ORDER BY ... LIMIT +# +# This must NOT have "Range checked for each record" without any +# provisions to produce rows in the required ordering: +explain +select +t1.id,t2.id +from +t1 left join +t2 on t2.id2 = t1.id and +t2.id = (select dd.id +from t2 dd +where +dd.id2 = t1.id and +d1 > '2019-02-06 00:00:00' + order by +dd.d1, dd.d2, dd.id limit 1 +); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 index NULL PRIMARY 4 NULL # Using index +1 PRIMARY t2 eq_ref PRIMARY,id2 PRIMARY 4 func # Using where +2 DEPENDENT SUBQUERY dd range id2,for_latest_sort for_latest_sort 6 NULL # Using where drop table t1,t2; # End of 10.2 tests diff --git a/mysql-test/t/order_by_innodb.test b/mysql-test/t/order_by_innodb.test index 97c043b8dbc..af12644c073 100644 --- a/mysql-test/t/order_by_innodb.test +++ b/mysql-test/t/order_by_innodb.test @@ -184,6 +184,28 @@ from order by dd.d1 desc, dd.d2 desc, dd.id desc limit 1 ); + +--echo # +--echo # MDEV-27270: Wrong query plan with Range Checked for Each Record and ORDER BY ... LIMIT +--echo # + +--echo # This must NOT have "Range checked for each record" without any +--echo # provisions to produce rows in the required ordering: +--replace_column 9 # +explain +select + t1.id,t2.id +from + t1 left join + t2 on t2.id2 = t1.id and + t2.id = (select dd.id + from t2 dd + where + dd.id2 = t1.id and + d1 > '2019-02-06 00:00:00' + order by + dd.d1, dd.d2, dd.id limit 1 + ); drop table t1,t2; --echo # End of 10.2 tests diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 47422116e38..3079145c1b9 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -21986,7 +21986,15 @@ check_reverse_order: } } else if (select && select->quick) + { + /* Cancel "Range checked for each record" */ + if (tab->use_quick == 2) + { + tab->use_quick= 1; + tab->read_first_record= join_init_read_record; + } select->quick->need_sorted_output(); + } tab->read_record.unlock_row= (tab->type == JT_EQ_REF) ? join_read_key_unlock_row : rr_unlock_row; From fff8ac2e966ac123a091a3e1be22b59cbc1c7e9b Mon Sep 17 00:00:00 2001 From: Dmitry Shulga <dmitry.shulga@mariadb.com> Date: Mon, 8 Nov 2021 15:22:06 +0700 Subject: [PATCH 14/21] MDEV-21866: Assertion `!result' failed in convert_const_to_int upon 2nd execution of PS Consider the following use case: MariaDB [test]> CREATE TABLE t1 (field1 BIGINT DEFAULT -1); MariaDB [test]> CREATE VIEW v1 AS SELECT DISTINCT field1 FROM t1; Repeated execution of the following query as a Prepared Statement MariaDB [test]> PREPARE stmt FROM 'SELECT * FROM v1 WHERE field1 <=> NULL'; MariaDB [test]> EXECUTE stmt; results in a crash for a server built with DEBUG. MariaDB [test]> EXECUTE stmt; ERROR 2013 (HY000): Lost connection to MySQL server during query Assertion failed: (!result), function convert_const_to_int, file item_cmpfunc.cc, line 476. Abort trap: 6 (core dumped) The crash inside the function convert_const_to_int() happens by the reason that the value -1 is stored in an instance of the class Field_longlong on restoring its original value in the statement result= field->store(orig_field_val, TRUE); that leads to assigning the value 1 to the variable 'result' with subsequent crash in the DBUG_ASSERT statement following it DBUG_ASSERT(!result); The main matter here is why this assertion failure happens on the second execution of the prepared statement and doens't on the first one. On first handling of the statement 'EXECUTE stmt;' a temporary table is created for serving the query involving the view 'v1'. The table is created by the function create_tmp_table() in the following calls trace: (trace #1) JOIN::prepare (at sql_select.cc:725) st_select_lex::handle_derived LEX::handle_list_of_derived TABLE_LIST::handle_derived mysql_handle_single_derived mysql_derived_prepare select_union::create_result_table create_tmp_table Note, that the data member TABLE::status of a TABLE instance returned by the function create_tmp_table() has the value 0. Later the function setup_table_map() is called on the TABLE instance just created for the sake of the temporary table (calls trace #2 is below): JOIN::prepare (at sql_select.cc:737) setup_tables_and_check_access setup_tables setup_table_map where the data member TABLE::status is set to the value STATUS_NO_RECORD. After that when execution of the method JOIN::prepare reaches calling of the function setup_without_group() the following calls trace is invoked JOIN::prepare setup_without_group setup_conds Item_func::fix_fields Item_func_equal::fix_length_and_dec Item_bool_rowready_func2::fix_length_and_dec Item_func::setup_args_and_comparator Item_func::convert_const_compared_to_int_field convert_const_to_int There is the following code snippet in the function convert_const_to_int() at the line item_cmpfunc.cc:448 bool save_field_value= (field_item->const_item() || !(field->table->status & STATUS_NO_RECORD)); Since field->table->status has bits STATUS_NO_RECORD set the variable save_field_value is false and therefore neither the method Field_longlong::val_int() nor the method Field_longlong::store is called on the Field instance that has the numeric value -1. That is the reason why first execution of the Prepared Statement for the query 'SELECT * FROM v1 WHERE field1 <=> NULL' is successful. On second running of the statement 'EXECUTE stmt' a new temporary tables is also created by running the calls trace #1 but the trace #2 is not executed by the reason that data member SELECT_LEX::first_cond_optimization has been set to false on first execution of the prepared statemet (in the method JOIN::optimize_inner()). As a consequence, the data member TABLE::status for a temporary table just created doesn't have the flags STATUS_NO_RECORD set and therefore on re-execution of the prepared statement the methods Field_longlong::val_int() and Field_longlong::store() are called for the field having the value -1 and the DBUG_ASSERT(!result) is fired. To fix the issue the data member TABLE::status has to be assigned the value STATUS_NO_RECORD in every place where the macros empty_record() is called to emptify a record for just instantiated TABLE object created on behalf the new temporary table. --- mysql-test/r/ps.result | 13 +++++++++++++ mysql-test/t/ps.test | 16 ++++++++++++++++ sql/sql_select.cc | 3 ++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/ps.result b/mysql-test/r/ps.result index f1f779ef03f..1cd41e5b3a4 100644 --- a/mysql-test/r/ps.result +++ b/mysql-test/r/ps.result @@ -5474,5 +5474,18 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables DEALLOCATE PREPARE stmt; DROP TABLE t1, t2, t3; +# +# MDEV-21866: Assertion `!result' failed in convert_const_to_int upon 2nd execution of PS +# +CREATE TABLE t1 (a BIGINT DEFAULT -1); +CREATE VIEW v1 AS SELECT DISTINCT a FROM t1; +PREPARE stmt FROM 'SELECT * FROM v1 WHERE a <=> NULL'; +EXECUTE stmt; +a +EXECUTE stmt; +a +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1; # End of 10.2 tests # diff --git a/mysql-test/t/ps.test b/mysql-test/t/ps.test index 2e7b43ad748..4097d28a949 100644 --- a/mysql-test/t/ps.test +++ b/mysql-test/t/ps.test @@ -4961,5 +4961,21 @@ EXECUTE stmt; DEALLOCATE PREPARE stmt; DROP TABLE t1, t2, t3; +--echo # +--echo # MDEV-21866: Assertion `!result' failed in convert_const_to_int upon 2nd execution of PS +--echo # + +CREATE TABLE t1 (a BIGINT DEFAULT -1); +CREATE VIEW v1 AS SELECT DISTINCT a FROM t1; +PREPARE stmt FROM 'SELECT * FROM v1 WHERE a <=> NULL'; +EXECUTE stmt; +EXECUTE stmt; + +# Cleanup +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1; + + --echo # End of 10.2 tests --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 3079145c1b9..a331f4f3dbc 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -17556,7 +17556,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields, // Make empty record so random data is not written to disk empty_record(table); - + table->status= STATUS_NO_RECORD; thd->mem_root= mem_root_save; DBUG_RETURN(table); @@ -18557,6 +18557,7 @@ bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, return TRUE; // Make empty record so random data is not written to disk empty_record(table); + table->status= STATUS_NO_RECORD; } if (open_tmp_table(table)) return TRUE; From a65d01a4cf04c744f9355bec186430540ce6ba08 Mon Sep 17 00:00:00 2001 From: Dmitry Shulga <dmitry.shulga@mariadb.com> Date: Wed, 20 Oct 2021 19:24:31 +0700 Subject: [PATCH 15/21] MDEV-23182: Server crashes in Item::fix_fields_if_needed / table_value_constr::prepare upon 2nd execution of PS Repeating execution of a query containing the clause IN with string literals in environment where the server variable in_predicate_conversion_threshold is set results in server abnormal termination in case the query is run as a Prepared Statement and conversion of charsets for string values in the query are required. The reason for server abnormal termination is that instances of the class Item_string created on transforming the IN clause into subquery were created on runtime memory root that is deallocated on finishing execution of Prepared statement. On the other hand, references to Items placed on deallocated memory root still exist in objects of the class table_value_constr. Subsequent running of the same prepared statement leads to dereferencing of pointers to already deallocated memory that could lead to undefined behaviour. To fix the issue the values being pushed into a values list for TVC are created by cloning their original items. This way the cloned items are allocate on the PS memroot and as consequences no dangling pointer does more exist. --- mysql-test/main/table_value_constr.result | 42 +++++++++++++++++++++ mysql-test/main/table_value_constr.test | 46 +++++++++++++++++++++++ sql/sql_tvc.cc | 8 +++- 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/table_value_constr.result b/mysql-test/main/table_value_constr.result index 1d1cd054556..7ff5fb7fa5e 100644 --- a/mysql-test/main/table_value_constr.result +++ b/mysql-test/main/table_value_constr.result @@ -3100,4 +3100,46 @@ select * from (values (3),(7),(1) union values (2),(4) order by 1 limit 2) as dt 1 2 drop table t1; +# +# MDEV-23182: Server crashes in +# Item::fix_fields_if_needed / table_value_constr::prepare upon 2nd execution of PS +# +SET @save_in_predicate_conversion_threshold=@@in_predicate_conversion_threshold; +SET in_predicate_conversion_threshold=2; +CREATE TABLE t1 (c VARCHAR(10)) DEFAULT CHARSET=utf8; +PREPARE stmt FROM "SELECT * FROM t1 WHERE c IN ('10','20')"; +EXECUTE stmt; +c +# Without the patch second execution of the prepared statement 'stmt' +# results in crash. +EXECUTE stmt; +c +DEALLOCATE PREPARE stmt; +DROP TABLE t1; +# Check that the query without conversion doesn't crash server +CREATE TABLE t1 (c VARCHAR(10)); +PREPARE stmt FROM "SELECT * FROM t1 WHERE c IN ('10','20')"; +EXECUTE stmt; +c +EXECUTE stmt; +c +DEALLOCATE PREPARE stmt; +DROP TABLE t1; +# Test case for a row expression in the left part of the IN clause +CREATE TABLE t1 (a VARCHAR(3), b VARCHAR(3)) DEFAULT CHARSET=utf8; +PREPARE stmt FROM "SELECT * FROM t1 WHERE (a, b) IN (('10', '10'), ('20', '20'))"; +EXECUTE stmt; +a b +EXECUTE stmt; +a b +DROP TABLE t1; +# Check that the query without conversion is handled successfully +CREATE TABLE t1 (a VARCHAR(3), b VARCHAR(3)); +PREPARE stmt FROM "SELECT * FROM t1 WHERE (a, b) IN (('10', '10'), ('20', '20'))"; +EXECUTE stmt; +a b +EXECUTE stmt; +a b +DROP TABLE t1; +SET @@in_predicate_conversion_threshold = @save_in_predicate_conversion_threshold; End of 10.3 tests diff --git a/mysql-test/main/table_value_constr.test b/mysql-test/main/table_value_constr.test index d13962579cc..673e7dcdc6e 100644 --- a/mysql-test/main/table_value_constr.test +++ b/mysql-test/main/table_value_constr.test @@ -1650,4 +1650,50 @@ select * from (values (3),(7),(1) union values (2),(4) order by 1 limit 2) as dt drop table t1; +--echo # +--echo # MDEV-23182: Server crashes in +--echo # Item::fix_fields_if_needed / table_value_constr::prepare upon 2nd execution of PS +--echo # +SET @save_in_predicate_conversion_threshold=@@in_predicate_conversion_threshold; +SET in_predicate_conversion_threshold=2; + +CREATE TABLE t1 (c VARCHAR(10)) DEFAULT CHARSET=utf8; +PREPARE stmt FROM "SELECT * FROM t1 WHERE c IN ('10','20')"; +EXECUTE stmt; +--echo # Without the patch second execution of the prepared statement 'stmt' +--echo # results in crash. +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +DROP TABLE t1; + +--echo # Check that the query without conversion doesn't crash server +CREATE TABLE t1 (c VARCHAR(10)); +PREPARE stmt FROM "SELECT * FROM t1 WHERE c IN ('10','20')"; +EXECUTE stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +DROP TABLE t1; + +--echo # Test case for a row expression in the left part of the IN clause +CREATE TABLE t1 (a VARCHAR(3), b VARCHAR(3)) DEFAULT CHARSET=utf8; +PREPARE stmt FROM "SELECT * FROM t1 WHERE (a, b) IN (('10', '10'), ('20', '20'))"; + +EXECUTE stmt; +EXECUTE stmt; + +DROP TABLE t1; + +--echo # Check that the query without conversion is handled successfully +CREATE TABLE t1 (a VARCHAR(3), b VARCHAR(3)); +PREPARE stmt FROM "SELECT * FROM t1 WHERE (a, b) IN (('10', '10'), ('20', '20'))"; + +EXECUTE stmt; +EXECUTE stmt; + +DROP TABLE t1; + +SET @@in_predicate_conversion_threshold = @save_in_predicate_conversion_threshold; + --echo End of 10.3 tests diff --git a/sql/sql_tvc.cc b/sql/sql_tvc.cc index 96c5223ee6a..3866b7c9352 100644 --- a/sql/sql_tvc.cc +++ b/sql/sql_tvc.cc @@ -539,7 +539,10 @@ bool Item_func_in::create_value_list_for_tvc(THD *thd, if (is_list_of_rows) { - Item_row *row_list= (Item_row *)(args[i]); + Item_row *row_list= (Item_row *)(args[i]->build_clone(thd)); + + if (!row_list) + return true; for (uint j=0; j < row_list->cols(); j++) { @@ -561,7 +564,8 @@ bool Item_func_in::create_value_list_for_tvc(THD *thd, sprintf(col_name, "_col_%i", 1); args[i]->set_name(thd, col_name, strlen(col_name), thd->charset()); } - if (tvc_value->push_back(args[i]->real_item())) + Item *arg_clone= args[i]->build_clone(thd); + if (!arg_clone || tvc_value->push_back(arg_clone)) return true; } From 3fd80d08740d3ad2426bfb8eb9debc40d96e2a20 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov <midenok@gmail.com> Date: Thu, 16 Dec 2021 23:13:45 +0300 Subject: [PATCH 16/21] MDEV-27244 Table corruption upon adding serial data type MDEV-25803 excluded some cases from key sort upon alter table. That particularly depends on ALTER_ADD_INDEX flag. Creating a column of SERIAL data type missed that flag. Though equivalent operation alter table t1 add x bigint unsigned not null auto_increment unique; has ALTER_ADD_INDEX flag. --- mysql-test/suite/versioning/r/partition.result | 9 +++++++++ mysql-test/suite/versioning/t/partition.test | 10 ++++++++++ sql/sql_yacc.yy | 1 + sql/sql_yacc_ora.yy | 1 + 4 files changed, 21 insertions(+) diff --git a/mysql-test/suite/versioning/r/partition.result b/mysql-test/suite/versioning/r/partition.result index 6472c0480c3..8344e91a34f 100644 --- a/mysql-test/suite/versioning/r/partition.result +++ b/mysql-test/suite/versioning/r/partition.result @@ -746,4 +746,13 @@ explain partitions select * from t1; id select_type table partitions type possible_keys key key_len ref rows Extra 1 SIMPLE t1 pn # NULL NULL NULL NULL # # drop table t1; +# +# MDEV-27244 Table corruption upon adding serial data type +# +create table t1 (f int, key(f)) with system versioning +partition by system_time limit 10 (partition p0 history, partition pn current); +alter table t1 add x serial; +alter table t1 add partition (partition p1 history); +alter table t1 add partition (partition p2 history); +drop table t1; # End of 10.3 tests diff --git a/mysql-test/suite/versioning/t/partition.test b/mysql-test/suite/versioning/t/partition.test index c5f6b242b53..6eeb305c218 100644 --- a/mysql-test/suite/versioning/t/partition.test +++ b/mysql-test/suite/versioning/t/partition.test @@ -727,6 +727,16 @@ explain partitions select * from t1 for system_time as of '2000-01-01 02:00:00'; explain partitions select * from t1; drop table t1; +--echo # +--echo # MDEV-27244 Table corruption upon adding serial data type +--echo # +create table t1 (f int, key(f)) with system versioning +partition by system_time limit 10 (partition p0 history, partition pn current); +alter table t1 add x serial; +alter table t1 add partition (partition p1 history); +alter table t1 add partition (partition p2 history); +drop table t1; + --echo # End of 10.3 tests --source suite/versioning/common_finish.inc diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index b26ddcbb9e5..01e3922040b 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -6783,6 +6783,7 @@ field_type_or_serial: Lex->last_field->set_handler(&type_handler_longlong); Lex->last_field->flags|= AUTO_INCREMENT_FLAG | NOT_NULL_FLAG | UNSIGNED_FLAG | UNIQUE_KEY_FLAG; + Lex->alter_info.flags|= ALTER_ADD_INDEX; } opt_serial_attribute ; diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index 93315add399..5e17e41a940 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -6625,6 +6625,7 @@ field_type_or_serial: Lex->last_field->set_handler(&type_handler_longlong); Lex->last_field->flags|= AUTO_INCREMENT_FLAG | NOT_NULL_FLAG | UNSIGNED_FLAG | UNIQUE_KEY_FLAG; + Lex->alter_info.flags|= ALTER_ADD_INDEX; } opt_serial_attribute ; From 85defc4764860613742af23acf954cfdd9397553 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky <julius.goryavsky@mariadb.com> Date: Fri, 17 Dec 2021 04:44:43 +0100 Subject: [PATCH 17/21] MDEV-27181 fixup: compatibility with Windows + small corrections 1) Removed symlinks that are not very well supported in tar under Windows. 2) Added comment + changed code formatting in viosslfactories.c 3) Fixed a small bug in the yassl code. 4) Fixed a typo in the script code. --- extra/yassl/src/ssl.cpp | 2 +- mysql-test/std_data/capath/3106f582.0 | 80 +++++++++++++++++++++++++- mysql-test/std_data/capath/cacert.pem | 79 -------------------------- mysql-test/std_data/capath/ed1f42db.0 | 80 +++++++++++++++++++++++++- scripts/wsrep_sst_common.sh | 2 +- vio/viosslfactories.c | 81 +++++++++++++++++++++++---- 6 files changed, 229 insertions(+), 95 deletions(-) mode change 120000 => 100644 mysql-test/std_data/capath/3106f582.0 delete mode 100644 mysql-test/std_data/capath/cacert.pem mode change 120000 => 100644 mysql-test/std_data/capath/ed1f42db.0 diff --git a/extra/yassl/src/ssl.cpp b/extra/yassl/src/ssl.cpp index c4adb4692be..eac691315b7 100644 --- a/extra/yassl/src/ssl.cpp +++ b/extra/yassl/src/ssl.cpp @@ -773,7 +773,7 @@ void SSL_CTX_set_verify(SSL_CTX* ctx, int mode, VerifyCallback vc) int SSL_CTX_load_verify_locations(SSL_CTX* ctx, const char* file, const char* path) { - int ret = SSL_FAILURE; + int ret = SSL_SUCCESS; if (file) ret = read_file(ctx, file, SSL_FILETYPE_PEM, CA); diff --git a/mysql-test/std_data/capath/3106f582.0 b/mysql-test/std_data/capath/3106f582.0 deleted file mode 120000 index 1310cfcff20..00000000000 --- a/mysql-test/std_data/capath/3106f582.0 +++ /dev/null @@ -1 +0,0 @@ -cacert.pem \ No newline at end of file diff --git a/mysql-test/std_data/capath/3106f582.0 b/mysql-test/std_data/capath/3106f582.0 new file mode 100644 index 00000000000..23dda2318e1 --- /dev/null +++ b/mysql-test/std_data/capath/3106f582.0 @@ -0,0 +1,79 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + d0:4d:23:85:ee:59:b3:fa + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB + Validity + Not Before: Jan 27 10:11:10 2019 GMT + Not After : Jan 22 10:11:10 2039 GMT + Subject: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e8:0e:a7:84:d3:75:30:06:30:b2:10:b9:d1:88: + 36:2b:5e:f8:c8:44:57:cb:67:72:ab:96:95:33:d5: + 88:d1:8f:23:50:98:ba:6d:20:00:80:bd:35:d5:c1: + bf:98:49:c4:0a:15:4a:34:a6:21:9b:2e:8c:15:09: + f0:63:81:02:c2:7c:e2:53:e0:f7:a1:1a:40:5e:8f: + 41:4a:4c:56:d4:20:f1:d5:a7:c1:53:2e:ff:7e:37: + 17:cc:7e:74:bd:e2:22:33:ce:8c:77:62:a4:c5:3f: + 44:35:7b:7e:b9:f5:7d:8c:7a:27:58:fd:2c:42:86: + 2e:e7:6b:01:99:7b:fe:7d:a7:a1:4f:3e:39:39:54: + 1f:61:de:74:66:d1:77:4f:43:1b:66:70:29:85:de: + fc:8f:8e:1b:7b:a2:66:48:26:7f:9b:a6:fd:4a:e4: + dc:eb:ed:bd:f8:e3:f1:57:98:13:6f:f1:a3:2a:e3: + 73:bd:8d:7c:6f:4b:59:35:bc:b5:42:3e:99:a7:13: + 8d:be:2e:5c:9a:c6:5b:ab:ae:bf:00:e9:c8:ee:05: + 22:8e:d5:67:1a:47:9a:6d:9c:f9:42:3e:15:34:f8: + 31:ec:b4:7e:d3:92:95:b0:b8:f9:66:f3:bd:1d:31: + 2c:b1:90:62:a1:f8:4e:a6:5d:26:22:f0:e1:fe:16: + 2b:69 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 + X509v3 Authority Key Identifier: + keyid:CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha256WithRSAEncryption + df:fd:74:29:5b:5e:9a:8b:09:02:40:59:73:cb:71:47:3f:97: + 3d:a9:fd:c4:8c:01:29:c9:86:b8:71:55:ff:72:0e:50:dc:c8: + b5:e6:91:41:52:47:21:30:cc:4d:e7:3b:4b:db:55:ea:7d:46: + eb:53:e0:b7:1b:80:7c:b1:0c:d3:d1:bc:a0:73:ae:96:1f:fd: + 05:52:7e:54:d5:03:52:69:7b:34:5f:27:d7:98:da:98:76:73: + e6:bb:50:59:2a:94:90:67:03:1c:a4:76:2f:ee:ef:59:60:09: + 48:33:03:2b:52:ed:83:42:f8:71:19:7f:d8:be:40:ed:20:01: + 90:3c:7e:1c:8b:d2:9f:f3:2f:09:1f:50:c8:10:e1:8a:d9:a5: + 49:9c:0b:74:17:b9:2b:68:f6:1e:73:c2:73:10:38:b3:35:e2: + 87:91:1b:a1:d1:9b:81:9d:1b:32:cc:03:6e:4c:82:95:81:11: + 42:56:e2:16:2b:22:65:db:40:2c:ca:dc:03:f4:d5:07:cf:f5: + 13:b2:cf:51:5b:24:cd:c7:d1:9b:42:8e:f9:df:5d:1e:5a:09: + a3:4f:a9:0b:f4:21:c5:bb:ff:02:93:67:e8:2d:ee:ab:d9:59: + 76:03:2c:a1:bd:fb:dc:af:b6:82:94:71:85:53:a8:18:0d:3a: + 9e:42:eb:59 +-----BEGIN CERTIFICATE----- +MIIDfzCCAmegAwIBAgIJANBNI4XuWbP6MA0GCSqGSIb3DQEBCwUAMFYxDzANBgNV +BAMMBmNhY2VydDELMAkGA1UEBhMCRkkxETAPBgNVBAgMCEhlbHNpbmtpMREwDwYD +VQQHDAhIZWxzaW5raTEQMA4GA1UECgwHTWFyaWFEQjAeFw0xOTAxMjcxMDExMTBa +Fw0zOTAxMjIxMDExMTBaMFYxDzANBgNVBAMMBmNhY2VydDELMAkGA1UEBhMCRkkx +ETAPBgNVBAgMCEhlbHNpbmtpMREwDwYDVQQHDAhIZWxzaW5raTEQMA4GA1UECgwH +TWFyaWFEQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOgOp4TTdTAG +MLIQudGINite+MhEV8tncquWlTPViNGPI1CYum0gAIC9NdXBv5hJxAoVSjSmIZsu +jBUJ8GOBAsJ84lPg96EaQF6PQUpMVtQg8dWnwVMu/343F8x+dL3iIjPOjHdipMU/ +RDV7frn1fYx6J1j9LEKGLudrAZl7/n2noU8+OTlUH2HedGbRd09DG2ZwKYXe/I+O +G3uiZkgmf5um/Urk3Ovtvfjj8VeYE2/xoyrjc72NfG9LWTW8tUI+macTjb4uXJrG +W6uuvwDpyO4FIo7VZxpHmm2c+UI+FTT4Mey0ftOSlbC4+WbzvR0xLLGQYqH4TqZd +JiLw4f4WK2kCAwEAAaNQME4wHQYDVR0OBBYEFMpxmYnwcqt1ZrtlagMEcqV7laaT +MB8GA1UdIwQYMBaAFMpxmYnwcqt1ZrtlagMEcqV7laaTMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggEBAN/9dClbXpqLCQJAWXPLcUc/lz2p/cSMASnJhrhx +Vf9yDlDcyLXmkUFSRyEwzE3nO0vbVep9RutT4LcbgHyxDNPRvKBzrpYf/QVSflTV +A1JpezRfJ9eY2ph2c+a7UFkqlJBnAxykdi/u71lgCUgzAytS7YNC+HEZf9i+QO0g +AZA8fhyL0p/zLwkfUMgQ4YrZpUmcC3QXuSto9h5zwnMQOLM14oeRG6HRm4GdGzLM +A25MgpWBEUJW4hYrImXbQCzK3AP01QfP9ROyz1FbJM3H0ZtCjvnfXR5aCaNPqQv0 +IcW7/wKTZ+gt7qvZWXYDLKG9+9yvtoKUcYVTqBgNOp5C61k= +-----END CERTIFICATE----- diff --git a/mysql-test/std_data/capath/cacert.pem b/mysql-test/std_data/capath/cacert.pem deleted file mode 100644 index 23dda2318e1..00000000000 --- a/mysql-test/std_data/capath/cacert.pem +++ /dev/null @@ -1,79 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - d0:4d:23:85:ee:59:b3:fa - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB - Validity - Not Before: Jan 27 10:11:10 2019 GMT - Not After : Jan 22 10:11:10 2039 GMT - Subject: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:e8:0e:a7:84:d3:75:30:06:30:b2:10:b9:d1:88: - 36:2b:5e:f8:c8:44:57:cb:67:72:ab:96:95:33:d5: - 88:d1:8f:23:50:98:ba:6d:20:00:80:bd:35:d5:c1: - bf:98:49:c4:0a:15:4a:34:a6:21:9b:2e:8c:15:09: - f0:63:81:02:c2:7c:e2:53:e0:f7:a1:1a:40:5e:8f: - 41:4a:4c:56:d4:20:f1:d5:a7:c1:53:2e:ff:7e:37: - 17:cc:7e:74:bd:e2:22:33:ce:8c:77:62:a4:c5:3f: - 44:35:7b:7e:b9:f5:7d:8c:7a:27:58:fd:2c:42:86: - 2e:e7:6b:01:99:7b:fe:7d:a7:a1:4f:3e:39:39:54: - 1f:61:de:74:66:d1:77:4f:43:1b:66:70:29:85:de: - fc:8f:8e:1b:7b:a2:66:48:26:7f:9b:a6:fd:4a:e4: - dc:eb:ed:bd:f8:e3:f1:57:98:13:6f:f1:a3:2a:e3: - 73:bd:8d:7c:6f:4b:59:35:bc:b5:42:3e:99:a7:13: - 8d:be:2e:5c:9a:c6:5b:ab:ae:bf:00:e9:c8:ee:05: - 22:8e:d5:67:1a:47:9a:6d:9c:f9:42:3e:15:34:f8: - 31:ec:b4:7e:d3:92:95:b0:b8:f9:66:f3:bd:1d:31: - 2c:b1:90:62:a1:f8:4e:a6:5d:26:22:f0:e1:fe:16: - 2b:69 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Key Identifier: - CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 - X509v3 Authority Key Identifier: - keyid:CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 - - X509v3 Basic Constraints: - CA:TRUE - Signature Algorithm: sha256WithRSAEncryption - df:fd:74:29:5b:5e:9a:8b:09:02:40:59:73:cb:71:47:3f:97: - 3d:a9:fd:c4:8c:01:29:c9:86:b8:71:55:ff:72:0e:50:dc:c8: - b5:e6:91:41:52:47:21:30:cc:4d:e7:3b:4b:db:55:ea:7d:46: - eb:53:e0:b7:1b:80:7c:b1:0c:d3:d1:bc:a0:73:ae:96:1f:fd: - 05:52:7e:54:d5:03:52:69:7b:34:5f:27:d7:98:da:98:76:73: - e6:bb:50:59:2a:94:90:67:03:1c:a4:76:2f:ee:ef:59:60:09: - 48:33:03:2b:52:ed:83:42:f8:71:19:7f:d8:be:40:ed:20:01: - 90:3c:7e:1c:8b:d2:9f:f3:2f:09:1f:50:c8:10:e1:8a:d9:a5: - 49:9c:0b:74:17:b9:2b:68:f6:1e:73:c2:73:10:38:b3:35:e2: - 87:91:1b:a1:d1:9b:81:9d:1b:32:cc:03:6e:4c:82:95:81:11: - 42:56:e2:16:2b:22:65:db:40:2c:ca:dc:03:f4:d5:07:cf:f5: - 13:b2:cf:51:5b:24:cd:c7:d1:9b:42:8e:f9:df:5d:1e:5a:09: - a3:4f:a9:0b:f4:21:c5:bb:ff:02:93:67:e8:2d:ee:ab:d9:59: - 76:03:2c:a1:bd:fb:dc:af:b6:82:94:71:85:53:a8:18:0d:3a: - 9e:42:eb:59 ------BEGIN CERTIFICATE----- -MIIDfzCCAmegAwIBAgIJANBNI4XuWbP6MA0GCSqGSIb3DQEBCwUAMFYxDzANBgNV -BAMMBmNhY2VydDELMAkGA1UEBhMCRkkxETAPBgNVBAgMCEhlbHNpbmtpMREwDwYD -VQQHDAhIZWxzaW5raTEQMA4GA1UECgwHTWFyaWFEQjAeFw0xOTAxMjcxMDExMTBa -Fw0zOTAxMjIxMDExMTBaMFYxDzANBgNVBAMMBmNhY2VydDELMAkGA1UEBhMCRkkx -ETAPBgNVBAgMCEhlbHNpbmtpMREwDwYDVQQHDAhIZWxzaW5raTEQMA4GA1UECgwH -TWFyaWFEQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOgOp4TTdTAG -MLIQudGINite+MhEV8tncquWlTPViNGPI1CYum0gAIC9NdXBv5hJxAoVSjSmIZsu -jBUJ8GOBAsJ84lPg96EaQF6PQUpMVtQg8dWnwVMu/343F8x+dL3iIjPOjHdipMU/ -RDV7frn1fYx6J1j9LEKGLudrAZl7/n2noU8+OTlUH2HedGbRd09DG2ZwKYXe/I+O -G3uiZkgmf5um/Urk3Ovtvfjj8VeYE2/xoyrjc72NfG9LWTW8tUI+macTjb4uXJrG -W6uuvwDpyO4FIo7VZxpHmm2c+UI+FTT4Mey0ftOSlbC4+WbzvR0xLLGQYqH4TqZd -JiLw4f4WK2kCAwEAAaNQME4wHQYDVR0OBBYEFMpxmYnwcqt1ZrtlagMEcqV7laaT -MB8GA1UdIwQYMBaAFMpxmYnwcqt1ZrtlagMEcqV7laaTMAwGA1UdEwQFMAMBAf8w -DQYJKoZIhvcNAQELBQADggEBAN/9dClbXpqLCQJAWXPLcUc/lz2p/cSMASnJhrhx -Vf9yDlDcyLXmkUFSRyEwzE3nO0vbVep9RutT4LcbgHyxDNPRvKBzrpYf/QVSflTV -A1JpezRfJ9eY2ph2c+a7UFkqlJBnAxykdi/u71lgCUgzAytS7YNC+HEZf9i+QO0g -AZA8fhyL0p/zLwkfUMgQ4YrZpUmcC3QXuSto9h5zwnMQOLM14oeRG6HRm4GdGzLM -A25MgpWBEUJW4hYrImXbQCzK3AP01QfP9ROyz1FbJM3H0ZtCjvnfXR5aCaNPqQv0 -IcW7/wKTZ+gt7qvZWXYDLKG9+9yvtoKUcYVTqBgNOp5C61k= ------END CERTIFICATE----- diff --git a/mysql-test/std_data/capath/ed1f42db.0 b/mysql-test/std_data/capath/ed1f42db.0 deleted file mode 120000 index 1310cfcff20..00000000000 --- a/mysql-test/std_data/capath/ed1f42db.0 +++ /dev/null @@ -1 +0,0 @@ -cacert.pem \ No newline at end of file diff --git a/mysql-test/std_data/capath/ed1f42db.0 b/mysql-test/std_data/capath/ed1f42db.0 new file mode 100644 index 00000000000..23dda2318e1 --- /dev/null +++ b/mysql-test/std_data/capath/ed1f42db.0 @@ -0,0 +1,79 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + d0:4d:23:85:ee:59:b3:fa + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB + Validity + Not Before: Jan 27 10:11:10 2019 GMT + Not After : Jan 22 10:11:10 2039 GMT + Subject: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e8:0e:a7:84:d3:75:30:06:30:b2:10:b9:d1:88: + 36:2b:5e:f8:c8:44:57:cb:67:72:ab:96:95:33:d5: + 88:d1:8f:23:50:98:ba:6d:20:00:80:bd:35:d5:c1: + bf:98:49:c4:0a:15:4a:34:a6:21:9b:2e:8c:15:09: + f0:63:81:02:c2:7c:e2:53:e0:f7:a1:1a:40:5e:8f: + 41:4a:4c:56:d4:20:f1:d5:a7:c1:53:2e:ff:7e:37: + 17:cc:7e:74:bd:e2:22:33:ce:8c:77:62:a4:c5:3f: + 44:35:7b:7e:b9:f5:7d:8c:7a:27:58:fd:2c:42:86: + 2e:e7:6b:01:99:7b:fe:7d:a7:a1:4f:3e:39:39:54: + 1f:61:de:74:66:d1:77:4f:43:1b:66:70:29:85:de: + fc:8f:8e:1b:7b:a2:66:48:26:7f:9b:a6:fd:4a:e4: + dc:eb:ed:bd:f8:e3:f1:57:98:13:6f:f1:a3:2a:e3: + 73:bd:8d:7c:6f:4b:59:35:bc:b5:42:3e:99:a7:13: + 8d:be:2e:5c:9a:c6:5b:ab:ae:bf:00:e9:c8:ee:05: + 22:8e:d5:67:1a:47:9a:6d:9c:f9:42:3e:15:34:f8: + 31:ec:b4:7e:d3:92:95:b0:b8:f9:66:f3:bd:1d:31: + 2c:b1:90:62:a1:f8:4e:a6:5d:26:22:f0:e1:fe:16: + 2b:69 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 + X509v3 Authority Key Identifier: + keyid:CA:71:99:89:F0:72:AB:75:66:BB:65:6A:03:04:72:A5:7B:95:A6:93 + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha256WithRSAEncryption + df:fd:74:29:5b:5e:9a:8b:09:02:40:59:73:cb:71:47:3f:97: + 3d:a9:fd:c4:8c:01:29:c9:86:b8:71:55:ff:72:0e:50:dc:c8: + b5:e6:91:41:52:47:21:30:cc:4d:e7:3b:4b:db:55:ea:7d:46: + eb:53:e0:b7:1b:80:7c:b1:0c:d3:d1:bc:a0:73:ae:96:1f:fd: + 05:52:7e:54:d5:03:52:69:7b:34:5f:27:d7:98:da:98:76:73: + e6:bb:50:59:2a:94:90:67:03:1c:a4:76:2f:ee:ef:59:60:09: + 48:33:03:2b:52:ed:83:42:f8:71:19:7f:d8:be:40:ed:20:01: + 90:3c:7e:1c:8b:d2:9f:f3:2f:09:1f:50:c8:10:e1:8a:d9:a5: + 49:9c:0b:74:17:b9:2b:68:f6:1e:73:c2:73:10:38:b3:35:e2: + 87:91:1b:a1:d1:9b:81:9d:1b:32:cc:03:6e:4c:82:95:81:11: + 42:56:e2:16:2b:22:65:db:40:2c:ca:dc:03:f4:d5:07:cf:f5: + 13:b2:cf:51:5b:24:cd:c7:d1:9b:42:8e:f9:df:5d:1e:5a:09: + a3:4f:a9:0b:f4:21:c5:bb:ff:02:93:67:e8:2d:ee:ab:d9:59: + 76:03:2c:a1:bd:fb:dc:af:b6:82:94:71:85:53:a8:18:0d:3a: + 9e:42:eb:59 +-----BEGIN CERTIFICATE----- +MIIDfzCCAmegAwIBAgIJANBNI4XuWbP6MA0GCSqGSIb3DQEBCwUAMFYxDzANBgNV +BAMMBmNhY2VydDELMAkGA1UEBhMCRkkxETAPBgNVBAgMCEhlbHNpbmtpMREwDwYD +VQQHDAhIZWxzaW5raTEQMA4GA1UECgwHTWFyaWFEQjAeFw0xOTAxMjcxMDExMTBa +Fw0zOTAxMjIxMDExMTBaMFYxDzANBgNVBAMMBmNhY2VydDELMAkGA1UEBhMCRkkx +ETAPBgNVBAgMCEhlbHNpbmtpMREwDwYDVQQHDAhIZWxzaW5raTEQMA4GA1UECgwH +TWFyaWFEQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOgOp4TTdTAG +MLIQudGINite+MhEV8tncquWlTPViNGPI1CYum0gAIC9NdXBv5hJxAoVSjSmIZsu +jBUJ8GOBAsJ84lPg96EaQF6PQUpMVtQg8dWnwVMu/343F8x+dL3iIjPOjHdipMU/ +RDV7frn1fYx6J1j9LEKGLudrAZl7/n2noU8+OTlUH2HedGbRd09DG2ZwKYXe/I+O +G3uiZkgmf5um/Urk3Ovtvfjj8VeYE2/xoyrjc72NfG9LWTW8tUI+macTjb4uXJrG +W6uuvwDpyO4FIo7VZxpHmm2c+UI+FTT4Mey0ftOSlbC4+WbzvR0xLLGQYqH4TqZd +JiLw4f4WK2kCAwEAAaNQME4wHQYDVR0OBBYEFMpxmYnwcqt1ZrtlagMEcqV7laaT +MB8GA1UdIwQYMBaAFMpxmYnwcqt1ZrtlagMEcqV7laaTMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggEBAN/9dClbXpqLCQJAWXPLcUc/lz2p/cSMASnJhrhx +Vf9yDlDcyLXmkUFSRyEwzE3nO0vbVep9RutT4LcbgHyxDNPRvKBzrpYf/QVSflTV +A1JpezRfJ9eY2ph2c+a7UFkqlJBnAxykdi/u71lgCUgzAytS7YNC+HEZf9i+QO0g +AZA8fhyL0p/zLwkfUMgQ4YrZpUmcC3QXuSto9h5zwnMQOLM14oeRG6HRm4GdGzLM +A25MgpWBEUJW4hYrImXbQCzK3AP01QfP9ROyz1FbJM3H0ZtCjvnfXR5aCaNPqQv0 +IcW7/wKTZ+gt7qvZWXYDLKG9+9yvtoKUcYVTqBgNOp5C61k= +-----END CERTIFICATE----- diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 5c84aa7c17f..040cd31caa9 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1111,7 +1111,7 @@ verify_ca_matches_cert() [ -n "$ca" ] && [ ! -r "$ca" ] && readable=0 [ -n "$cap" ] && [ ! -r "$cap" ] && readable=0 - if [ readable -eq 0 ]; then + if [ $readable -eq 0 ]; then wsrep_log_error \ "Both PEM file and CA file (or path) must be readable" exit 22 diff --git a/vio/viosslfactories.c b/vio/viosslfactories.c index 08f0905e044..fe980322736 100644 --- a/vio/viosslfactories.c +++ b/vio/viosslfactories.c @@ -179,10 +179,29 @@ new_VioSSLFd(const char *key_file, const char *cert_file, long ssl_ctx_options= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3; DBUG_ENTER("new_VioSSLFd"); - if (ca_file && ! ca_file[0]) ca_file = NULL; - if (ca_path && ! ca_path[0]) ca_path = NULL; - if (crl_file && ! crl_file[0]) crl_file = NULL; - if (crl_path && ! crl_path[0]) crl_path = NULL; + /* + If some optional parameters indicate empty strings, then + for compatibility with SSL libraries, replace them with NULL, + otherwise these libraries will try to open files with an empty + name, etc., and they will return an error code instead performing + the necessary operations: + */ + if (ca_file && !ca_file[0]) + { + ca_file = NULL; + } + if (ca_path && !ca_path[0]) + { + ca_path = NULL; + } + if (crl_file && !crl_file[0]) + { + crl_file = NULL; + } + if (crl_path && !crl_path[0]) + { + crl_path = NULL; + } DBUG_PRINT("enter", ("key_file: '%s' cert_file: '%s' ca_file: '%s' ca_path: '%s' " @@ -314,10 +333,29 @@ new_VioSSLConnectorFd(const char *key_file, const char *cert_file, struct st_VioSSLFd *ssl_fd; int verify= SSL_VERIFY_PEER; - if (ca_file && ! ca_file[0]) ca_file = NULL; - if (ca_path && ! ca_path[0]) ca_path = NULL; - if (crl_file && ! crl_file[0]) crl_file = NULL; - if (crl_path && ! crl_path[0]) crl_path = NULL; + /* + If some optional parameters indicate empty strings, then + for compatibility with SSL libraries, replace them with NULL, + otherwise these libraries will try to open files with an empty + name, etc., and they will return an error code instead performing + the necessary operations: + */ + if (ca_file && !ca_file[0]) + { + ca_file = NULL; + } + if (ca_path && !ca_path[0]) + { + ca_path = NULL; + } + if (crl_file && !crl_file[0]) + { + crl_file = NULL; + } + if (crl_path && !crl_path[0]) + { + crl_path = NULL; + } /* Turn off verification of servers certificate if both @@ -351,10 +389,29 @@ new_VioSSLAcceptorFd(const char *key_file, const char *cert_file, struct st_VioSSLFd *ssl_fd; int verify= SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE; - if (ca_file && ! ca_file[0]) ca_file = NULL; - if (ca_path && ! ca_path[0]) ca_path = NULL; - if (crl_file && ! crl_file[0]) crl_file = NULL; - if (crl_path && ! crl_path[0]) crl_path = NULL; + /* + If some optional parameters indicate empty strings, then + for compatibility with SSL libraries, replace them with NULL, + otherwise these libraries will try to open files with an empty + name, etc., and they will return an error code instead performing + the necessary operations: + */ + if (ca_file && !ca_file[0]) + { + ca_file = NULL; + } + if (ca_path && !ca_path[0]) + { + ca_path = NULL; + } + if (crl_file && !crl_file[0]) + { + crl_file = NULL; + } + if (crl_path && !crl_path[0]) + { + crl_path = NULL; + } if (!(ssl_fd= new_VioSSLFd(key_file, cert_file, ca_file, ca_path, cipher, FALSE, error, From 3b33593f80442214640eefb2ce75c34698c8043b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com> Date: Tue, 21 Dec 2021 11:07:25 +0200 Subject: [PATCH 18/21] MDEV-27332 SIGSEGV in fetch_data_into_cache() Since commit fb335b48b5e3263698b7d9a74ff3f20ef406df9f we may have a null pointer in purge_sys.query when fetch_data_into_cache() is invoked and innodb_force_recovery>4. This is because the call to purge_sys.create() would be skipped. fetch_data_into_cache(): Load the purge_sys pseudo transaction pointer to a local variable (null pointer if purge_sys is not initialized). --- .../suite/innodb/r/read_only_recover_committed.result | 7 +++++++ mysql-test/suite/innodb/t/read_only_recover_committed.test | 7 +++++++ storage/innobase/trx/trx0i_s.cc | 4 +++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/innodb/r/read_only_recover_committed.result b/mysql-test/suite/innodb/r/read_only_recover_committed.result index 278905a976b..0bfba662c15 100644 --- a/mysql-test/suite/innodb/r/read_only_recover_committed.result +++ b/mysql-test/suite/innodb/r/read_only_recover_committed.result @@ -63,6 +63,13 @@ SELECT * FROM t; a 3 20 +# +# MDEV-27332 SIGSEGV in fetch_data_into_cache +# +BEGIN; +SELECT trx_state FROM information_schema.innodb_trx; +trx_state +COMMIT; SELECT * FROM t; a 3 diff --git a/mysql-test/suite/innodb/t/read_only_recover_committed.test b/mysql-test/suite/innodb/t/read_only_recover_committed.test index 9ad09bb9b3a..439f57e9fad 100644 --- a/mysql-test/suite/innodb/t/read_only_recover_committed.test +++ b/mysql-test/suite/innodb/t/read_only_recover_committed.test @@ -78,6 +78,13 @@ SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; SELECT * FROM t; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT * FROM t; + +--echo # +--echo # MDEV-27332 SIGSEGV in fetch_data_into_cache +--echo # +BEGIN; +SELECT trx_state FROM information_schema.innodb_trx; +COMMIT; --let $restart_parameters= --source include/restart_mysqld.inc SELECT * FROM t; diff --git a/storage/innobase/trx/trx0i_s.cc b/storage/innobase/trx/trx0i_s.cc index afd644f3c70..951770e5bb7 100644 --- a/storage/innobase/trx/trx0i_s.cc +++ b/storage/innobase/trx/trx0i_s.cc @@ -1275,6 +1275,8 @@ static void fetch_data_into_cache_low(trx_i_s_cache_t *cache, const trx_t *trx) static void fetch_data_into_cache(trx_i_s_cache_t *cache) { + const trx_t *const purge_trx= purge_sys.query ? purge_sys.query->trx : NULL; + ut_ad(lock_mutex_own()); trx_i_s_cache_clear(cache); @@ -1284,7 +1286,7 @@ static void fetch_data_into_cache(trx_i_s_cache_t *cache) trx != NULL; trx= UT_LIST_GET_NEXT(trx_list, trx)) { - if (trx->state != TRX_STATE_NOT_STARTED && trx != purge_sys.query->trx) + if (trx != purge_trx && trx->state != TRX_STATE_NOT_STARTED) { mutex_enter(&trx->mutex); if (trx->state != TRX_STATE_NOT_STARTED) From a5ef74e7eb4bab09c9bda4fcd7fab12302526ea4 Mon Sep 17 00:00:00 2001 From: Alexander Barkov <bar@mariadb.com> Date: Wed, 22 Dec 2021 13:12:40 +0400 Subject: [PATCH 19/21] MDEV-27195 SIGSEGV in Table_scope_and_contents_source_st::vers_check_system_fields The old code erroneously used default_charset_info to compare field names. default_charset_info can point to any arbitrary collation, including ucs2*, utf16*, utf32*, including those that do not support strcasecmp(). my_charset_utf8mb4_unicode_ci, which is used in this scenario: CREATE TABLE t1 ENGINE=InnoDB WITH SYSTEM VERSIONING AS SELECT 0; does not support strcasecmp(). Fixing the code to use Lex_ident::streq(), which uses system_charset_info instead of default_charset_info. --- mysql-test/main/ctype_utf8mb4_unicode_ci_def.opt | 1 + .../main/ctype_utf8mb4_unicode_ci_def.result | 11 +++++++++++ mysql-test/main/ctype_utf8mb4_unicode_ci_def.test | 15 +++++++++++++++ sql/handler.cc | 3 +-- 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 mysql-test/main/ctype_utf8mb4_unicode_ci_def.opt create mode 100644 mysql-test/main/ctype_utf8mb4_unicode_ci_def.result create mode 100644 mysql-test/main/ctype_utf8mb4_unicode_ci_def.test diff --git a/mysql-test/main/ctype_utf8mb4_unicode_ci_def.opt b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.opt new file mode 100644 index 00000000000..e430a45c10e --- /dev/null +++ b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.opt @@ -0,0 +1 @@ +--character-set-server=utf8mb4,latin1 --collation-server=utf8mb4_unicode_ci diff --git a/mysql-test/main/ctype_utf8mb4_unicode_ci_def.result b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.result new file mode 100644 index 00000000000..2e15931248b --- /dev/null +++ b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.result @@ -0,0 +1,11 @@ +# +# Start of 10.3 tests +# +# +# MDEV-27195 SIGSEGV in Table_scope_and_contents_source_st::vers_check_system_fields +# +CREATE TABLE t1 ENGINE=MyISAM WITH SYSTEM VERSIONING AS SELECT 0; +DROP TABLE t1; +# +# End of 10.3 tests +# diff --git a/mysql-test/main/ctype_utf8mb4_unicode_ci_def.test b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.test new file mode 100644 index 00000000000..fb7fbe04e3b --- /dev/null +++ b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.test @@ -0,0 +1,15 @@ +--echo # +--echo # Start of 10.3 tests +--echo # + +--echo # +--echo # MDEV-27195 SIGSEGV in Table_scope_and_contents_source_st::vers_check_system_fields +--echo # + +CREATE TABLE t1 ENGINE=MyISAM WITH SYSTEM VERSIONING AS SELECT 0; +DROP TABLE t1; + + +--echo # +--echo # End of 10.3 tests +--echo # diff --git a/sql/handler.cc b/sql/handler.cc index 11a387fb4e3..f8702c27a39 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -7264,8 +7264,7 @@ bool Table_scope_and_contents_source_st::vers_check_system_fields( { List_iterator<Create_field> dup_it(alter_info->create_list); for (Create_field *dup= dup_it++; !is_dup && dup != f; dup= dup_it++) - is_dup= my_strcasecmp(default_charset_info, - dup->field_name.str, f->field_name.str) == 0; + is_dup= Lex_ident(dup->field_name).streq(f->field_name); } if (!(f->flags & VERS_UPDATE_UNVERSIONED_FLAG) && !is_dup) From 12087d67579e641cfc07ee6b8c46b75171e9708c Mon Sep 17 00:00:00 2001 From: Daniel Black <daniel@mariadb.org> Date: Wed, 22 Dec 2021 01:05:18 +0100 Subject: [PATCH 20/21] MDEV-23175: my_timer_milliseconds clock_gettime for multiple platfomrs Small postfix to MDEV-23175 to ensure faster option on FreeBSD and compatibility to Solaris that isn't high resolution. ftime is left as a backup in case an implementation doesn't contain any of these clocks. FreeBSD $ ./unittest/mysys/my_rdtsc-t 1..11 # ----- Routine --------------- # myt.cycles.routine : 5 # myt.nanoseconds.routine : 11 # myt.microseconds.routine : 13 # myt.milliseconds.routine : 11 # myt.ticks.routine : 17 # ----- Frequency ------------- # myt.cycles.frequency : 3610295566 # myt.nanoseconds.frequency : 1000000000 # myt.microseconds.frequency : 1000000 # myt.milliseconds.frequency : 899 # myt.ticks.frequency : 136 # ----- Resolution ------------ # myt.cycles.resolution : 1 # myt.nanoseconds.resolution : 1 # myt.microseconds.resolution : 1 # myt.milliseconds.resolution : 7 # myt.ticks.resolution : 1 # ----- Overhead -------------- # myt.cycles.overhead : 26 # myt.nanoseconds.overhead : 19140 # myt.microseconds.overhead : 19036 # myt.milliseconds.overhead : 578 # myt.ticks.overhead : 21544 ok 1 - my_timer_init() did not crash ok 2 - The cycle timer is strictly increasing ok 3 - The cycle timer is implemented ok 4 - The nanosecond timer is increasing ok 5 - The nanosecond timer is implemented ok 6 - The microsecond timer is increasing ok 7 - The microsecond timer is implemented ok 8 - The millisecond timer is increasing ok 9 - The millisecond timer is implemented ok 10 - The tick timer is increasing ok 11 - The tick timer is implemented --- mysys/my_rdtsc.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/mysys/my_rdtsc.c b/mysys/my_rdtsc.c index b020e224a59..605ac20350c 100644 --- a/mysys/my_rdtsc.c +++ b/mysys/my_rdtsc.c @@ -356,9 +356,29 @@ ulonglong my_timer_microseconds(void) milliseconds. */ +#if defined(HAVE_CLOCK_GETTIME) +#if defined(CLOCK_MONOTONIC_FAST) +/* FreeBSD */ +#define MY_CLOCK_ID CLOCK_MONOTONIC_FAST +#elif defined(CLOCK_MONOTONIC_COARSE) +/* Linux */ +#define MY_CLOCK_ID CLOCK_MONOTONIC_COARSE +#elif defined(CLOCK_MONOTONIC) +/* POSIX (includes OSX) */ +#define MY_CLOCK_ID CLOCK_MONOTONIC +#elif defined(CLOCK_REALTIME) +/* Solaris (which doesn't seem to have MONOTONIC) */ +#define MY_CLOCK_ID CLOCK_REALTIME +#endif +#endif + ulonglong my_timer_milliseconds(void) { -#if defined(HAVE_SYS_TIMEB_H) && defined(HAVE_FTIME) +#if defined(MY_CLOCK_ID) + struct timespec tp; + clock_gettime(MY_CLOCK_ID, &tp); + return (ulonglong)tp.tv_sec * 1000 + (ulonglong)tp.tv_nsec / 1000000; +#elif defined(HAVE_SYS_TIMEB_H) && defined(HAVE_FTIME) /* ftime() is obsolete but maybe the platform is old */ struct timeb ft; ftime(&ft); @@ -631,7 +651,9 @@ void my_timer_init(MY_TIMER_INFO *mti) /* milliseconds */ mti->milliseconds.frequency= 1000; /* initial assumption */ -#if defined(HAVE_SYS_TIMEB_H) && defined(HAVE_FTIME) +#ifdef MY_CLOCK_ID + mti->milliseconds.routine= MY_TIMER_ROUTINE_CLOCK_GETTIME; +#elif defined(HAVE_SYS_TIMEB_H) && defined(HAVE_FTIME) mti->milliseconds.routine= MY_TIMER_ROUTINE_FTIME; #elif defined(_WIN32) mti->milliseconds.routine= MY_TIMER_ROUTINE_GETSYSTEMTIMEASFILETIME; From b5cbe50604046647b52c1ed6b5af596483a120be Mon Sep 17 00:00:00 2001 From: Julius Goryavsky <julius.goryavsky@mariadb.com> Date: Thu, 23 Dec 2021 14:19:45 +0100 Subject: [PATCH 21/21] MDEV-24097: galera[_3nodes] suite tests in MTR sporadically fails This is the first part of the fixes for MDEV-24097. This commit contains the fixes for instability when testing Galera and when restarting nodes quickly: 1) Protection against a "stuck" old SST process during the execution of the new SST (after restarting the node) is now implemented for mariabackup / xtrabackup, which should help to avoid almost all conflicts due to the use of the same ports - both during testing with mtr, so and when restarting nodes quickly in a production environment. 2) Added more protection to scripts against unexpected return of the rc != 0 (in the commands for deleting temporary files, etc). 3) Added protection against unexpected crashes during binlog transfer (in SST scripts for rsync). 4) Spaces and some special characters in binlog filenames shouldn't be a problem now (at the script level). 5) Daemon process termination tracking has been made more robust against crashes due to unexpected termination of the previous SST process while new scripts are running. 6) Reading ssl encryption parameters has been moved from specific SST scripts to a common wsrep_sst_common.sh script, which allows unified error handling, unified diagnostics and simplifies script revisions in the future. 7) Improved diagnostics of errors related to the use of openssl. 8) Corrections have been made for xtrabackup-v2 (both in tests and in the script code) that restore the work of xtrabackup with updated versions of innodb. 9) Fixed some tests for galera_3nodes, although the complete solution for the problem of starting three nodes at the same time on fast machines will be done in a separate commit. No additional tests are required as this commit fixes problems with existing tests. --- mysql-test/suite/galera/disabled.def | 2 - .../galera_sst_xtrabackup-v2_data_dir.result | 26 +++ .../t/galera_autoinc_sst_xtrabackup.cnf | 1 + .../galera/t/galera_ist_xtrabackup-v2.cnf | 1 + .../t/galera_sst_xtrabackup-v2-options.cnf | 1 + .../galera/t/galera_sst_xtrabackup-v2.cnf | 1 + .../t/galera_sst_xtrabackup-v2_data_dir.cnf | 1 + ...era_sst_xtrabackup-v2_encrypt_with_key.cnf | 1 + .../r/galera_ist_gcache_rollover.result | 4 +- .../t/galera_ist_gcache_rollover.test | 10 +- scripts/wsrep_sst_common.sh | 142 ++++++++++----- scripts/wsrep_sst_mariabackup.sh | 109 +++++------- scripts/wsrep_sst_rsync.sh | 161 ++++++++---------- scripts/wsrep_sst_xtrabackup-v2.sh | 117 +++++-------- 14 files changed, 292 insertions(+), 285 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 955a2c82ebc..84babda2fa0 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -28,5 +28,3 @@ query_cache: MDEV-15805 Test failure on galera.query_cache versioning_trx_id: MDEV-18590: galera.versioning_trx_id: Test failure: mysqltest: Result content mismatch galera_wsrep_provider_unset_set: wsrep_provider is read-only for security reasons pxc-421: wsrep_provider is read-only for security reasons -galera_sst_xtrabackup-v2: Test fails due to innodb issues -galera_sst_xtrabackup-v2_data_dir: Test fails due to innodb issues diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result index cec0f21ee22..ff85a7d6c0f 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result @@ -1,4 +1,7 @@ +connection node_1; +connection node_2; Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -8,6 +11,7 @@ INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); COMMIT; +connection node_2; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node2_committed_before'); @@ -17,6 +21,7 @@ INSERT INTO t1 VALUES ('node2_committed_before'); INSERT INTO t1 VALUES ('node2_committed_before'); COMMIT; Shutting down server ... +connection node_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_committed_during'); @@ -31,6 +36,7 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -38,6 +44,7 @@ INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; Starting server ... SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -47,6 +54,7 @@ INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); COMMIT; +connection node_1; INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); @@ -61,6 +69,7 @@ INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); COMMIT; +connection node_1a_galera_st_shutdown_slave; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -75,6 +84,7 @@ COUNT(*) = 0 1 COMMIT; SET AUTOCOMMIT=ON; +connection node_1; SELECT COUNT(*) = 35 FROM t1; COUNT(*) = 35 1 @@ -86,6 +96,7 @@ COMMIT; SET AUTOCOMMIT=ON; Performing State Transfer on a server that starts from a clean var directory This is accomplished by shutting down node #2 and removing its var directory before restarting it +connection node_1; CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -95,6 +106,7 @@ INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); COMMIT; +connection node_2; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node2_committed_before'); @@ -104,6 +116,7 @@ INSERT INTO t1 VALUES ('node2_committed_before'); INSERT INTO t1 VALUES ('node2_committed_before'); COMMIT; Shutting down server ... +connection node_1; Cleaning var directory ... SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -119,6 +132,7 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -126,6 +140,7 @@ INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; Starting server ... SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -135,6 +150,7 @@ INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); COMMIT; +connection node_1; INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); @@ -149,6 +165,7 @@ INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); COMMIT; +connection node_1a_galera_st_clean_slave; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -163,6 +180,7 @@ COUNT(*) = 0 1 COMMIT; SET AUTOCOMMIT=ON; +connection node_1; SELECT COUNT(*) = 35 FROM t1; COUNT(*) = 35 1 @@ -173,6 +191,7 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; Performing State Transfer on a server that has been killed and restarted +connection node_1; CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -182,6 +201,7 @@ INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); COMMIT; +connection node_2; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node2_committed_before'); @@ -191,6 +211,7 @@ INSERT INTO t1 VALUES ('node2_committed_before'); INSERT INTO t1 VALUES ('node2_committed_before'); COMMIT; Killing server ... +connection node_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_committed_during'); @@ -205,6 +226,7 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -212,6 +234,7 @@ INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; Performing --wsrep-recover ... Starting server ... Using --wsrep-start-position when starting mysqld ... @@ -223,6 +246,7 @@ INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); COMMIT; +connection node_1; INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); @@ -237,6 +261,7 @@ INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); COMMIT; +connection node_1a_galera_st_kill_slave; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -251,6 +276,7 @@ COUNT(*) = 0 1 COMMIT; SET AUTOCOMMIT=ON; +connection node_1; SELECT COUNT(*) = 35 FROM t1; COUNT(*) = 35 1 diff --git a/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf b/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf index 2e25ecae317..f6ee3c56e5e 100644 --- a/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf +++ b/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf @@ -4,6 +4,7 @@ wsrep_sst_method=xtrabackup-v2 wsrep_sst_auth="root:" innodb_safe_truncate=OFF +# innodb_undo_logs=1 [mysqld.1] wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' diff --git a/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf b/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf index 5b35701aeab..7242606fe05 100644 --- a/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf +++ b/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf @@ -4,6 +4,7 @@ wsrep_sst_method=xtrabackup-v2 wsrep_sst_auth=root: innodb_safe_truncate=OFF +innodb_undo_logs=1 [mysqld.1] wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true' diff --git a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf index af3a7aecec4..ec836592965 100644 --- a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf +++ b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf @@ -5,6 +5,7 @@ wsrep_sst_method=xtrabackup-v2 wsrep_sst_auth="root:" wsrep_debug=ON innodb_safe_truncate=OFF +innodb_undo_logs=1 [xtrabackup] backup-locks diff --git a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.cnf b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.cnf index e347e259897..ad99bd909b8 100644 --- a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.cnf +++ b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.cnf @@ -5,6 +5,7 @@ wsrep_sst_method=xtrabackup-v2 wsrep_sst_auth="root:" wsrep_debug=ON innodb_safe_truncate=OFF +innodb_undo_logs=1 [mysqld.1] wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' diff --git a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_data_dir.cnf b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_data_dir.cnf index 43180173425..8665c517629 100644 --- a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_data_dir.cnf +++ b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_data_dir.cnf @@ -5,6 +5,7 @@ wsrep_sst_method=xtrabackup-v2 wsrep_sst_auth="root:" wsrep_debug=ON innodb_safe_truncate=OFF +innodb_undo_logs=1 [mysqld.1] wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true' diff --git a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_encrypt_with_key.cnf b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_encrypt_with_key.cnf index 7a98fd2aa58..984d040c6ab 100644 --- a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_encrypt_with_key.cnf +++ b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_encrypt_with_key.cnf @@ -5,6 +5,7 @@ wsrep_sst_method=xtrabackup-v2 wsrep_sst_auth="root:" wsrep_debug=ON innodb_safe_truncate=OFF +innodb_undo_logs=1 [SST] tkey=@ENV.MYSQL_TEST_DIR/std_data/cakey.pem diff --git a/mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result b/mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result index aa3e349eda7..996363a1ea8 100644 --- a/mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result +++ b/mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result @@ -6,10 +6,10 @@ CREATE TABLE t1 (f1 INTEGER PRIMARY KEY); INSERT INTO t1 VALUES (01), (02), (03), (04), (05); connection node_2; Unloading wsrep provider ... -SET GLOBAL wsrep_provider = 'none'; +SET GLOBAL wsrep_cluster_address = ''; connection node_3; Unloading wsrep provider ... -SET GLOBAL wsrep_provider = 'none'; +SET GLOBAL wsrep_cluster_address = ''; connection node_1; INSERT INTO t1 VALUES (11), (12), (13), (14), (15); INSERT INTO t1 VALUES (21), (22), (23), (24), (25); diff --git a/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test b/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test index 0032f8d6ee1..2719316fb84 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test +++ b/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test @@ -29,9 +29,11 @@ INSERT INTO t1 VALUES (01), (02), (03), (04), (05); # Disconnect nodes #2 and #3 --connection node_2 +--let $wsrep_cluster_address_orig2 = `select @@wsrep_cluster_address` --source suite/galera/include/galera_stop_replication.inc --connection node_3 +--let $wsrep_cluster_address_orig3 = `select @@wsrep_cluster_address` --source suite/galera/include/galera_stop_replication.inc --connection node_1 @@ -51,8 +53,8 @@ INSERT INTO t1 VALUES (21), (22), (23), (24), (25); # ... and restart providers to force IST --connection node_2 --disable_query_log ---eval SET GLOBAL wsrep_provider = '$wsrep_provider_orig'; ---eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig'; +SET GLOBAL wsrep_cluster_address=''; +--eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig2'; --enable_query_log --connection node_1 @@ -60,8 +62,8 @@ INSERT INTO t1 VALUES (31), (32), (33), (34), (35); --connection node_3 --disable_query_log ---eval SET GLOBAL wsrep_provider = '$wsrep_provider_orig'; ---eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig'; +SET GLOBAL wsrep_cluster_address=''; +--eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig3'; --enable_query_log --connection node_1 diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 040cd31caa9..deebe7cf820 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -17,7 +17,7 @@ # This is a common command line parser to be sourced by other SST scripts -set -u +set -ue # Setting the path for some utilities on CentOS export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" @@ -879,9 +879,9 @@ fi wsrep_cleanup_progress_file() { - [ -n "$SST_PROGRESS_FILE" -a \ - -f "$SST_PROGRESS_FILE" ] && \ - rm -f "$SST_PROGRESS_FILE" 2>/dev/null || : + if [ -n "$SST_PROGRESS_FILE" -a -f "$SST_PROGRESS_FILE" ]; then + rm -f "$SST_PROGRESS_FILE" 2>/dev/null || : + fi } wsrep_check_program() @@ -897,13 +897,10 @@ wsrep_check_program() wsrep_check_programs() { local ret=0 - - while [ $# -gt 0 ] - do + while [ $# -gt 0 ]; do wsrep_check_program "$1" || ret=$? shift done - return $ret } @@ -1028,11 +1025,11 @@ check_sockets_utils() # check_port() { - local pid="$1" + local pid="${1:-0}" local port="$2" local utils="$3" - [ -z "$pid" ] || [ $pid -eq 0 ] && pid='[0-9]+' + [ $pid -le 0 ] && pid='[0-9]+' local rc=1 @@ -1070,14 +1067,20 @@ check_for_dhparams() if [ ! -r "$ssl_dhparams" ]; then get_openssl if [ -n "$OPENSSL_BINARY" ]; then - wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams" - if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1 - then + wsrep_log_info \ + "Could not find dhparams file, creating $ssl_dhparams" + local bug=0 + local errmsg + errmsg=$("$OPENSSL_BINARY" \ + dhparam -out "$ssl_dhparams" 2048 2>&1) || bug=1 + if [ $bug -ne 0 ]; then + wsrep_log_info "run: \"$OPENSSL_BINARY\" dhparam -out \"$ssl_dhparams\" 2048" + wsrep_log_info "output: $errmsg" wsrep_log_error "******** ERROR *****************************************" wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *" wsrep_log_error "********************************************************" ssl_dhparams="" - fi + fi else # Rollback: if openssl is not installed, then use # the default parameters: @@ -1099,6 +1102,16 @@ verify_ca_matches_cert() local ca="$2" local cap="$3" + local readable=1; [ ! -r "$cert" ] && readable=0 + [ -n "$ca" -a ! -r "$ca" ] && readable=0 + [ -n "$cap" -a ! -r "$cap" ] && readable=0 + + if [ $readable -eq 0 ]; then + wsrep_log_error \ + "Both PEM file and CA file (or path) must be readable" + exit 22 + fi + # If the openssl utility is not installed, then # we will not do this certificate check: get_openssl @@ -1107,22 +1120,12 @@ verify_ca_matches_cert() return fi - local readable=1; [ ! -r "$cert" ] && readable=0 - [ -n "$ca" ] && [ ! -r "$ca" ] && readable=0 - [ -n "$cap" ] && [ ! -r "$cap" ] && readable=0 - - if [ $readable -eq 0 ]; then - wsrep_log_error \ - "Both PEM file and CA file (or path) must be readable" - exit 22 - fi - local not_match=0 local errmsg errmsg=$("$OPENSSL_BINARY" verify -verbose \ - ${ca:+ -CAfile} ${ca:+ "$ca"} \ - ${cap:+ -CApath} ${cap:+ "$cap"} \ - "$cert" 2>&1) || not_match=1 + ${ca:+ -CAfile} ${ca:+ "$ca"} \ + ${cap:+ -CApath} ${cap:+ "$cap"} \ + "$cert" 2>&1) || not_match=1 if [ $not_match -eq 1 ]; then wsrep_log_info "run: \"$OPENSSL_BINARY\" verify -verbose${ca:+ -CAfile \"$ca\"}${cap:+ -CApath \"$cap\"} \"$cert\"" @@ -1158,6 +1161,7 @@ verify_cert_matches_key() # If the diff utility is not installed, then # we will not do this certificate check: if [ -z "$(commandex diff)" ]; then + wsrep_log_info "diff utility not found" return fi @@ -1165,6 +1169,7 @@ verify_cert_matches_key() # we will not do this certificate check: get_openssl if [ -z "$OPENSSL_BINARY" ]; then + wsrep_log_info "openssl utility not found" return fi @@ -1253,18 +1258,18 @@ check_pid() { local pid_file="$1" if [ -r "$pid_file" ]; then - local pid=$(cat "$pid_file" 2>/dev/null) + local pid=$(cat "$pid_file" 2>/dev/null || :) if [ -n "$pid" ]; then - if [ $pid -ne 0 ]; then - if ps -p "$pid" >/dev/null 2>&1; then + if [ $pid -gt 0 ]; then + if ps -p $pid >/dev/null 2>&1; then CHECK_PID=$pid return 0 fi fi fi local remove=${2:-0} - if [ $remove -eq 1 ]; then - rm -f "$pid_file" + if [ $remove -ne 0 ]; then + rm -f "$pid_file" || : fi fi CHECK_PID=0 @@ -1289,25 +1294,25 @@ cleanup_pid() local pid_file="${2:-}" local config="${3:-}" - if [ $pid -ne 0 ]; then + if [ $pid -gt 0 ]; then if ps -p $pid >/dev/null 2>&1; then if kill $pid >/dev/null 2>&1; then sleep 0.5 local round=0 local force=0 while ps -p $pid >/dev/null 2>&1; do - sleep 1 - round=$(( round+1 )) - if [ $round -eq 16 ]; then - if [ $force -eq 0 ]; then - round=8 - force=1 - kill -9 $pid >/dev/null 2>&1 - sleep 0.5 - else - return 1 - fi - fi + sleep 1 + round=$(( round+1 )) + if [ $round -eq 16 ]; then + if [ $force -eq 0 ]; then + round=8 + force=1 + kill -9 $pid >/dev/null 2>&1 || : + sleep 0.5 + else + return 1 + fi + fi done elif ps -p $pid >/dev/null 2>&1; then wsrep_log_warning "Unable to kill PID=$pid ($pid_file)" @@ -1316,8 +1321,8 @@ cleanup_pid() fi fi - [ -n "$pid_file" ] && [ -f "$pid_file" ] && rm -f "$pid_file" - [ -n "$config" ] && [ -f "$config" ] && rm -f "$config" + [ -n "$pid_file" -a -f "$pid_file" ] && rm -f "$pid_file" || : + [ -n "$config" -a -f "$config" ] && rm -f "$config" || : return 0 } @@ -1339,3 +1344,46 @@ get_proc() fi fi } + +check_server_ssl_config() +{ + # backward-compatible behavior: + tcert=$(parse_cnf 'sst' 'tca') + tcap=$(parse_cnf 'sst' 'tcapath') + tpem=$(parse_cnf 'sst' 'tcert') + tkey=$(parse_cnf 'sst' 'tkey') + # reading new ssl configuration options: + local tcert2=$(parse_cnf "$encgroups" 'ssl-ca') + local tcap2=$(parse_cnf "$encgroups" 'ssl-capath') + local tpem2=$(parse_cnf "$encgroups" 'ssl-cert') + local tkey2=$(parse_cnf "$encgroups" 'ssl-key') + # if there are no old options, then we take new ones: + if [ -z "$tcert" -a -z "$tcap" -a -z "$tpem" -a -z "$tkey" ]; then + tcert="$tcert2" + tcap="$tcap2" + tpem="$tpem2" + tkey="$tkey2" + # checking for presence of the new-style SSL configuration: + elif [ -n "$tcert2" -o -n "$tcap2" -o -n "$tpem2" -o -n "$tkey2" ]; then + if [ "$tcert" != "$tcert2" -o \ + "$tcap" != "$tcap2" -o \ + "$tpem" != "$tpem2" -o \ + "$tkey" != "$tkey2" ] + then + wsrep_log_info \ + "new ssl configuration options (ssl-ca[path], ssl-cert" \ + "and ssl-key) are ignored by SST due to presence" \ + "of the tca[path], tcert and/or tkey in the [sst] section" + fi + fi + if [ -n "$tcert" ]; then + tcert=$(trim_string "$tcert") + if [ "${tcert%/}" != "$tcert" -o -d "$tcert" ]; then + tcap="$tcert" + tcert="" + fi + fi + if [ -n "$tcap" ]; then + tcap=$(trim_string "$tcap") + fi +} diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 4bca785fcad..aa9442b0601 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -30,7 +30,6 @@ eformat="" ekey="" ekeyfile="" encrypt=0 -ecode=0 ssyslog="" ssystag="" BACKUP_PID="" @@ -465,49 +464,6 @@ adjust_progress() encgroups='--mysqld|sst|xtrabackup' -check_server_ssl_config() -{ - # backward-compatible behavior: - tcert=$(parse_cnf 'sst' 'tca') - tcap=$(parse_cnf 'sst' 'tcapath') - tpem=$(parse_cnf 'sst' 'tcert') - tkey=$(parse_cnf 'sst' 'tkey') - # reading new ssl configuration options: - local tcert2=$(parse_cnf "$encgroups" 'ssl-ca') - local tcap2=$(parse_cnf "$encgroups" 'ssl-capath') - local tpem2=$(parse_cnf "$encgroups" 'ssl-cert') - local tkey2=$(parse_cnf "$encgroups" 'ssl-key') - # if there are no old options, then we take new ones: - if [ -z "$tcert" -a -z "$tcap" -a -z "$tpem" -a -z "$tkey" ]; then - tcert="$tcert2" - tcap="$tcap2" - tpem="$tpem2" - tkey="$tkey2" - # checking for presence of the new-style SSL configuration: - elif [ -n "$tcert2" -o -n "$tcap2" -o -n "$tpem2" -o -n "$tkey2" ]; then - if [ "$tcert" != "$tcert2" -o \ - "$tcap" != "$tcap2" -o \ - "$tpem" != "$tpem2" -o \ - "$tkey" != "$tkey2" ] - then - wsrep_log_info \ - "new ssl configuration options (ssl-ca[path], ssl-cert" \ - "and ssl-key) are ignored by SST due to presence" \ - "of the tca[path], tcert and/or tkey in the [sst] section" - fi - fi - if [ -n "$tcert" ]; then - tcert=$(trim_string "$tcert") - if [ "${tcert%/}" != "$tcert" ] || [ -d "$tcert" ]; then - tcap="$tcert" - tcert="" - fi - fi - if [ -n "$tcap" ]; then - tcap=$(trim_string "$tcap") - fi -} - read_cnf() { sfmt=$(parse_cnf sst streamfmt 'mbstream') @@ -647,7 +603,7 @@ cleanup_at_exit() cleanup_pid $CHECK_PID "$BACKUP_PID" fi fi - [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" + [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" || : fi if [ -n "$progress" -a -p "$progress" ]; then @@ -658,27 +614,31 @@ cleanup_at_exit() wsrep_log_info "Cleaning up temporary directories" if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if [ -n "$STATDIR" ]; then - [ -d "$STATDIR" ] && rm -rf "$STATDIR" - fi + [ -n "$STATDIR" -a -d "$STATDIR" ] && rm -rf "$STATDIR" || : else [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || : [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || : fi # Final cleanup - pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. - if [ $$ -eq $pgid ]; then - # This means a signal was delivered to the process. - # So, more cleanup. - if [ $estatus -ge 128 ]; then - kill -KILL -- -$$ || : + if [ -n "$pgid" ]; then + if [ $$ -eq $pgid ]; then + # This means a signal was delivered to the process. + # So, more cleanup. + if [ $estatus -ge 128 ]; then + kill -KILL -- -$$ || : + fi fi fi + if [ -n "${SST_PID:-}" ]; then + [ -f "$SST_PID" ] && rm -f "$SST_PID" || : + fi + exit $estatus } @@ -967,7 +927,7 @@ setup_commands() fi INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY" INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" - INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts $tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP" + INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP" } get_stream @@ -995,7 +955,7 @@ then fi wsrep_log_info "Using '$xtmpdir' as mariabackup temporary directory" - tmpopts="--tmpdir='$xtmpdir'" + tmpopts=" --tmpdir='$xtmpdir'" itmpdir="$(mktemp -d)" wsrep_log_info "Using '$itmpdir' as mariabackup working directory" @@ -1161,10 +1121,23 @@ then impts="--parallel=$backup_threads${impts:+ }$impts" fi - stagemsg='Joiner-Recv' + SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" - sencrypted=1 - nthreads=1 + # give some time for previous SST to complete: + check_round=0 + while check_pid "$SST_PID" 0; do + wsrep_log_info "previous SST is not completed, waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "previous SST script still running." + exit 114 # EALREADY + fi + sleep 1 + done + + echo $$ > "$SST_PID" + + stagemsg='Joiner-Recv' MODULE="xtrabackup_sst" @@ -1208,7 +1181,7 @@ then fi get_keys - if [ $encrypt -eq 1 -a $sencrypted -eq 1 ]; then + if [ $encrypt -eq 1 ]; then strmcmd="$ecmd | $strmcmd" fi @@ -1263,12 +1236,14 @@ then if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ || : - cd "$OLD_PWD" + if [ -d "$binlog_dir" ]; then + cd "$binlog_dir" + wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" + rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : + [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ + rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ + cd "$OLD_PWD" + fi fi TDATA="$DATA" @@ -1285,7 +1260,7 @@ then fi # Compact backups are not supported by mariabackup - if grep -q -F 'compact = 1' "$DATA/xtrabackup_checkpoints"; then + if grep -qw -F 'compact = 1' "$DATA/xtrabackup_checkpoints"; then wsrep_log_info "Index compaction detected" wsrel_log_error "Compact backups are not supported by mariabackup" exit 2 diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index b0cc8cb3066..28dfed18218 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston # MA 02110-1335 USA. -# This is a reference script for rsync-based state snapshot tansfer +# This is a reference script for rsync-based state snapshot transfer RSYNC_REAL_PID=0 # rsync process id STUNNEL_REAL_PID=0 # stunnel process id @@ -41,7 +41,7 @@ cleanup_joiner() if cleanup_pid $STUNNEL_REAL_PID "$STUNNEL_PID" "$STUNNEL_CONF"; then if [ $RSYNC_REAL_PID -eq 0 ]; then if [ -r "$RSYNC_PID" ]; then - RSYNC_REAL_PID=$(cat "$RSYNC_PID" 2>/dev/null) + RSYNC_REAL_PID=$(cat "$RSYNC_PID" 2>/dev/null || :) if [ -z "$RSYNC_REAL_PID" ]; then RSYNC_REAL_PID=0 fi @@ -79,7 +79,7 @@ check_pid_and_port() local utils='rsync|stunnel' - if ! check_port "$pid" "$port" "$utils"; then + if ! check_port $pid "$port" "$utils"; then local port_info local busy=0 @@ -90,7 +90,7 @@ check_pid_and_port() grep -q -E "[[:space:]](\\*|\\[?::\\]?):$port[[:space:]]" && busy=1 else local filter='([^[:space:]]+[[:space:]]+){4}[^[:space:]]+' - if [ $sockstat_available -eq 1 ]; then + if [ $sockstat_available -ne 0 ]; then local opts='-p' if [ "$OS" = 'FreeBSD' ]; then # sockstat on FreeBSD requires the "-s" option @@ -110,18 +110,20 @@ check_pid_and_port() fi if [ $busy -eq 0 ]; then - if echo "$port_info" | grep -qw -F "[$addr]:$port" || \ - echo "$port_info" | grep -qw -F -- "$addr:$port" + if ! echo "$port_info" | grep -qw -F "[$addr]:$port" && \ + ! echo "$port_info" | grep -qw -F -- "$addr:$port" then - busy=1 + if ! ps -p $pid >/dev/null 2>&1; then + wsrep_log_error \ + "rsync or stunnel daemon (PID: $pid)" \ + "terminated unexpectedly." + exit 16 # EBUSY + fi + return 1 fi fi - if [ $busy -eq 0 ]; then - return 1 - fi - - if ! check_port "$pid" "$port" "$utils"; then + if ! check_port $pid "$port" "$utils"; then wsrep_log_error "rsync or stunnel daemon port '$port'" \ "has been taken by another program" exit 16 # EBUSY @@ -197,60 +199,16 @@ INNODB_UNDO_DIR=$(pwd -P) cd "$OLD_PWD" -# Old filter - include everything except selected -# FILTER=(--exclude '*.err' --exclude '*.pid' --exclude '*.sock' \ -# --exclude '*.conf' --exclude core --exclude 'galera.*' \ -# --exclude grastate.txt --exclude '*.pem' \ -# --exclude '*.[0-9][0-9][0-9][0-9][0-9][0-9]' --exclude '*.index') +encgroups='--mysqld|sst' -# New filter - exclude everything except dirs (schemas) and innodb files -FILTER="-f '- /lost+found' - -f '- /.zfs' - -f '- /.fseventsd' - -f '- /.Trashes' - -f '- /.pid' - -f '- /.conf' - -f '+ /wsrep_sst_binlog.tar' - -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' - -f '- $INNODB_DATA_HOME_DIR/ibdata*' - -f '+ $INNODB_UNDO_DIR/undo*' - -f '+ /*/' - -f '- /*'" +check_server_ssl_config -# old-style SSL config -SSTKEY=$(parse_cnf 'sst' 'tkey') -SSTCERT=$(parse_cnf 'sst' 'tcert') -SSTCA=$(parse_cnf 'sst' 'tca') -SSTCAP=$(parse_cnf 'sst' 'tcapath') +SSTKEY="$tkey" +SSTCERT="$tpem" +SSTCA="$tcert" +SSTCAP="$tcap" -SST_SECTIONS="--mysqld|sst" - -check_server_ssl_config() -{ - SSTKEY=$(parse_cnf "$SST_SECTIONS" 'ssl-key') - SSTCERT=$(parse_cnf "$SST_SECTIONS" 'ssl-cert') - SSTCA=$(parse_cnf "$SST_SECTIONS" 'ssl-ca') - SSTCAP=$(parse_cnf "$SST_SECTIONS" 'ssl-capath') -} - -SSLMODE=$(parse_cnf "$SST_SECTIONS" 'ssl-mode' | tr [:lower:] [:upper:]) - -# no old-style SSL config in [sst], check for new one: -if [ -z "$SSTKEY" -a -z "$SSTCERT" -a -z "$SSTCA" -a -z "$SSTCAP" ]; then - check_server_ssl_config -fi - -if [ -n "$SSTCA" ]; then - SSTCA=$(trim_string "$SSTCA") - if [ "${SSTCA%/}" != "$SSTCA" ] || [ -d "$SSTCA" ]; then - SSTCAP="$SSTCA" - SSTCA="" - fi -fi - -if [ -n "$SSTCAP" ]; then - SSTCAP=$(trim_string "$SSTCAP") -fi +SSLMODE=$(parse_cnf "$encgroups" 'ssl-mode' | tr [:lower:] [:upper:]) if [ -z "$SSLMODE" ]; then # Implicit verification if CA is set and the SSL mode @@ -266,7 +224,7 @@ if [ -z "$SSLMODE" ]; then fi fi -if [ -n "$SSTCERT" -a -n "$SSTKEY" ]; then +if [ -n "$SSTKEY" -a -n "$SSTCERT" ]; then verify_cert_matches_key "$SSTCERT" "$SSTKEY" fi @@ -287,8 +245,7 @@ fi VERIFY_OPT="" CHECK_OPT="" CHECK_OPT_LOCAL="" -if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ] -then +if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then case "$SSLMODE" in 'VERIFY_IDENTITY') VERIFY_OPT='verifyPeer = yes' @@ -364,8 +321,9 @@ EOF [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" fi - if [ $WSREP_SST_OPT_BYPASS -eq 0 ] - then + RC=0 + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ]; then FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed" ERROR="$WSREP_SST_OPT_DATA/sst_error" @@ -380,11 +338,11 @@ EOF # (b) Cluster state ID & wsrep_gtid_domain_id to be written to the file, OR # (c) ERROR file, in case flush tables operation failed. - while [ ! -r "$FLUSHED" ] && ! grep -q -F ':' "$FLUSHED" >/dev/null 2>&1 + while [ ! -r "$FLUSHED" ] && \ + ! grep -q -F ':' '--' "$FLUSHED" >/dev/null 2>&1 do # Check whether ERROR file exists. - if [ -f "$ERROR" ] - then + if [ -f "$ERROR" ]; then # Flush tables operation failed. rm -f "$ERROR" exit 255 @@ -397,7 +355,7 @@ EOF sync - if [ -n "$WSREP_SST_OPT_BINLOG" ] + if [ -n "$WSREP_SST_OPT_BINLOG" -a -d "${BINLOG_DIRNAME:-}" ] then # Prepare binlog files cd "$BINLOG_DIRNAME" @@ -405,16 +363,14 @@ EOF binlog_files_full=$(tail -n $BINLOG_N_FILES \ "$WSREP_SST_OPT_BINLOG_INDEX") binlog_files="" - for ii in $binlog_files_full - do - binlog_file=$(basename "$ii") - binlog_files="$binlog_files $binlog_file" + for file in $binlog_files_full; do + binlog_file=$(basename "$file") + binlog_files="$binlog_files${binlog_files:+ }'$binlog_file'" done - if [ -n "$binlog_files" ] - then + if [ -n "$binlog_files" ]; then wsrep_log_info "Preparing binlog files for transfer:" - tar -cvf "$BINLOG_TAR_FILE" $binlog_files >&2 + eval tar -cvf "'$BINLOG_TAR_FILE'" $binlog_files >&2 fi cd "$OLD_PWD" @@ -427,9 +383,28 @@ EOF WHOLE_FILE_OPT="--whole-file" fi +# Old filter - include everything except selected +# FILTER=(--exclude '*.err' --exclude '*.pid' --exclude '*.sock' \ +# --exclude '*.conf' --exclude core --exclude 'galera.*' \ +# --exclude grastate.txt --exclude '*.pem' \ +# --exclude '*.[0-9][0-9][0-9][0-9][0-9][0-9]' --exclude '*.index') + +# New filter - exclude everything except dirs (schemas) and innodb files +FILTER="-f '- /lost+found' + -f '- /.zfs' + -f '- /.fseventsd' + -f '- /.Trashes' + -f '- /.pid' + -f '- /.conf' + -f '+ /wsrep_sst_binlog.tar' + -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' + -f '- $INNODB_DATA_HOME_DIR/ibdata*' + -f '+ $INNODB_UNDO_DIR/undo*' + -f '+ /*/' + -f '- /*'" + # first, the normal directories, so that we can detect # incompatible protocol: - RC=0 eval rsync ${STUNNEL:+"--rsh='$STUNNEL'"} \ --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ @@ -484,7 +459,7 @@ EOF cd "$WSREP_SST_OPT_DATA" - backup_threads=$(parse_cnf "--mysqld|sst" 'backup-threads') + backup_threads=$(parse_cnf '--mysqld|sst' 'backup-threads') if [ -z "$backup_threads" ]; then get_proc backup_threads=$nproc @@ -527,7 +502,12 @@ EOF rsync ${STUNNEL:+--rsh="$STUNNEL"} \ --archive --quiet --checksum "$MAGIC_FILE" \ - "rsync://$WSREP_SST_OPT_ADDR" + "rsync://$WSREP_SST_OPT_ADDR" >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync $MAGIC_FILE returned code $RC:" + exit 255 # unknown error + fi echo "done $STATE" @@ -540,12 +520,11 @@ elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then check_sockets_utils - SST_PID="$WSREP_SST_OPT_DATA/wsrep_rsync_sst.pid" + SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" # give some time for previous SST to complete: check_round=0 - while check_pid "$SST_PID" 0 - do + while check_pid "$SST_PID" 0 'wsrep_sst_'; do wsrep_log_info "previous SST is not completed, waiting for it to exit" check_round=$(( check_round + 1 )) if [ $check_round -eq 10 ]; then @@ -555,10 +534,11 @@ then sleep 1 done + echo $$ > "$SST_PID" + # give some time for stunnel from the previous SST to complete: check_round=0 - while check_pid "$STUNNEL_PID" 1 - do + while check_pid "$STUNNEL_PID" 1; do wsrep_log_info "Lingering stunnel daemon found at startup," \ "waiting for it to exit" check_round=$(( check_round + 1 )) @@ -575,8 +555,7 @@ then # give some time for rsync from the previous SST to complete: check_round=0 - while check_pid "$RSYNC_PID" 1 - do + while check_pid "$RSYNC_PID" 1; do wsrep_log_info "Lingering rsync daemon found at startup," \ "waiting for it to exit" check_round=$(( check_round + 1 )) @@ -590,7 +569,7 @@ then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - [ -z "$STUNNEL" ] && [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" + [ -z "$STUNNEL" -a -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" ADDR="$WSREP_SST_OPT_ADDR" RSYNC_PORT="$WSREP_SST_OPT_PORT" @@ -639,8 +618,6 @@ EOF RSYNC_ADDR="*" fi - echo $$ > "$SST_PID" - if [ -z "$STUNNEL" ]; then rsync --daemon --no-detach --port "$RSYNC_PORT" \ --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS & diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index d5c978c4147..569d5084c98 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -31,7 +31,6 @@ eformat="" ekey="" ekeyfile="" encrypt=0 -ecode=0 ssyslog="" ssystag="" BACKUP_PID="" @@ -160,7 +159,7 @@ get_keys() return fi - wsrep_log_info "Key based encryption enabled in my.cnf - supported only from Xtrabackup 2.1.4" + wsrep_log_info "Key based encryption enabled in my.cnf" if [ -z "$ealgo" ]; then wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" @@ -470,49 +469,6 @@ adjust_progress() encgroups='--mysqld|sst|xtrabackup' -check_server_ssl_config() -{ - # backward-compatible behavior: - tcert=$(parse_cnf 'sst' 'tca') - tcap=$(parse_cnf 'sst' 'tcapath') - tpem=$(parse_cnf 'sst' 'tcert') - tkey=$(parse_cnf 'sst' 'tkey') - # reading new ssl configuration options: - local tcert2=$(parse_cnf "$encgroups" 'ssl-ca') - local tcap2=$(parse_cnf "$encgroups" 'ssl-capath') - local tpem2=$(parse_cnf "$encgroups" 'ssl-cert') - local tkey2=$(parse_cnf "$encgroups" 'ssl-key') - # if there are no old options, then we take new ones: - if [ -z "$tcert" -a -z "$tcap" -a -z "$tpem" -a -z "$tkey" ]; then - tcert="$tcert2" - tcap="$tcap2" - tpem="$tpem2" - tkey="$tkey2" - # checking for presence of the new-style SSL configuration: - elif [ -n "$tcert2" -o -n "$tcap2" -o -n "$tpem2" -o -n "$tkey2" ]; then - if [ "$tcert" != "$tcert2" -o \ - "$tcap" != "$tcap2" -o \ - "$tpem" != "$tpem2" -o \ - "$tkey" != "$tkey2" ] - then - wsrep_log_info \ - "new ssl configuration options (ssl-ca[path], ssl-cert" \ - "and ssl-key) are ignored by SST due to presence" \ - "of the tca[path], tcert and/or tkey in the [sst] section" - fi - fi - if [ -n "$tcert" ]; then - tcert=$(trim_string "$tcert") - if [ "${tcert%/}" != "$tcert" ] || [ -d "$tcert" ]; then - tcap="$tcert" - tcert="" - fi - fi - if [ -n "$tcap" ]; then - tcap=$(trim_string "$tcap") - fi -} - read_cnf() { sfmt=$(parse_cnf sst streamfmt 'xbstream') @@ -659,7 +615,7 @@ cleanup_at_exit() cleanup_pid $CHECK_PID "$BACKUP_PID" fi fi - [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" + [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" || : fi if [ -n "$progress" -a -p "$progress" ]; then @@ -670,27 +626,31 @@ cleanup_at_exit() wsrep_log_info "Cleaning up temporary directories" if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if [ -n "$STATDIR" ]; then - [ -d "$STATDIR" ] && rm -rf "$STATDIR" - fi + [ -n "$STATDIR" -a -d "$STATDIR" ] && rm -rf "$STATDIR" || : else [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || : [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || : fi # Final cleanup - pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. - if [ $$ -eq $pgid ]; then - # This means a signal was delivered to the process. - # So, more cleanup. - if [ $estatus -ge 128 ]; then - kill -KILL -- -$$ || : + if [ -n "$pgid" ]; then + if [ $$ -eq $pgid ]; then + # This means a signal was delivered to the process. + # So, more cleanup. + if [ $estatus -ge 128 ]; then + kill -KILL -- -$$ || : + fi fi fi + if [ -n "${SST_PID:-}" ]; then + [ -f "$SST_PID" ] && rm -f "$SST_PID" || : + fi + exit $estatus } @@ -862,7 +822,7 @@ monitor_process() done } -# check the version, we require XB-2.4 to ensure that we can pass the +# check the version, we require XB-2.3.5 to ensure that we can pass the # datadir via the command-line option XB_REQUIRED_VERSION="2.3.5" @@ -992,13 +952,13 @@ setup_commands() if [ -n "$INNODB_FORCE_RECOVERY" ]; then recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi - INNOAPPLY="$BACKUP_BIN$disver$recovery${iapts:+ }$iapts$INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY" - INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories '$DATA' $INNOMOVE" + INNOAPPLY="$BACKUP_BIN$disver$recovery${iapts:+ }$iapts$INNOEXTRA --apply-log${rebuildcmd:+ }$rebuildcmd --datadir='$DATA' '$DATA' $INNOAPPLY" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --datadir='${TDATA:-$DATA}' '$DATA' $INNOMOVE" local sfmt_work="$sfmt" if [ "$sfmt" = 'mbstream' ]; then sfmt_work='xbstream' fi - INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF$disver${iopts:+ }$iopts $tmpopts$INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP" + INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt_work --datadir='$DATA' '$itmpdir' $INNOBACKUP" } get_stream @@ -1026,7 +986,7 @@ then fi wsrep_log_info "Using '$xtmpdir' as xtrabackup temporary directory" - tmpopts="--tmpdir='$xtmpdir'" + tmpopts=" --tmpdir='$xtmpdir'" itmpdir="$(mktemp -d)" wsrep_log_info "Using '$itmpdir' as xtrabackup working directory" @@ -1192,10 +1152,23 @@ then impts="--parallel=$backup_threads${impts:+ }$impts" fi - stagemsg='Joiner-Recv' + SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" - sencrypted=1 - nthreads=1 + # give some time for previous SST to complete: + check_round=0 + while check_pid "$SST_PID" 0; do + wsrep_log_info "previous SST is not completed, waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "previous SST script still running." + exit 114 # EALREADY + fi + sleep 1 + done + + echo $$ > "$SST_PID" + + stagemsg='Joiner-Recv' MODULE="xtrabackup_sst" @@ -1239,7 +1212,7 @@ then fi get_keys - if [ $encrypt -eq 1 -a $sencrypted -eq 1 ]; then + if [ $encrypt -eq 1 ]; then strmcmd="$ecmd | $strmcmd" fi @@ -1294,12 +1267,14 @@ then if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ || : - cd "$OLD_PWD" + if [ -d "$binlog_dir" ]; then + cd "$binlog_dir" + wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" + rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : + [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ + rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ + cd "$OLD_PWD" + fi fi TDATA="$DATA" @@ -1316,7 +1291,7 @@ then fi # Rebuild indexes for compact backups - if grep -q -F 'compact = 1' "$DATA/xtrabackup_checkpoints"; then + if grep -qw -F 'compact = 1' "$DATA/xtrabackup_checkpoints"; then wsrep_log_info "Index compaction detected" get_proc nthreads=$(parse_cnf "$encgroups" 'rebuild-threads' $nproc)