From fd9a11d8a5b820f8d08b28fb2f198edc6f196342 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Mon, 6 Jan 2025 01:43:42 +0100 Subject: [PATCH] MDEV-35749: Add support for --use-memory option for SST with mariabackup Mariabackup (mariadb-backup) supports the --use-memory option that sets the buffer pool size for innodb. However, current SST scripts do not use this option. This commit adds support for this option, the value for which can be specified via the "use_memory" parameter in the configuration file in the [sst], [mariabackup] or [xtrabackup] sections (supported only for compatibility with old configurations). In addition, if the innodb_buffer_pool_size option is specified in the user configuration (in the main server configuration sections) or passed to the SST scripts or the server via arguments, its value is also passed to mariadb-backup as the value for the --use-memory option. A new section name [mariabackup] has also been added, which can be used instead of the deprecated [xtrabackup] (the section name "mariabackup" was specified in the documentation, but was not actually supported by SST scripts before this commit). --- .../galera_sst_mariabackup_use_memory.result | 13 +++++ .../t/galera_sst_mariabackup_use_memory.cnf | 8 +++ .../t/galera_sst_mariabackup_use_memory.test | 53 +++++++++++++++++++ scripts/wsrep_sst_common.sh | 24 +++++++-- scripts/wsrep_sst_mariabackup.sh | 40 +++++++++----- 5 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_sst_mariabackup_use_memory.result create mode 100644 mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.cnf create mode 100644 mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.test diff --git a/mysql-test/suite/galera/r/galera_sst_mariabackup_use_memory.result b/mysql-test/suite/galera/r/galera_sst_mariabackup_use_memory.result new file mode 100644 index 00000000000..1d631198990 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sst_mariabackup_use_memory.result @@ -0,0 +1,13 @@ +connection node_2; +connection node_1; +connection node_1; +connection node_2; +connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +include/assert_grep.inc [mariabackup: Using 128974848 bytes for buffer pool \(set by --use-memory parameter\)] +disconnect node_2; +disconnect node_1; diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.cnf b/mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.cnf new file mode 100644 index 00000000000..05a647eb090 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.cnf @@ -0,0 +1,8 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=mariabackup +wsrep_sst_auth="root:" + +[mariabackup] +use_memory=123m diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.test b/mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.test new file mode 100644 index 00000000000..f6ba873976e --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.test @@ -0,0 +1,53 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_mariabackup.inc + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_2 + +--echo Shutting down server ... +--source include/shutdown_mysqld.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +# +# Force SST +# +--echo Cleaning var directory ... +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mtr +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/performance_schema +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/test +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mysql +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data + +--connection node_2 + +--echo Starting server ... +let $restart_noprint=2; +--source include/start_mysqld.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +# Confirm that IST did not take place +--let $assert_text = mariabackup: Using 128974848 bytes for buffer pool \(set by --use-memory parameter\) +--let $assert_select = mariabackup: Using 128974848 bytes for buffer pool \(set by --use-memory parameter\) +--let $assert_count = 1 +--let $assert_file = $MYSQLTEST_VARDIR/mysqld.2/data/mariabackup.prepare.log +--let $assert_only_after = Starting InnoDB instance for recovery +--source include/assert_grep.inc + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc + +--source include/galera_end.inc diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index ff4befa6680..550165b0c3a 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -209,6 +209,7 @@ INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}") INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}") INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}") INNODB_BUFFER_POOL="" +INNODB_BUFFER_POOL_SIZE="" INNODB_FORCE_RECOVERY="" INNOEXTRA="" @@ -329,6 +330,10 @@ case "$1" in readonly INNODB_BUFFER_POOL=$(trim_string "$2") shift ;; + '--innodb-buffer-pool-size') + readonly INNODB_BUFFER_POOL_SIZE=$(trim_string "$2") + shift + ;; '--defaults-file') file=$(trim_string "$2") readonly WSREP_SST_OPT_DEFAULT="$1=$file" @@ -607,6 +612,12 @@ case "$1" in fi skip_mysqld_arg=1 ;; + '--innodb-buffer-pool-size') + if [ -z "$INNODB_BUFFER_POOL_SIZE" ]; then + MYSQLD_OPT_INNODB_BUFFER_POOL_SIZE=$(trim_string "$value") + fi + skip_mysqld_arg=1 + ;; '--innodb-force-recovery') if [ -n "$value" -a "$value" != "0" ]; then INNODB_FORCE_RECOVERY=$(trim_string "$value") @@ -696,6 +707,10 @@ if [ -n "${MYSQLD_OPT_INNODB_BUFFER_POOL:-}" -a \ -z "$INNODB_BUFFER_POOL" ]; then readonly INNODB_BUFFER_POOL="$MYSQLD_OPT_INNODB_BUFFER_POOL" fi +if [ -n "${MYSQLD_OPT_INNODB_BUFFER_POOL_SIZE:-}" -a \ + -z "$INNODB_BUFFER_POOL_SIZE" ]; then + readonly INNODB_BUFFER_POOL_SIZE="$MYSQLD_OPT_INNODB_BUFFER_POOL_SIZE" +fi if [ -n "${MYSQLD_OPT_LOG_BIN:-}" -a \ -z "$WSREP_SST_OPT_BINLOG" ]; then readonly WSREP_SST_OPT_BINLOG="$MYSQLD_OPT_LOG_BIN" @@ -752,6 +767,9 @@ fi if [ -n "$INNODB_BUFFER_POOL" ]; then INNOEXTRA="$INNOEXTRA --innodb-buffer-pool-filename='$INNODB_BUFFER_POOL'" fi +if [ -n "$INNODB_BUFFER_POOL_SIZE" ]; then + INNOEXTRA="$INNOEXTRA --innodb-buffer-pool-size='$INNODB_BUFFER_POOL_SIZE'" +fi if [ -n "$WSREP_SST_OPT_BINLOG" ]; then INNOEXTRA="$INNOEXTRA --log-bin='$WSREP_SST_OPT_BINLOG'" if [ -n "$WSREP_SST_OPT_BINLOG_INDEX" ]; then @@ -1775,14 +1793,14 @@ simple_cleanup() create_data() { OLD_PWD="$(pwd)" + DATA_DIR="$OLD_PWD" if [ -n "$DATA" -a "$DATA" != '.' ]; then [ ! -d "$DATA" ] && mkdir -p "$DATA" cd "$DATA" + DATA_DIR="$(pwd)" + cd "$OLD_PWD" fi - DATA_DIR="$(pwd)" - - cd "$OLD_PWD" } create_dirs() diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index e50704b7cdc..3e13efb6c6d 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -540,7 +540,8 @@ adjust_progress() fi } -encgroups='--mysqld|sst|xtrabackup' +bkgroups='sst|xtrabackup|mariabackup' +encgroups="--mysqld|$bkgroups" read_cnf() { @@ -596,26 +597,34 @@ read_cnf() ssl_dhparams=$(parse_cnf "$encgroups" 'ssl-dhparams') fi - sockopt=$(parse_cnf sst sockopt "") - progress=$(parse_cnf sst progress "") + sockopt=$(parse_cnf sst sockopt) + progress=$(parse_cnf sst progress) ttime=$(parse_cnf sst time 0) cpat='.*\.pem$\|.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$' [ "$OS" = 'FreeBSD' ] && cpat=$(echo "$cpat" | sed 's/\\|/|/g') cpat=$(parse_cnf sst cpat "$cpat") - scomp=$(parse_cnf sst compressor "") - sdecomp=$(parse_cnf sst decompressor "") + scomp=$(parse_cnf sst compressor) + sdecomp=$(parse_cnf sst decompressor) - rlimit=$(parse_cnf sst rlimit "") + rlimit=$(parse_cnf sst rlimit) uextra=$(parse_cnf sst use-extra 0) - speciald=$(parse_cnf sst sst-special-dirs 1) - iopts=$(parse_cnf sst inno-backup-opts "") - iapts=$(parse_cnf sst inno-apply-opts "") - impts=$(parse_cnf sst inno-move-opts "") - stimeout=$(parse_cnf sst sst-initial-timeout 300) - ssyslog=$(parse_cnf sst sst-syslog 0) - ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}") + speciald=$(parse_cnf sst 'sst-special-dirs' 1) + iopts=$(parse_cnf "$bkgroups" 'inno-backup-opts') + iapts=$(parse_cnf "$bkgroups" 'inno-apply-opts') + impts=$(parse_cnf "$bkgroups" 'inno-move-opts') + use_memory=$(parse_cnf "$bkgroups" 'use-memory') + if [ -z "$use_memory" ]; then + if [ -n "$INNODB_BUFFER_POOL_SIZE" ]; then + use_memory="$INNODB_BUFFER_POOL_SIZE" + else + use_memory=$(parse_cnf '--mysqld' 'innodb-buffer-pool-size') + fi + fi + stimeout=$(parse_cnf sst 'sst-initial-timeout' 300) + ssyslog=$(parse_cnf sst 'sst-syslog' 0) + ssystag=$(parse_cnf mysqld_safe 'syslog-tag' "${SST_SYSLOG_TAG:-}") ssystag="$ssystag-" - sstlogarchive=$(parse_cnf sst sst-log-archive 1) + sstlogarchive=$(parse_cnf sst 'sst-log-archive' 1) sstlogarchivedir="" if [ $sstlogarchive -ne 0 ]; then sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir \ @@ -1053,6 +1062,9 @@ setup_commands() if [ -n "$INNODB_FORCE_RECOVERY" ]; then recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi + if [ -n "$use_memory" ]; then + INNOEXTRA="$INNOEXTRA --use-memory=$use_memory" + fi INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY" INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --galera-info --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP"