Fixes after review and correction of the problems caused by the fact that during the SST innodb plugin is not yet initialized, as well as problems with running tests from the root user (not directly related to the MDEV-10754).

mysql-test/suite/galera/include/galera_wsrep_recover.inc

@@ -1,5 +1,14 @@
 --echo Performing --wsrep-recover ...
+#
+# When mysqld is run by a root user it will fail to start unless
+# we specify what user is ran it (using "--user root" option):
+#
+if ($MYSQL_TEST_ROOT == 1) {
+--exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --innodb --wsrep-recover --user root > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1
+}
+if ($MYSQL_TEST_ROOT != 1) {
 --exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --innodb --wsrep-recover > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1
+}
 
 --perl
 	use strict;

mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf

@@ -2,10 +2,10 @@
 
 [mysqld]
 wsrep_sst_method=rsync
-innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test
 
 [mysqld.1]
 wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'
 
 [mysqld.2]
+innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test_2
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'

mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test

@@ -13,5 +13,4 @@
 --source include/auto_increment_offset_restore.inc
 
 # cleanup temporary database files:
---remove_files_wildcard $MYSQL_TMP_DIR/rsync_test *
---rmdir $MYSQL_TMP_DIR/rsync_test
+--remove_files_wildcard $MYSQL_TMP_DIR/rsync_test_2 *

scripts/wsrep_sst_common.sh

@@ -27,6 +27,7 @@ WSREP_SST_OPT_PSWD=${WSREP_SST_OPT_PSWD:-}
 WSREP_SST_OPT_DEFAULT=""
 WSREP_SST_OPT_EXTRA_DEFAULT=""
 WSREP_SST_OPT_SUFFIX_DEFAULT=""
+WSREP_SST_OPT_SUFFIX_VALUE=""
 
 while [ $# -gt 0 ]; do
 case "$1" in
@@ -76,6 +77,7 @@ case "$1" in
         ;;
     '--defaults-group-suffix')
         readonly WSREP_SST_OPT_SUFFIX_DEFAULT="$1=$2"
+        readonly WSREP_SST_OPT_SUFFIX_VALUE="$2"
         shift
         ;;
     '--host')

scripts/wsrep_sst_rsync.sh

@@ -123,8 +123,16 @@ fi
 
 INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""}
 # if INNODB_DATA_HOME_DIR env. variable is not set, try to get it from my.cnf
-if [ -z "INNODB_DATA_HOME_DIR" ]; then
-    INNODB_DATA_HOME_DIR=$(parse_cnf --mysqld innodb-data-home-dir '')
+if [ -z "$INNODB_DATA_HOME_DIR" ]; then
+    INNODB_DATA_HOME_DIR=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-data-home-dir '')
+fi
+
+if [ -n "$INNODB_DATA_HOME_DIR" ]; then
+    # handle both relative and absolute paths
+    INNODB_DATA_HOME_DIR=$(cd $WSREP_SST_OPT_DATA; mkdir -p "$INNODB_DATA_HOME_DIR"; cd $INNODB_DATA_HOME_DIR; pwd -P)
+else
+    # default to datadir
+    INNODB_DATA_HOME_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P)
 fi
 
 # Old filter - include everything except selected
@@ -135,8 +143,8 @@ fi
 
 # New filter - exclude everything except dirs (schemas) and innodb files
 FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes'
-        -f '+ /wsrep_sst_binlog.tar' -f '+ $INNODB_DATA_HOME_DIR/ib_lru_dump'
-        -f '+ $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*')
+        -f '+ /wsrep_sst_binlog.tar' -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump'
+        -f '- $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*'"
 
 SSTKEY=$(parse_cnf sst tkey "")
 SSTCERT=$(parse_cnf sst tcert "")
@@ -242,6 +250,19 @@ EOF
             exit $RC
         fi
 
+        # Transfer InnoDB data files
+        rsync ${STUNNEL:+--rsh="$STUNNEL"} \
+              --owner --group --perms --links --specials \
+              --ignore-times --inplace --dirs --delete --quiet \
+              $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \
+              -f '- **' "$INNODB_DATA_HOME_DIR/" \
+              rsync://$WSREP_SST_OPT_ADDR-data_dir >&2 || RC=$?
+
+        if [ $RC -ne 0 ]; then
+            wsrep_log_error "rsync innodb_data_home_dir returned code $RC:"
+            exit 255 # unknown error
+        fi
+
         # second, we transfer InnoDB log files
         rsync ${STUNNEL:+--rsh="$STUNNEL"} \
               --owner --group --perms --links --specials \
@@ -340,8 +361,12 @@ $SILENT
     path = $WSREP_SST_OPT_DATA
 [$MODULE-log_dir]
     path = $WSREP_LOG_DIR
+[$MODULE-data_dir]
+    path = $INNODB_DATA_HOME_DIR
 EOF
 
+[ "$(whoami)" != root ] || sed -i '/read only = no/s/.*/&\nuid = root\ngid = root\n/' "$RSYNC_CONF"
+
 #    rm -rf "$DATA"/ib_logfile* # we don't want old logs around
 
     # listen at all interfaces (for firewalled setups)

sql/wsrep_mysqld.h

@@ -160,6 +160,7 @@ extern "C" time_t wsrep_thd_query_start(THD *thd);
 extern "C" query_id_t wsrep_thd_query_id(THD *thd);
 extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd);
 extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id);
+extern "C" void wsrep_set_data_home_dir(const char *data_dir);
 
 extern void wsrep_close_client_connections(my_bool wait_to_end);
 extern int  wsrep_wait_committing_connections_close(int wait_time);

sql/wsrep_sst.cc

@@ -67,6 +67,13 @@ bool wsrep_sst_method_update (sys_var *self, THD* thd, enum_var_type type)
     return 0;
 }
 
+static const char* data_home_dir = NULL;
+
+extern "C"
+void wsrep_set_data_home_dir(const char *data_dir)
+{
+  data_home_dir= (data_dir && *data_dir) ? data_dir : NULL;
+}
 
 static void make_wsrep_defaults_file()
 {
@@ -549,6 +556,29 @@ static int sst_append_auth_env(wsp::env& env, const char* sst_auth)
   return -env.error();
 }
 
+#define DATA_HOME_DIR_ENV "INNODB_DATA_HOME_DIR"
+
+static int sst_append_data_dir(wsp::env& env, const char* data_dir)
+{
+  int const data_dir_size= strlen(DATA_HOME_DIR_ENV) + 1 /* = */
+    + (data_dir ? strlen(data_dir) : 0) + 1 /* \0 */;
+
+  wsp::string data_dir_str(data_dir_size); // for automatic cleanup on return
+  if (!data_dir_str()) return -ENOMEM;
+
+  int ret= snprintf(data_dir_str(), data_dir_size, "%s=%s",
+                    DATA_HOME_DIR_ENV, data_dir ? data_dir : "");
+
+  if (ret < 0 || ret >= data_dir_size)
+  {
+    WSREP_ERROR("sst_append_data_dir(): snprintf() failed: %d", ret);
+    return (ret < 0 ? ret : -EMSGSIZE);
+  }
+
+  env.append(data_dir_str());
+  return -env.error();
+}
+
 static ssize_t sst_prepare_other (const char*  method,
                                   const char*  sst_auth,
                                   const char*  addr_in,
@@ -610,6 +640,16 @@ static ssize_t sst_prepare_other (const char*  method,
     return ret;
   }
 
+  if (data_home_dir)
+  {
+    if ((ret= sst_append_data_dir(env, data_home_dir)))
+    {
+      WSREP_ERROR("sst_prepare_other(): appending data "
+                  "directory failed: %d", ret);
+      return ret;
+    }
+  }
+
   pthread_t tmp;
   sst_thread_arg arg(cmd_str(), env());
   mysql_mutex_lock (&arg.lock);
@@ -1301,6 +1341,16 @@ wsrep_cb_status_t wsrep_sst_donate_cb (void* app_ctx, void* recv_ctx,
     return WSREP_CB_FAILURE;
   }
 
+  if (data_home_dir)
+  {
+    if ((ret= sst_append_data_dir(env, data_home_dir)))
+    {
+      WSREP_ERROR("wsrep_sst_donate_cb(): appending data "
+                  "directory failed: %d", ret);
+      return WSREP_CB_FAILURE;
+    }
+  }
+
   if (!strcmp (WSREP_SST_MYSQLDUMP, method))
   {
     ret = sst_donate_mysqldump(data, &current_gtid->uuid, uuid_str,

storage/innobase/handler/ha_innodb.cc

@@ -3623,6 +3623,12 @@ innobase_init(
 	srv_data_home = (innobase_data_home_dir ? innobase_data_home_dir :
 			 default_path);
 
+#ifdef WITH_WSREP
+	/* If we use the wsrep API, then we need to tell the server
+	the path to the data files (for passing it to the SST scripts): */
+	wsrep_set_data_home_dir(innobase_data_home_dir);
+#endif /* WITH_WSREP */
+
 	/* Set default InnoDB data file size to 12 MB and let it be
 	auto-extending. Thus users can use InnoDB in >= 4.0 without having
 	to specify any startup options. */