Merge 10.8 into 10.9

extra/mariabackup/backup_copy.cc

@@ -1483,8 +1483,6 @@ bool backup_start(CorruptedPages &corrupted_pages)
 		if (!write_galera_info(mysql_connection)) {
 			return(false);
 		}
-                // copied from xtrabackup. what is it needed for here?
-		write_current_binlog_file(mysql_connection);
 	}
 
 	if (opt_binlog_info == BINLOG_INFO_ON) {

extra/mariabackup/backup_mysql.cc

@@ -1444,14 +1444,18 @@ write_galera_info(MYSQL *connection)
 
 	if ((state_uuid == NULL && state_uuid55 == NULL)
 		|| (last_committed == NULL && last_committed55 == NULL)) {
-		msg("Failed to get master wsrep state from SHOW STATUS.");
-		result = false;
+		msg("Warning: failed to get master wsrep state from SHOW STATUS.");
+		result = true;
 		goto cleanup;
 	}
 
 	result = backup_file_printf(XTRABACKUP_GALERA_INFO,
 		"%s:%s\n", state_uuid ? state_uuid : state_uuid55,
 			last_committed ? last_committed : last_committed55);
+	if (result)
+	{
+		write_current_binlog_file(connection);
+	}
 
 cleanup:
 	free_mysql_variables(status);

extra/mariabackup/fil_cur.cc

@@ -129,6 +129,7 @@ xb_fil_cur_open(
 	in case of error */
 	cursor->buf = NULL;
 	cursor->node = NULL;
+	cursor->n_process_batch = 0;
 
 	cursor->space_id = node->space->id;
 
@@ -374,6 +375,8 @@ xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t*	cursor,
 		return(XB_FIL_CUR_EOF);
 	}
 
+reinit_buf:
+	cursor->n_process_batch++;
 	if (to_read > (ib_int64_t) cursor->buf_size) {
 		to_read = (ib_int64_t) cursor->buf_size;
 	}
@@ -416,8 +419,26 @@ read_retry:
 
 	if (os_file_read(IORequestRead, cursor->file, cursor->buf, offset,
 			 (ulint) to_read, nullptr) != DB_SUCCESS) {
-		ret = XB_FIL_CUR_ERROR;
-		goto func_exit;
+		if (!srv_is_undo_tablespace(cursor->space_id)) {
+			ret = XB_FIL_CUR_ERROR;
+			goto func_exit;
+		}
+
+		if (cursor->buf_page_no
+		    >= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES) {
+			ret = XB_FIL_CUR_SKIP;
+			goto func_exit;
+		}
+
+		to_read = SRV_UNDO_TABLESPACE_SIZE_IN_PAGES * page_size;
+
+		if (cursor->n_process_batch > 1) {
+			ret = XB_FIL_CUR_ERROR;
+			goto func_exit;
+		}
+
+		space->release();
+		goto reinit_buf;
 	}
 
 	defer = UT_LIST_GET_FIRST(space->chain)->deferred;

extra/mariabackup/fil_cur.h

@@ -58,6 +58,7 @@ struct xb_fil_cur_t {
 	uint		thread_n;	/*!< thread number for diagnostics */
 	uint32_t	space_id;	/*!< ID of tablespace */
 	uint32_t	space_size;	/*!< space size in pages */
+	uint32_t	n_process_batch;/*!< Number of batch processed */
 
 	/** @return whether this is not a file-per-table tablespace */
 	bool is_system() const

extra/mariabackup/xtrabackup.cc

@@ -187,6 +187,7 @@ struct xb_filter_entry_t{
 	xb_filter_entry_t *name_hash;
 };
 
+/** whether log_copying_thread() is active; protected by recv_sys.mutex */
 static bool log_copying_running;
 
 int xtrabackup_parallel;
@@ -241,6 +242,10 @@ long innobase_file_io_threads = 4;
 ulong innobase_read_io_threads = 4;
 ulong innobase_write_io_threads = 4;
 
+/** Store the failed read of undo tablespace ids. Protected by
+recv_sys.mutex. */
+static std::set<uint32_t> fail_undo_ids;
+
 longlong innobase_page_size = (1LL << 14); /* 16KB */
 char*	innobase_buffer_pool_filename = NULL;
 
@@ -401,6 +406,10 @@ struct ddl_tracker_t {
 
 static ddl_tracker_t ddl_tracker;
 
+/** Store the space ids of truncated undo log tablespaces. Protected
+by recv_sys.mutex */
+static std::set<uint32_t> undo_trunc_ids;
+
 /** Stores the space ids of page0 INIT_PAGE redo records. It is
 used to indicate whether the given deferred tablespace can
 be reconstructed. */
@@ -913,6 +922,11 @@ static void backup_file_op_fail(uint32_t space_id, int type,
 	}
 }
 
+static void backup_undo_trunc(uint32_t space_id)
+{
+  undo_trunc_ids.insert(space_id);
+}
+
 /* Function to store the space id of page0 INIT_PAGE
 @param	space_id	space id which has page0 init page */
 static void backup_first_page_op(space_id_t space_id)
@@ -2140,7 +2154,6 @@ static bool innodb_init_param()
 	srv_buf_pool_size = (ulint) xtrabackup_use_memory;
 	srv_buf_pool_chunk_unit = srv_buf_pool_size;
 
-	srv_n_file_io_threads = (uint) innobase_file_io_threads;
 	srv_n_read_io_threads = (uint) innobase_read_io_threads;
 	srv_n_write_io_threads = (uint) innobase_write_io_threads;
 
@@ -2912,15 +2925,27 @@ static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n,
 	}
 
 	/* The main copy loop */
-	while ((res = xb_fil_cur_read(&cursor, corrupted_pages)) ==
-		XB_FIL_CUR_SUCCESS) {
+	while (1) {
+		res = xb_fil_cur_read(&cursor, corrupted_pages);
+		if (res == XB_FIL_CUR_ERROR) {
+		       goto error;
+		}
+
+		if (res == XB_FIL_CUR_EOF) {
+			break;
+		}
+
 		if (!write_filter.process(&write_filt_ctxt, dstfile)) {
 			goto error;
 		}
-	}
 
-	if (res == XB_FIL_CUR_ERROR) {
-		goto error;
+		if (res == XB_FIL_CUR_SKIP) {
+			mysql_mutex_lock(&recv_sys.mutex);
+			fail_undo_ids.insert(
+				static_cast<uint32_t>(cursor.space_id));
+			mysql_mutex_unlock(&recv_sys.mutex);
+			break;
+		}
 	}
 
 	if (write_filter.finalize
@@ -3122,16 +3147,18 @@ static void log_copying_thread()
   my_thread_end();
 }
 
+/** whether io_watching_thread() is active; protected by log_sys.mutex */
 static bool have_io_watching_thread;
-static pthread_t io_watching_thread_id;
 
 /* io throttle watching (rough) */
-static void *io_watching_thread(void*)
+static void io_watching_thread()
 {
+  my_thread_init();
   /* currently, for --backup only */
-  ut_a(xtrabackup_backup);
+  ut_ad(xtrabackup_backup);
 
   mysql_mutex_lock(&recv_sys.mutex);
+  ut_ad(have_io_watching_thread);
 
   while (log_copying_running && !metadata_to_lsn)
   {
@@ -3144,9 +3171,10 @@ static void *io_watching_thread(void*)
 
   /* stop io throttle */
   xtrabackup_throttle= 0;
+  have_io_watching_thread= false;
   mysql_cond_broadcast(&wait_throttle);
   mysql_mutex_unlock(&recv_sys.mutex);
-  return nullptr;
+  my_thread_end();
 }
 
 #ifndef DBUG_OFF
@@ -4405,27 +4433,29 @@ end:
 # define xb_set_max_open_files(x) 0UL
 #endif
 
-static void stop_backup_threads(bool running)
+static void stop_backup_threads()
 {
-  if (running)
+  mysql_cond_broadcast(&log_copying_stop);
+
+  if (log_copying_running || have_io_watching_thread)
   {
+    mysql_mutex_unlock(&recv_sys.mutex);
     fputs("mariabackup: Stopping log copying thread", stderr);
     fflush(stderr);
-    while (log_copying_running)
+    mysql_mutex_lock(&recv_sys.mutex);
+    while (log_copying_running || have_io_watching_thread)
     {
+      mysql_cond_broadcast(&log_copying_stop);
+      mysql_mutex_unlock(&recv_sys.mutex);
       putc('.', stderr);
       fflush(stderr);
       std::this_thread::sleep_for(std::chrono::milliseconds(200));
+      mysql_mutex_lock(&recv_sys.mutex);
     }
     putc('\n', stderr);
-    mysql_cond_destroy(&log_copying_stop);
   }
 
-  if (have_io_watching_thread)
-  {
-    pthread_join(io_watching_thread_id, nullptr);
-    mysql_cond_destroy(&wait_throttle);
-  }
+  mysql_cond_destroy(&log_copying_stop);
 }
 
 /** Implement the core of --backup
@@ -4449,11 +4479,7 @@ static bool xtrabackup_backup_low()
 		}
 
 		recv_sys.lsn = lsn;
-		mysql_cond_broadcast(&log_copying_stop);
-		const bool running= log_copying_running;
-		mysql_mutex_unlock(&recv_sys.mutex);
-		stop_backup_threads(running);
-		mysql_mutex_lock(&recv_sys.mutex);
+		stop_backup_threads();
 	}
 
 	if (metadata_to_lsn && xtrabackup_copy_logfile()) {
@@ -4472,6 +4498,23 @@ static bool xtrabackup_backup_low()
 
 	dst_log_file = NULL;
 
+	std::vector<uint32_t> failed_ids;
+	std::set_difference(
+		fail_undo_ids.begin(), fail_undo_ids.end(),
+		undo_trunc_ids.begin(), undo_trunc_ids.end(),
+		std::inserter(failed_ids, failed_ids.begin()));
+
+	for (uint32_t id : failed_ids) {
+		msg("mariabackup: Failed to read undo log "
+		    "tablespace space id %d and there is no undo "
+		    "tablespace truncation redo record.",
+		    id);
+	}
+
+	if (failed_ids.size() > 0) {
+		return false;
+	}
+
 	if(!xtrabackup_incremental) {
 		strcpy(metadata_type, "full-backuped");
 		metadata_from_lsn = 0;
@@ -4545,6 +4588,7 @@ static bool xtrabackup_backup_func()
 
 	srv_operation = SRV_OPERATION_BACKUP;
 	log_file_op = backup_file_op;
+	undo_space_trunc = backup_undo_trunc;
 	first_page_init = backup_first_page_op;
 	metadata_to_lsn = 0;
 
@@ -4554,12 +4598,12 @@ fail:
 		if (log_copying_running) {
 			mysql_mutex_lock(&recv_sys.mutex);
 			metadata_to_lsn = 1;
-			mysql_cond_broadcast(&log_copying_stop);
+			stop_backup_threads();
 			mysql_mutex_unlock(&recv_sys.mutex);
-			stop_backup_threads(true);
 		}
 
 		log_file_op = NULL;
+		undo_space_trunc = NULL;
 		first_page_init = NULL;
 		if (dst_log_file) {
 			ds_close(dst_log_file);
@@ -4595,7 +4639,6 @@ fail:
 	xb_filters_init();
 
 	xb_fil_io_init();
-	srv_n_file_io_threads = srv_n_read_io_threads;
 
 	if (os_aio_init()) {
 		msg("Error: cannot initialize AIO subsystem");
@@ -4666,13 +4709,15 @@ fail:
 		goto fail;
 	}
 	log_copying_running = true;
+
+	mysql_cond_init(0, &log_copying_stop, nullptr);
+
 	/* start io throttle */
-	if(xtrabackup_throttle) {
+	if (xtrabackup_throttle) {
 		io_ticket = xtrabackup_throttle;
 		have_io_watching_thread = true;
 		mysql_cond_init(0, &wait_throttle, nullptr);
-		mysql_thread_create(0, &io_watching_thread_id, nullptr,
-				    io_watching_thread, nullptr);
+		std::thread(io_watching_thread).detach();
 	}
 
 	/* Populate fil_system with tablespaces to copy */
@@ -4699,7 +4744,6 @@ fail:
 
 	DBUG_MARIABACKUP_EVENT("before_innodb_log_copy_thread_started", {});
 
-	mysql_cond_init(0, &log_copying_stop, nullptr);
 	std::thread(log_copying_thread).detach();
 
 	/* FLUSH CHANGED_PAGE_BITMAPS call */
@@ -4796,6 +4840,7 @@ fail:
 
 	innodb_shutdown();
 	log_file_op = NULL;
+	undo_space_trunc = NULL;
 	first_page_init = NULL;
 	pthread_cond_destroy(&scanned_lsn_cond);
 	if (!corrupted_pages.empty()) {
@@ -5998,12 +6043,6 @@ error:
 
 	fil_system.freeze_space_list = 0;
 
-	/* increase IO threads */
-	if (srv_n_file_io_threads < 10) {
-		srv_n_read_io_threads = 4;
-		srv_n_write_io_threads = 4;
-	}
-
 	msg("Starting InnoDB instance for recovery.");
 
 	msg("mariabackup: Using %lld bytes for buffer pool "