diff --git a/mysql-test/suite/binlog_in_engine/binlog_flush_purge.test b/mysql-test/suite/binlog_in_engine/binlog_flush_purge.test index f97b6f4ffe9..f8b74f56992 100644 --- a/mysql-test/suite/binlog_in_engine/binlog_flush_purge.test +++ b/mysql-test/suite/binlog_in_engine/binlog_flush_purge.test @@ -1,6 +1,8 @@ --source include/have_binlog_format_row.inc --source include/have_innodb_binlog.inc +RESET MASTER; + CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE t2 (a INT PRIMARY KEY, b VARCHAR(2048)) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); @@ -18,4 +20,16 @@ SHOW BINARY LOGS; FLUSH BINARY LOGS; SHOW BINARY LOGS; +RESET MASTER; +# ToDo: This will be racy, the second binlog file binlog-000001.ibb is +# pre-allocated in the background, so will be visible or not depending on exact +# timing. So just omit this SHOW BINARY LOGS or wait for both to be created +# or something. +SHOW BINARY LOGS; + +INSERT INTO t1 VALUES (100); +INSERT INTO t2 VALUES (100, 'xyzzy'); + DROP TABLE t1, t2; + +--exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 --start-position=0-1-1 > $MYSQLTEST_VARDIR/tmp/mysqlbinlog.txt diff --git a/mysql-test/suite/binlog_in_engine/binlog_in_engine.test b/mysql-test/suite/binlog_in_engine/binlog_in_engine.test index 1994c58b1de..f591eea4dae 100644 --- a/mysql-test/suite/binlog_in_engine/binlog_in_engine.test +++ b/mysql-test/suite/binlog_in_engine/binlog_in_engine.test @@ -6,6 +6,8 @@ # files and only need to include the one. --source include/have_innodb_binlog.inc +RESET MASTER; + CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; --let $gtid_pos= `SELECT @@last_gtid` INSERT INTO t1 VALUES (1); diff --git a/mysql-test/suite/binlog_in_engine/binlog_in_engine2.test b/mysql-test/suite/binlog_in_engine/binlog_in_engine2.test index eea57212f5a..262ddfd14b3 100644 --- a/mysql-test/suite/binlog_in_engine/binlog_in_engine2.test +++ b/mysql-test/suite/binlog_in_engine/binlog_in_engine2.test @@ -1,6 +1,9 @@ --source include/have_binlog_format_mixed.inc --source include/have_innodb_binlog.inc +# RESET MASTER deliberately omitted here for now; the test should be able to +# work on top of any existing binlog. + CREATE TABLE sbtest1( id INTEGER NOT NULL AUTO_INCREMENT, k INTEGER DEFAULT '0' NOT NULL, diff --git a/mysql-test/suite/binlog_in_engine/binlog_in_engine_restart.test b/mysql-test/suite/binlog_in_engine/binlog_in_engine_restart.test index e69601409b8..eccb0bb462e 100644 --- a/mysql-test/suite/binlog_in_engine/binlog_in_engine_restart.test +++ b/mysql-test/suite/binlog_in_engine/binlog_in_engine_restart.test @@ -6,6 +6,8 @@ # Note: This test also tests the --binlog-directory option by putting it # in binlog_in_engine_restart.opt . +RESET MASTER; + CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; INSERT INTO t1 VALUES (1, 0); let $i= 0; diff --git a/sql/handler.h b/sql/handler.h index 8ff048df5fe..b19d230a09b 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1570,6 +1570,8 @@ struct handlerton Used to implement FLUSH BINARY LOGS. */ bool (*binlog_flush)(); + /* Engine implementation of RESET MASTER. */ + bool (*reset_binlogs)(); /* Optional clauses in the CREATE/ALTER TABLE diff --git a/sql/log.cc b/sql/log.cc index f8de6d07773..6128d2eae3a 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -4466,6 +4466,17 @@ bool MYSQL_BIN_LOG::reset_logs(THD *thd, bool create_new_log, DBUG_RETURN(1); } + if (opt_binlog_engine_hton) + { + if (next_log_number) + { + my_error(ER_ENGINE_BINLOG_NO_RESET_FILE_NUMBER, MYF(0)); + DBUG_RETURN(true); + } + DBUG_ASSERT(create_new_log); + DBUG_RETURN(reset_engine_binlogs(thd, init_state, init_state_len)); + } + /* Mark that a RESET MASTER is in progress. This ensures that a binlog checkpoint will not try to write binlog @@ -4728,6 +4739,29 @@ err: } +bool +MYSQL_BIN_LOG::reset_engine_binlogs(THD *thd, rpl_gtid *init_state, + uint32 init_state_len) +{ + bool err; + DBUG_ASSERT(!is_relay_log); + + mysql_mutex_lock(&LOCK_log); + mysql_mutex_lock(&LOCK_index); + + err= (*opt_binlog_engine_hton->reset_binlogs)(); + if (init_state) + rpl_global_gtid_binlog_state.load(init_state, init_state_len); + else + rpl_global_gtid_binlog_state.reset(); + + mysql_mutex_unlock(&LOCK_index); + mysql_mutex_unlock(&LOCK_log); + + return err; +} + + void MYSQL_BIN_LOG::wait_for_last_checkpoint_event() { mysql_mutex_lock(&LOCK_xid_list); diff --git a/sql/log.h b/sql/log.h index 403dcd9d2a6..dc461ea9173 100644 --- a/sql/log.h +++ b/sql/log.h @@ -1100,6 +1100,8 @@ public: bool reset_logs(THD* thd, bool create_new_log, rpl_gtid *init_state, uint32 init_state_len, ulong next_log_number); + bool reset_engine_binlogs(THD *thd, rpl_gtid *init_state, + uint32 init_state_len); void wait_for_last_checkpoint_event(); void close(uint exiting); void clear_inuse_flag_when_closing(File file); diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 78ba61f0a6e..d5171f707c7 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -12276,3 +12276,5 @@ ER_CANNOT_INIT_ENGINE_BINLOG_READER eng "Cannot initialize binlog reader from storage engine %s" ER_ENGINE_BINLOG_REQUIRES_GTID eng "GTID starting position is required on master with --binlog-storage-engine enabled" +ER_ENGINE_BINLOG_NO_RESET_FILE_NUMBER + eng "RESET MASTER TO is not available when --binlog-storage-engine is enabled" diff --git a/storage/innobase/fsp/fsp_binlog.cc b/storage/innobase/fsp/fsp_binlog.cc index 3a6cf77c44c..7286d83cc6f 100644 --- a/storage/innobase/fsp/fsp_binlog.cc +++ b/storage/innobase/fsp/fsp_binlog.cc @@ -104,6 +104,14 @@ fsp_binlog_init() } +void +fsp_binlog_shutdown() +{ + pthread_cond_destroy(&active_binlog_cond); + mysql_mutex_destroy(&active_binlog_mutex); +} + + /** Write out all pages, flush, and close/detach a binlog tablespace. @param[in] file_no Index of the binlog tablespace @return DB_SUCCESS or error code */ @@ -739,6 +747,7 @@ binlog_chunk_reader::fetch_current_page() actually maintaining that, to save unnecessary buffer pool lookup. */ + goto_next_file: if (cur_file_handle >= (File)0) { my_close(cur_file_handle, MYF(0)); @@ -754,6 +763,8 @@ binlog_chunk_reader::fetch_current_page() s.page_no << srv_page_size_shift, MYF(MY_WME)); if (res == (size_t)-1) return CHUNK_READER_ERROR; + if (res == 0 && my_errno == HA_ERR_FILE_TOO_SHORT) + goto goto_next_file; page_ptr= page_buffer; return CHUNK_READER_FOUND; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index dde58b04190..693bad1dd77 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4137,6 +4137,7 @@ static int innodb_init(void* p) innobase_hton->get_binlog_reader= innodb_get_binlog_reader; innobase_hton->get_binlog_file_list= innodb_get_binlog_file_list; innobase_hton->binlog_flush= innodb_binlog_flush; + innobase_hton->reset_binlogs= innodb_reset_binlogs; innodb_remember_check_sysvar_funcs(); diff --git a/storage/innobase/handler/innodb_binlog.cc b/storage/innobase/handler/innodb_binlog.cc index 6325c2b2b50..26e7b78472b 100644 --- a/storage/innobase/handler/innodb_binlog.cc +++ b/storage/innobase/handler/innodb_binlog.cc @@ -430,6 +430,37 @@ innodb_binlog_startup_init() } +static void +innodb_binlog_init_state() +{ + first_open_binlog_file_no= ~(uint64_t)0; + binlog_cur_end_offset[0].store(~(uint64_t)0, std::memory_order_relaxed); + binlog_cur_end_offset[1].store(~(uint64_t)0, std::memory_order_relaxed); + last_created_binlog_file_no= ~(uint64_t)0; + active_binlog_file_no.store(~(uint64_t)0, std::memory_order_release); + active_binlog_space= nullptr; + binlog_cur_page_no= 0; + binlog_cur_page_offset= FIL_PAGE_DATA; + current_binlog_state_interval= innodb_binlog_state_interval; +} + + +/* Start the thread that pre-allocates new binlog files. */ +static void +start_binlog_prealloc_thread() +{ + prealloc_thread_end= false; + binlog_prealloc_thr_obj= std::thread{innodb_binlog_prealloc_thread}; + + mysql_mutex_lock(&active_binlog_mutex); + while (last_created_binlog_file_no == ~(uint64_t)0) { + /* Wait for the first binlog file to be available. */ + my_cond_wait(&active_binlog_cond, &active_binlog_mutex.m_mutex); + } + mysql_mutex_unlock(&active_binlog_mutex); +} + + /* Open the InnoDB binlog implementation. This is called from server binlog layer if the user configured the binlog to @@ -462,15 +493,8 @@ innodb_binlog_init(size_t binlog_size, const char *directory) } innodb_binlog_directory= directory; - first_open_binlog_file_no= ~(uint64_t)0; - binlog_cur_end_offset[0].store(~(uint64_t)0, std::memory_order_relaxed); - binlog_cur_end_offset[1].store(~(uint64_t)0, std::memory_order_relaxed); - last_created_binlog_file_no= ~(uint64_t)0; - active_binlog_file_no.store(~(uint64_t)0, std::memory_order_release); - active_binlog_space= nullptr; - binlog_cur_page_no= 0; - binlog_cur_page_offset= FIL_PAGE_DATA; - current_binlog_state_interval= innodb_binlog_state_interval; + innodb_binlog_init_state(); + /* Find any existing binlog files and continue writing in them. */ int res= innodb_binlog_discover(); if (res < 0) @@ -485,15 +509,7 @@ innodb_binlog_init(size_t binlog_size, const char *directory) return true; } - /* Start pre-allocating new binlog files. */ - binlog_prealloc_thr_obj= std::thread{innodb_binlog_prealloc_thread}; - - mysql_mutex_lock(&active_binlog_mutex); - while (last_created_binlog_file_no == ~(uint64_t)0) { - /* Wait for the first binlog file to be available. */ - my_cond_wait(&active_binlog_cond, &active_binlog_mutex.m_mutex); - } - mysql_mutex_unlock(&active_binlog_mutex); + start_binlog_prealloc_thread(); return false; } @@ -763,7 +779,7 @@ innodb_binlog_discover() } -void innodb_binlog_close() +void innodb_binlog_close(bool shutdown) { if (binlog_prealloc_thr_obj.joinable()) { mysql_mutex_lock(&active_binlog_mutex); @@ -782,17 +798,11 @@ void innodb_binlog_close() } } } - /* - ToDo: This doesn't seem to free all memory. I'm still getting leaks in eg. --valgrind. Find out why and fix. Example: -==3464576== at 0x48407B4: malloc (vg_replace_malloc.c:381) -==3464576== by 0x15318CD: mem_strdup(char const*) (mem0mem.inl:452) -==3464576== by 0x15321DF: fil_space_t::add(char const*, pfs_os_file_t, unsigned int, bool, bool, unsigned int) (fil0fil.cc:306) -==3464576== by 0x1558445: fsp_binlog_tablespace_create(unsigned long) (fsp0fsp.cc:3900) -==3464576== by 0x1558C70: fsp_binlog_write_cache(st_io_cache*, unsigned long, mtr_t*) (fsp0fsp.cc:4013) - */ - binlog_diff_state.free(); - pthread_cond_destroy(&active_binlog_cond); - mysql_mutex_destroy(&active_binlog_mutex); + if (shutdown) + { + binlog_diff_state.free(); + fsp_binlog_shutdown(); + } } @@ -2176,3 +2186,49 @@ innodb_find_binlogs(uint64_t *out_first, uint64_t *out_last) *out_last= last_file_no; return false; } + + +bool +innodb_reset_binlogs() +{ + bool err= false; + + /* Close existing binlog tablespaces and stop the pre-alloc thread. */ + innodb_binlog_close(false); + + /* Delete all binlog files in the directory. */ + MY_DIR *dir= my_dir(innodb_binlog_directory, MYF(MY_WME)); + if (!dir) + { + ib::error() << "Could not read the binlog directory '" << + innodb_binlog_directory << "', error code " << my_errno << "."; + err= true; + } + else + { + size_t num_entries= dir->number_of_files; + fileinfo *entries= dir->dir_entry; + for (size_t i= 0; i < num_entries; ++i) { + const char *name= entries[i].name; + uint64_t file_no; + if (!is_binlog_name(name, &file_no)) + continue; + char full_path[OS_FILE_MAX_PATH]; + binlog_name_make(full_path, file_no); + if (my_delete(full_path, MYF(MY_WME))) + err= true; + } + my_dirend(dir); + } + /* + If we get an error deleting any of the existing files, we report the error + back up. But we still try to initialize an empty binlog state, better than + leaving a non-functional binlog with corrupt internal state. + */ + + /* Re-initialize empty binlog state and start the pre-alloc thread. */ + innodb_binlog_init_state(); + start_binlog_prealloc_thread(); + + return err; +} diff --git a/storage/innobase/include/fsp_binlog.h b/storage/innobase/include/fsp_binlog.h index b09204c6d9d..c1e3566d2a3 100644 --- a/storage/innobase/include/fsp_binlog.h +++ b/storage/innobase/include/fsp_binlog.h @@ -202,6 +202,7 @@ extern std::atomic<uint64_t> binlog_cur_written_offset[2]; extern std::atomic<uint64_t> binlog_cur_end_offset[2]; extern void fsp_binlog_init(); +extern void fsp_binlog_shutdown(); extern dberr_t fsp_binlog_tablespace_close(uint64_t file_no); extern fil_space_t *fsp_binlog_open(const char *file_name, pfs_os_file_t fh, uint64_t file_no, size_t file_size, diff --git a/storage/innobase/include/innodb_binlog.h b/storage/innobase/include/innodb_binlog.h index d0a787e7cad..a627f0c5d3e 100644 --- a/storage/innobase/include/innodb_binlog.h +++ b/storage/innobase/include/innodb_binlog.h @@ -85,7 +85,7 @@ binlog_name_make(char name_buf[OS_FILE_MAX_PATH], uint64_t file_no) extern void innodb_binlog_startup_init(); extern bool innodb_binlog_init(size_t binlog_size, const char *directory); -extern void innodb_binlog_close(); +extern void innodb_binlog_close(bool shutdown); extern bool binlog_gtid_state(rpl_binlog_state_base *state, mtr_t *mtr, buf_block_t * &block, uint32_t &page_no, uint32_t &page_offset, fil_space_t *space); @@ -98,5 +98,6 @@ extern bool innobase_binlog_write_direct (IO_CACHE *cache, handler_binlog_event_group_info *binlog_info, const rpl_gtid *gtid); extern bool innodb_find_binlogs(uint64_t *out_first, uint64_t *out_last); +extern bool innodb_reset_binlogs(); #endif /* innodb_binlog_h */ diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 1747f2496c3..d406ef03a75 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -2066,7 +2066,7 @@ void innodb_shutdown() logs_empty_and_mark_files_at_shutdown(); } - innodb_binlog_close(); + innodb_binlog_close(true); os_aio_free(); fil_space_t::close_all(); /* Exit any remaining threads. */