Binlog-in-engine: Fix RESET MASTER with active XA or OOB

If there is an active transaction that is larger than the trx cache and has
binlogged OOB data, or any XA PREPAREd transaction with OOB data in the
binlog, the reference to that OOB data will be in the commit record of those
transactions.

There was a problem that RESET MASTER before the commit would remove the
binlogs containing those OOB data records. When the transactions then
committed, the commit records would point to non-existing or (if new binlog
files had time to be filled) invalid data.

Fix by giving an error on RESET MASTER if there are any active transactions
with OOB references to any binlog files, same as for PURGE.

Signed-off-by: Kristian Nielsen <knielsen@knielsen-hq.org>

mysql-test/suite/binlog_in_engine/binlog_oob_reset_purge.result

@@ -0,0 +1,28 @@
+include/reset_master.inc
+CREATE TABLE t1(a INT PRIMARY KEY, b INT, c LONGBLOB) ENGINE=InnoDB;
+*** Test that an active transaction with OOB data prevents PURGE and RESET MASTER.
+connect con1,localhost,root,,;
+BEGIN;
+INSERT INTO t1 VALUES (1, 1, REPEAT('$', 100000));
+connection default;
+INSERT INTO t1 VALUES (2, 0, 'abd');
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (3, 0, 'def');
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (4, 0, 'ghi');
+FLUSH BINARY LOGS;
+SET @old_min_slaves= @@GLOBAL.slave_connections_needed_for_purge;
+SET GLOBAL slave_connections_needed_for_purge= 0;
+PURGE BINARY LOGS TO 'binlog-000001.ibb';
+ERROR HY000: A purgeable log is in use, will not purge
+RESET MASTER;
+ERROR HY000: Cannot execute RESET MASTER as the binlog is in use by an active transaction
+connection con1;
+COMMIT;
+disconnect con1;
+connection default;
+PURGE BINARY LOGS TO 'binlog-000001.ibb';
+RESET MASTER;
+connection default;
+SET GLOBAL slave_connections_needed_for_purge= @old_min_slaves;
+DROP TABLE t1;

mysql-test/suite/binlog_in_engine/binlog_oob_reset_purge.test

@@ -0,0 +1,44 @@
+--source include/have_binlog_format_row.inc
+--source include/have_innodb_binlog.inc
+
+--source include/reset_master.inc
+
+CREATE TABLE t1(a INT PRIMARY KEY, b INT, c LONGBLOB) ENGINE=InnoDB;
+
+--echo *** Test that an active transaction with OOB data prevents PURGE and RESET MASTER.
+
+--connect con1,localhost,root,,
+# Create an active transaction large enough to generate OOB data.
+BEGIN;
+INSERT INTO t1 VALUES (1, 1, REPEAT('$', 100000));
+
+--connection default
+INSERT INTO t1 VALUES (2, 0, 'abd');
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (3, 0, 'def');
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (4, 0, 'ghi');
+FLUSH BINARY LOGS;
+
+# PURGE and RESET MASTER are blocked because con1 has pending references to
+# binlog-000000.ibb
+SET @old_min_slaves= @@GLOBAL.slave_connections_needed_for_purge;
+SET GLOBAL slave_connections_needed_for_purge= 0;
+--error ER_LOG_IN_USE
+PURGE BINARY LOGS TO 'binlog-000001.ibb';
+--error ER_BINLOG_IN_USE_TRX
+RESET MASTER;
+
+# Committing the transaction allows the PURGE and RESET to succeed.
+--connection con1
+COMMIT;
+--disconnect con1
+
+--connection default
+PURGE BINARY LOGS TO 'binlog-000001.ibb';
+RESET MASTER;
+
+--connection default
+
+SET GLOBAL slave_connections_needed_for_purge= @old_min_slaves;
+DROP TABLE t1;

mysql-test/suite/binlog_in_engine/xa.result

@@ -10,6 +10,8 @@ XA END 'a';
 XA PREPARE 'a';
 connection default;
 disconnect con1;
+RESET MASTER;
+ERROR HY000: Cannot execute RESET MASTER as the binlog is in use by an active transaction
 connect con2,localhost,root,,;
 XA COMMIT 'a';
 connect con1_roll,localhost,root,,;

mysql-test/suite/binlog_in_engine/xa.test

@@ -25,6 +25,12 @@ XA PREPARE 'a';
 --let $wait_condition= SELECT NOT EXISTS (SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE ID=$con1_id)
 --source include/wait_condition.inc
 
+# Test that we will not allow RESET MASTER while there is an active XA
+# transaction whose commit will eventually reference oob data in the current
+# binlog.
+--error ER_BINLOG_IN_USE_TRX
+RESET MASTER;
+
 --connect con2,localhost,root,,
 XA COMMIT 'a';
 

sql/share/errmsg-utf8.txt

@@ -12294,6 +12294,8 @@ ER_PSEUDO_THREAD_ID_OVERWRITE
         eng "Pseudo thread id should not be modified by the client as it will be overwritten"
 ER_BINLOG_IN_USE
 	eng "Cannot execute RESET MASTER as the binlog is in use by a connected slave or other RESET MASTER or binlog reader. Check SHOW PROCESSLIST for "Binlog Dump" commands and use KILL to stop such readers"
+ER_BINLOG_IN_USE_TRX
+	eng "Cannot execute RESET MASTER as the binlog is in use by an active transaction"
 ER_CANNOT_INIT_ENGINE_BINLOG_READER
         eng "Cannot initialize binlog reader from storage engine %s"
 ER_ENGINE_BINLOG_REQUIRES_GTID

storage/innobase/fsp/fsp_binlog.cc

@@ -1109,6 +1109,30 @@ ibb_file_oob_refs::get_oob_ref_in_use(uint64_t file_no, LF_PINS *pins)
 }
 
 
+/*
+  Check if there are any of the in-use binlog files that have refcount > 0
+  (meaning any references to oob data from active transactions).
+  Any such references must prevent a RESET MASTER, as otherwise they could
+  be committed with OOB references pointing to garbage data.
+*/
+bool
+ibb_file_oob_refs::check_any_oob_ref_in_use(uint64_t start_file_no,
+                                            uint64_t end_file_no,
+                                            LF_PINS *lf_pins)
+{
+  if (unlikely(start_file_no == ~(uint64_t)0)
+      || unlikely(end_file_no == ~(uint64_t)0))
+    return false;
+
+  for (uint64_t file_no= start_file_no; file_no <= end_file_no; ++file_no)
+  {
+    if (get_oob_ref_in_use(file_no, lf_pins))
+      return true;
+  }
+  return false;
+}
+
+
 bool
 ibb_record_in_file_hash(uint64_t file_no, uint64_t oob_ref, uint64_t xa_ref,
                          LF_PINS *in_pins)

storage/innobase/handler/innodb_binlog.cc

@@ -4213,6 +4213,14 @@ innodb_reset_binlogs()
   ut_a(lf_pins);
   ut_a(innodb_binlog_inited >= 2);
 
+  uint64_t active= active_binlog_file_no.load(std::memory_order_relaxed);
+  if (ibb_file_hash.check_any_oob_ref_in_use(earliest_binlog_file_no,
+                                             active, lf_pins))
+  {
+    my_error(ER_BINLOG_IN_USE_TRX, MYF(0));
+    return true;
+  }
+
   /* Close existing binlog tablespaces and stop the pre-alloc thread. */
   innodb_binlog_close(false);
 

storage/innobase/include/fsp_binlog.h

@@ -359,6 +359,9 @@ public:
                            uint64_t *out_oob_ref_file_no);
   /* Check if file_no needed by active, not committed transaction. */
   bool get_oob_ref_in_use(uint64_t file_no, LF_PINS *pins);
+  /* Check if _any_ file_no is needed by active, not committed transactions. */
+  bool check_any_oob_ref_in_use(uint64_t start_file_no, uint64_t end_file_no,
+                                LF_PINS *lf_pins);
 };