mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2026-05-13 10:30:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

Binlog-in-engine: Support for new binlog format in mysqlbinlog

Browse source 
Signed-off-by: Kristian Nielsen <knielsen@knielsen-hq.org>
This commit is contained in:
Kristian Nielsen 2025-08-14 16:35:51 +02:00
commit c69b86d468
24 changed files with 1655 additions and 326 deletions
Download patch file Download diff file Expand all files Collapse all files

2
client/CMakeLists.txt
View file

@ -70,7 +70,7 @@ TARGET_LINK_LIBRARIES(mariadb-show ${CLIENT_LIB})
MYSQL_ADD_EXECUTABLE(mariadb-plugin mysql_plugin.c)
TARGET_LINK_LIBRARIES(mariadb-plugin ${CLIENT_LIB})
MYSQL_ADD_EXECUTABLE(mariadb-binlog mysqlbinlog.cc)
MYSQL_ADD_EXECUTABLE(mariadb-binlog mysqlbinlog.cc mysqlbinlog-engine.cc)
TARGET_LINK_LIBRARIES(mariadb-binlog ${CLIENT_LIB} mysys_ssl)
MYSQL_ADD_EXECUTABLE(mariadb-admin mysqladmin.cc ../sql/password.c)

1126
client/mysqlbinlog-engine.cc Normal file
View file

File diff suppressed because it is too large Load diff

31
client/mysqlbinlog-engine.h Normal file
View file

@ -0,0 +1,31 @@
/* Copyright (c) 2025, Kristian Nielsen.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */
#include <stdint.h>
#include <atomic>
#include "handler_binlog_reader.h"
extern const char *INNODB_BINLOG_MAGIC;
extern handler_binlog_reader *get_binlog_reader_innodb();
extern bool open_engine_binlog(handler_binlog_reader *reader,
ulonglong start_position,
const char *filename, IO_CACHE *opened_cache);
/* Shared functions defined in mysqlbinlog.cc */
extern void error(const char *format, ...) ATTRIBUTE_FORMAT(printf, 1, 2);

90
client/mysqlbinlog.cc
View file

@ -43,6 +43,8 @@
#include "sql_priv.h"
#include "sql_basic_types.h"
#include <atomic>
#include "handler_binlog_reader.h"
#include "mysqlbinlog-engine.h"
#include "log_event.h"
#include "compat56.h"
#include "sql_common.h"
@ -109,7 +111,7 @@ static const char *load_groups[]=
{ "mysqlbinlog", "mariadb-binlog", "client", "client-server", "client-mariadb",
0 };
static void error(const char *format, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
void error(const char *format, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
static void warning(const char *format, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
static bool one_database=0, one_table=0, to_last_remote_log= 0, disable_log_bin= 0;
@ -202,6 +204,14 @@ enum Exit_status {
OK_EOF,
};
static enum Binlog_format {
ORIGINAL_BINLOG_FORMAT, INNODB_BINLOG_FORMAT
} binlog_format= ORIGINAL_BINLOG_FORMAT;
static handler_binlog_reader *engine_binlog_reader;
/**
Pointer to the last read Annotate_rows_log_event. Having read an
Annotate_rows event, we should not print it immediately because all
@ -689,13 +699,15 @@ static bool print_base64(PRINT_EVENT_INFO *print_event_info, Log_event *ev)
{
/*
These events must be printed in base64 format, if printed.
In the original binlog format (no --binlog-storage-engine),
base64 format requires a FD event to be safe, so if no FD
event has been printed, we give an error. Except if user
passed --short-form, because --short-form disables printing
row events.
*/
if (!print_event_info->printed_fd_event && !short_form &&
if (binlog_format == ORIGINAL_BINLOG_FORMAT &&
!print_event_info->printed_fd_event && !short_form &&
opt_base64_output_mode != BASE64_OUTPUT_DECODE_ROWS &&
opt_base64_output_mode != BASE64_OUTPUT_NEVER)
{
@ -1753,7 +1765,7 @@ static void error_or_warning(const char *format, va_list args, const char *msg)
@param format Printf-style format string, followed by printf
varargs.
*/
static void error(const char *format,...)
void error(const char *format,...)
{
va_list args;
va_start(args, format);
@ -1849,6 +1861,7 @@ static void cleanup()
delete_dynamic(&binlog_events);
delete_dynamic(&events_in_stmt);
}
delete engine_binlog_reader;
DBUG_VOID_RETURN;
}
@ -2979,7 +2992,27 @@ static Exit_status check_header(IO_CACHE* file,
error("Failed reading header; probably an empty file.");
return ERROR_STOP;
}
if (memcmp(header, BINLOG_MAGIC, sizeof(header)))
if (0 == memcmp(header, INNODB_BINLOG_MAGIC, sizeof(header)))
{
binlog_format= INNODB_BINLOG_FORMAT;
engine_binlog_reader= get_binlog_reader_innodb();
if (!engine_binlog_reader)
{
error("Out of memory setting up reader for InnoDB-implemented binlog.");
return ERROR_STOP;
}
/*
New engine-implemented binlog always does checksum verification on the
page level.
*/
opt_verify_binlog_checksum= 0;
/*
New engine-implemented binlog does not contain format description
events.
*/
goto end;
}
else if (memcmp(header, BINLOG_MAGIC, sizeof(header)))
{
error("File is not a binary log file.");
return ERROR_STOP;
@ -3089,6 +3122,7 @@ static Exit_status check_header(IO_CACHE* file,
break;
}
}
end:
my_b_seek(file, pos);
return OK_CONTINUE;
}
@ -3183,15 +3217,56 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
error("Failed reading from file.");
goto err;
}
if (binlog_format == INNODB_BINLOG_FORMAT)
{
if (open_engine_binlog(engine_binlog_reader, start_position, logname, file))
goto err;
}
for (;;)
{
char llbuff[21];
my_off_t old_off = my_b_tell(file);
int read_error;
Log_event* ev;
Log_event* ev = Log_event::read_log_event(file, &read_error,
glob_description_event,
opt_verify_binlog_checksum);
if (binlog_format == INNODB_BINLOG_FORMAT)
{
String packet;
int res= engine_binlog_reader->read_log_event(&packet, 0, MAX_MAX_ALLOWED_PACKET);
if (res == LOG_READ_EOF)
{
ev= nullptr;
read_error= 0;
}
else if (res < 0)
{
ev= nullptr;
read_error= -1;
}
else
{
const char *errmsg= nullptr;
ev= Log_event::read_log_event((uchar *)packet.ptr(), packet.length(),
&errmsg, glob_description_event,
FALSE, FALSE);
if (!ev)
{
error("Error reading event: %s", errmsg);
read_error= -1;
}
else
{
ev->register_temp_buf((uchar *)packet.release(), true);
read_error= 0;
}
}
}
else
{
ev= Log_event::read_log_event(file, &read_error,
glob_description_event,
opt_verify_binlog_checksum);
}
if (!ev)
{
/*
@ -3217,6 +3292,7 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
the size of the event, unless the event is encrypted.
*/
DBUG_ASSERT(
binlog_format == INNODB_BINLOG_FORMAT ||
((ev->get_type_code() == UNKNOWN_EVENT &&
((Unknown_log_event *) ev)->what == Unknown_log_event::ENCRYPTED)) ||
old_off + ev->data_written == my_b_tell(file));

96
include/handler_binlog_reader.h Normal file
View file

@ -0,0 +1,96 @@
#ifndef HANDLER_BINLOG_READER_INCLUDED
#define HANDLER_BINLOG_READER_INCLUDED
/* Copyright (c) 2025, Kristian Nielsen.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */
class String;
class THD;
struct slave_connection_state;
struct rpl_binlog_state_base;
/*
Class for reading a binlog implemented in an engine.
*/
class handler_binlog_reader {
public:
/*
Approximate current position (from which next call to read_binlog_data()
will need to read). Updated by the engine. Used to know which binlog files
the active dump threads are currently reading from, to avoid purging
actively used binlogs.
*/
uint64_t cur_file_no;
uint64_t cur_file_pos;
private:
/* Position and length of any remaining data in buf[]. */
uint32_t buf_data_pos;
uint32_t buf_data_remain;
/* Buffer used when reading data out via read_binlog_data(). */
static constexpr size_t BUF_SIZE= 32768;
uchar *buf;
public:
handler_binlog_reader()
: cur_file_no(~(uint64_t)0), cur_file_pos(~(uint64_t)0),
buf_data_pos(0), buf_data_remain(0)
{
buf= (uchar *)my_malloc(PSI_INSTRUMENT_ME, BUF_SIZE, MYF(0));
}
virtual ~handler_binlog_reader() {
my_free(buf);
};
virtual int read_binlog_data(uchar *buf, uint32_t len) = 0;
virtual bool data_available()= 0;
/*
Wait for data to be available to read, for kill, or for timeout.
Returns true in case of timeout reached, false otherwise.
Caller should check for kill before calling again (to avoid busy-loop).
*/
virtual bool wait_available(THD *thd, const struct timespec *abstime) = 0;
/*
This initializes the current read position to the point of the slave GTID
position passed in as POS. It is permissible to start at a position a bit
earlier in the binlog, only cost is the extra read cost of reading not
needed event data.
If position is found, must return the corresponding binlog state in the
STATE output parameter and initialize cur_file_no and cur_file_pos members.
Returns:
-1 Error
0 The requested GTID position not found, needed binlogs have been purged
1 Ok, position found and returned.
*/
virtual int init_gtid_pos(slave_connection_state *pos,
rpl_binlog_state_base *state) = 0;
/*
Initialize to a legacy-type position (filename, offset). This mostly to
support legacy SHOW BINLOG EVENTS.
*/
virtual int init_legacy_pos(const char *filename, ulonglong offset) = 0;
/*
Can be called after init_gtid_pos() or init_legacy_pos() to make the reader
stop (return EOF) at the end of the binlog file. Used for SHOW BINLOG
EVENTS, which has a file-based interface based on legacy file name.
*/
virtual void enable_single_file() = 0;
int read_log_event(String *packet, uint32_t ev_offset, size_t max_allowed);
};
#endif /* HANDLER_BINLOG_READER_INCLUDED */

68
include/my_bit.h
View file

@ -221,4 +221,72 @@ static inline uint my_find_first_bit(ulonglong n)
}
C_MODE_END
/*
The helper function my_nlz(x) calculates the number of leading zeros
in the binary representation of the number "x", either using a
built-in compiler function or a substitute trick based on the use
of the multiplication operation and a table indexed by the prefix
of the multiplication result:
Moved to mysys from ha_innodb.cc to be able to use in non-InnoDB code.
*/
#ifdef __GNUC__
#define my_nlz(x) __builtin_clzll(x)
#elif defined(_MSC_VER) && !defined(_M_CEE_PURE) && \
(defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64))
#ifndef __INTRIN_H_
#pragma warning(push, 4)
#pragma warning(disable: 4255 4668)
#include <intrin.h>
#pragma warning(pop)
#endif
__forceinline unsigned int my_nlz (unsigned long long x)
{
#if defined(_M_IX86) || defined(_M_X64)
unsigned long n;
#ifdef _M_X64
_BitScanReverse64(&n, x);
return (unsigned int) n ^ 63;
#else
unsigned long y = (unsigned long) (x >> 32);
unsigned int m = 31;
if (y == 0)
{
y = (unsigned long) x;
m = 63;
}
_BitScanReverse(&n, y);
return (unsigned int) n ^ m;
#endif
#elif defined(_M_ARM64)
return _CountLeadingZeros64(x);
#endif
}
#else
inline unsigned int my_nlz (unsigned long long x)
{
static unsigned char table [48] = {
32, 6, 5, 0, 4, 12, 0, 20,
15, 3, 11, 0, 0, 18, 25, 31,
8, 14, 2, 0, 10, 0, 0, 0,
0, 0, 0, 21, 0, 0, 19, 26,
7, 0, 13, 0, 16, 1, 22, 27,
9, 0, 17, 23, 28, 24, 29, 30
};
unsigned int y= (unsigned int) (x >> 32);
unsigned int n= 0;
if (y == 0) {
y= (unsigned int) x;
n= 32;
}
y = y | (y >> 1); // Propagate leftmost 1-bit to the right.
y = y | (y >> 2);
y = y | (y >> 4);
y = y | (y >> 8);
y = y & ~(y >> 16);
y = y * 0x3EF5D037;
return n + table[y >> 26];
}
#endif
#endif /* MY_BIT_INCLUDED */

18
storage/innobase/include/ut0compr_int.h → include/my_compr_int.h
View file

@ -15,17 +15,17 @@ this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
*****************************************************************************/
/******************************************************************//**
@file include/ut0bitop.h
Reading and writing of compressed integers.
Created 2024-10-01 Kristian Nielsen <knielsen@knielsen-hq.org>
*******************************************************/
/*
Reading and writing of compressed integers.
#ifndef INNOBASE_UT0COMPR_INT_H
#define INNOBASE_UT0COMPR_INT_H
Created 2024-10-01 Kristian Nielsen <knielsen@knielsen-hq.org>
*/
#include "ut0bitop.h"
#ifndef MY_COMPR_INT_H
#define MY_COMPR_INT_H
#include "my_bit.h"
#include <stdint.h>
#include <utility>
@ -69,4 +69,4 @@ extern unsigned char *compr_int_write(unsigned char *p, uint64_t v);
extern std::pair<uint64_t, const unsigned char *>
compr_int_read(const unsigned char *p);
#endif /* INNOBASE_UT0COMPR_INT_H */
#endif /* MY_COMPR_INT_H */

10
mysql-test/suite/binlog/r/binlog_base64_flag.result
View file

@ -6,16 +6,6 @@ select * from t1;
a
1
1
==== Test BINLOG statement w/o FD event ====
BINLOG '
SVtYRxMBAAAAKQAAADQBAAAAABAAAAAAAAAABHRlc3QAAnQxAAEDAAE=
SVtYRxcBAAAAIgAAAFYBAAAQABAAAAAAAAEAAf/+AgAAAA==
';
ERROR HY000: The BINLOG statement of type Table_map was not preceded by a format description BINLOG statement
select * from t1;
a
1
1
==== Test BINLOG statement with FD event ====
BINLOG '
ODdYRw8BAAAAZgAAAGoAAAABAAQANS4xLjIzLXJjLWRlYnVnLWxvZwAAAAAAAAAAAAAAAAAAAAAA

14
mysql-test/suite/binlog/t/binlog_base64_flag.test
View file

@ -16,20 +16,6 @@ select * from t1;
# Test that a BINLOG statement encoding a row event fails unless a
# Format_description_event as been supplied with an earlier BINLOG
# statement.
--echo ==== Test BINLOG statement w/o FD event ====
# This is a binlog statement consisting of one Table_map_log_event and
# one Write_rows_log_event. Together, they correspond to the
# following query:
# INSERT INTO TABLE test.t1 VALUES (2)
error ER_NO_FORMAT_DESCRIPTION_EVENT_BEFORE_BINLOG_STATEMENT;
BINLOG '
SVtYRxMBAAAAKQAAADQBAAAAABAAAAAAAAAABHRlc3QAAnQxAAEDAAE=
SVtYRxcBAAAAIgAAAFYBAAAQABAAAAAAAAEAAf/+AgAAAA==
';
# The above line should fail and 2 should not be in the table
select * from t1;
# Test that it works to read a Format_description_log_event with a

1
mysql-test/suite/binlog_in_engine/mysqlbinlog-master.opt Normal file
View file

@ -0,0 +1 @@
--timezone=GMT-3

79
mysql-test/suite/binlog_in_engine/mysqlbinlog.result Normal file
View file

@ -0,0 +1,79 @@
include/reset_master.inc
set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22");
CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0);
UPDATE t1 SET b=1 WHERE a=1;
DELETE FROM t1 WHERE a=2;
REPLACE INTO t1 VALUES (3, 3);
FLUSH BINARY LOGS;
FLUSH BINARY LOGS;
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/;
/*!40019 SET @@session.max_delayed_threads=0*/;
/*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/;
DELIMITER /*!*/;
use `test`/*!*/;
SET TIMESTAMP=1773142/*!*/;
SET @@session.pseudo_thread_id=999999999/*!*/;
SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1, @@session.check_constraint_checks=1, @@session.system_versioning_insert_history=0/*!*/;
SET @@session.sql_mode=1411383296/*!*/;
SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/;
/*!\C latin1 *//*!*/;
SET @@session.character_set_client=X,@@session.collation_connection=X,@@session.collation_server=X/*!*/;
SET @@session.lc_time_names=0/*!*/;
SET @@session.collation_database=DEFAULT/*!*/;
CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB
/*!*/;
START TRANSACTION
/*!*/;
# Annotate_rows:
#Q> INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0)
COMMIT/*!*/;
START TRANSACTION
/*!*/;
# Annotate_rows:
#Q> UPDATE t1 SET b=1 WHERE a=1
COMMIT/*!*/;
START TRANSACTION
/*!*/;
# Annotate_rows:
#Q> DELETE FROM t1 WHERE a=2
COMMIT/*!*/;
START TRANSACTION
/*!*/;
# Annotate_rows:
#Q> REPLACE INTO t1 VALUES (3, 3)
COMMIT/*!*/;
DELIMITER ;
# End of log file
ROLLBACK /* added by mysqlbinlog */;
/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/;
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;
SELECT * FROM t1 ORDER BY a;
a b
1 1
3 3
SELECT * FROM t1 ORDER BY a;
a b
SELECT * FROM t1 ORDER BY a;
a b
1 1
3 3
SET SESSION binlog_format= MIXED;
INSERT INTO t1 SELECT a+10, 10 FROM t1;
UPDATE t1 SET b=11 WHERE a <= 12;
SELECT * FROM t1 ORDER BY a;
a b
1 11
3 11
11 11
13 10
FLUSH BINARY LOGS;
FLUSH BINARY LOGS;
TRUNCATE t1;
SELECT * FROM t1 ORDER BY a;
a b
1 11
3 11
11 11
13 10
DROP TABLE t1;

60
mysql-test/suite/binlog_in_engine/mysqlbinlog.test Normal file
View file

@ -0,0 +1,60 @@
--source include/have_binlog_format_row.inc
--source include/have_innodb_binlog.inc
--let $datadir= `SELECT @@datadir`
--source include/reset_master.inc
set TIMESTAMP= UNIX_TIMESTAMP("1970-01-21 15:32:22");
CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
--let $gtid_start= `SELECT @@gtid_binlog_pos`
INSERT INTO t1 VALUES (1, 0), (2, 0), (3, 0);
UPDATE t1 SET b=1 WHERE a=1;
DELETE FROM t1 WHERE a=2;
REPLACE INTO t1 VALUES (3, 3);
--let $gtid_stop= `SELECT @@gtid_binlog_pos`
# Force rotate the binlog a couple of times to make sure the one we want
# to dump is written to disk.
FLUSH BINARY LOGS;
FLUSH BINARY LOGS;
--replace_regex /collation_server=[0-9]+/collation_server=X/ /character_set_client=[a-zA-Z0-9]+/character_set_client=X/ /collation_connection=[0-9]+/collation_connection=X/
--exec $MYSQL_BINLOG --short-form --base64-output=never $datadir/binlog-000000.ibb
SELECT * FROM t1 ORDER BY a;
# Test flashback.
--exec $MYSQL_BINLOG --flashback --start-position=$gtid_start --stop-position=$gtid_stop $datadir/binlog-000000.ibb > $MYSQLTEST_VARDIR/tmp/mysqlbinlog_1.txt
--exec $MYSQL --abort-source-on-error -e "source $MYSQLTEST_VARDIR/tmp/mysqlbinlog_1.txt;" test
--remove_file $MYSQLTEST_VARDIR/tmp/mysqlbinlog_1.txt
SELECT * FROM t1 ORDER BY a;
# Test normal apply.
--exec $MYSQL_BINLOG --start-position=$gtid_start --stop-position=$gtid_stop $datadir/binlog-000000.ibb > $MYSQLTEST_VARDIR/tmp/mysqlbinlog_2.txt
--exec $MYSQL -e "source $MYSQLTEST_VARDIR/tmp/mysqlbinlog_2.txt;"
--remove_file $MYSQLTEST_VARDIR/tmp/mysqlbinlog_2.txt
SELECT * FROM t1 ORDER BY a;
SET SESSION binlog_format= MIXED;
INSERT INTO t1 SELECT a+10, 10 FROM t1;
UPDATE t1 SET b=11 WHERE a <= 12;
--let $gtid_stop= `SELECT @@gtid_binlog_pos`
SELECT * FROM t1 ORDER BY a;
FLUSH BINARY LOGS;
FLUSH BINARY LOGS;
# Test across multiple binlog files.
# Need --gtid-strict-mode=0 here, as the previous replay duplicated the
# original GTIDs.
TRUNCATE t1;
--exec $MYSQL_BINLOG --start-position=$gtid_start --stop-position=$gtid_stop --gtid-strict-mode=0 $datadir/binlog-000000.ibb $datadir/binlog-000001.ibb $datadir/binlog-000002.ibb > $MYSQLTEST_VARDIR/tmp/mysqlbinlog_3.txt
--exec $MYSQL -e "source $MYSQLTEST_VARDIR/tmp/mysqlbinlog_3.txt;"
--remove_file $MYSQLTEST_VARDIR/tmp/mysqlbinlog_3.txt
SELECT * FROM t1 ORDER BY a;
DROP TABLE t1;

2
mysys/CMakeLists.txt
View file

@ -47,7 +47,7 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c my_default.c
my_rdtsc.c psi_noop.c
my_atomic_writes.c my_cpu.c my_likely.c my_largepage.c
file_logger.c my_dlerror.c crc32/crc32c.cc
my_timezone.cc
my_timezone.cc my_compr_int.cc
my_virtual_mem.c)
IF (WIN32)

20
storage/innobase/ut/ut0compr_int.cc → mysys/my_compr_int.cc
View file

@ -15,15 +15,15 @@ this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
*****************************************************************************/
/******************************************************************//**
@file include/ut0bitop.h
Reading and writing of compressed integers.
Created 2024-10-01 Kristian Nielsen <knielsen@knielsen-hq.org>
*******************************************************/
/*
Reading and writing of compressed integers.
#include "univ.i"
#include "ut0compr_int.h"
Created 2024-10-01 Kristian Nielsen <knielsen@knielsen-hq.org>
*/
#include "mysys_priv.h"
#include "my_compr_int.h"
/* Read and write compressed (up to) 64-bit integers. */
@ -33,7 +33,7 @@ Created 2024-10-01 Kristian Nielsen <knielsen@knielsen-hq.org>
*/
unsigned char *compr_int_write(unsigned char *p, uint64_t v) {
// Compute bytes needed to store the value v plus 3 bits encoding length.
uint32_t needed_bits_minus_1= 66 - nlz(v|1);
uint32_t needed_bits_minus_1= 66 - my_nlz(v|1);
uint32_t needed_bytes= (needed_bits_minus_1 >> 3) + 1;
// Compute the encoding of the length.
@ -128,7 +128,7 @@ compr_int_read(const unsigned char *p)
// that there are up to 8 scratch bytes available after the value written.
unsigned char *compr_int_write_le_unaligned_buffer(unsigned char *p, uint64_t v) {
// Compute bytes needed to store the value v plus 3 bits encoding length.
uint32_t needed_bits_minus_1= 66 - nlz(v|1);
uint32_t needed_bits_minus_1= 66 - my_nlz(v|1);
uint32_t needed_bytes= (needed_bits_minus_1 >> 3) + 1;
// Compute the encoding of the length.
@ -148,7 +148,7 @@ unsigned char *compr_int_write_le_unaligned_buffer(unsigned char *p, uint64_t v)
// Generic version without assumptions.
unsigned char *compr_int_write_generic(unsigned char *p, uint64_t v) {
// Compute bytes needed to store the value v plus 3 bits encoding length.
uint32_t needed_bits_minus_1= 66 - nlz(v|1);
uint32_t needed_bits_minus_1= 66 - my_nlz(v|1);
uint32_t needed_bytes= (needed_bits_minus_1 >> 3) + 1;
// Compute the encoding of the length.

67
sql/handler.cc
View file

@ -9223,70 +9223,3 @@ void handler::set_optimizer_costs(THD *thd)
optimizer_where_cost= thd->variables.optimizer_where_cost;
optimizer_scan_setup_cost= thd->variables.optimizer_scan_setup_cost;
}
int handler_binlog_reader::read_log_event(String *packet, uint32_t ev_offset,
size_t max_allowed)
{
uint32_t sofar= 0;
bool header_read= false;
uint32_t target_size= EVENT_LEN_OFFSET + 4;
int res;
if (unlikely(!buf))
return LOG_READ_MEM;
/*
Loop, first reading the "length" field, and then continuing to read data
until a full event has been placed in the packet.
*/
for (;;)
{
if (buf_data_remain <= 0)
{
res= read_binlog_data(buf, BUF_SIZE);
if (res <= 0)
{
res= (res < 0 ? LOG_READ_IO : LOG_READ_EOF);
goto err;
}
buf_data_pos= 0;
buf_data_remain= res;
}
uint32_t amount= std::min(target_size - sofar, buf_data_remain);
packet->append((char *)buf + buf_data_pos, amount);
buf_data_pos+= amount;
buf_data_remain-= amount;
sofar+= amount;
if (target_size == sofar)
{
if (header_read)
break;
else
{
header_read= true;
target_size= uint4korr(&((*packet)[EVENT_LEN_OFFSET + ev_offset]));
if (target_size < LOG_EVENT_MINIMAL_HEADER_LEN)
{
res= LOG_READ_BOGUS;
goto err;
}
else if (target_size > max_allowed)
{
res= LOG_READ_TOO_LARGE;
goto err;
}
/*
Note that here we rely on the fact that all valid events have more
data after the length. This way we avoid conditional for the
(useless) special case where we don't need to read anything more
after having read the first part.
*/
DBUG_ASSERT(LOG_EVENT_MINIMAL_HEADER_LEN > EVENT_LEN_OFFSET+4);
}
}
}
res= 0; /* Success */
err:
return res;
}

72
sql/handler.h
View file

@ -38,6 +38,7 @@
#include "vers_string.h"
#include "ha_handler_stats.h"
#include "optimizer_costs.h"
#include "handler_binlog_reader.h"
#include "sql_analyze_stmt.h" // for Exec_time_tracker
@ -5941,77 +5942,6 @@ struct handler_binlog_event_group_info {
};
/*
Class for reading a binlog implemented in an engine.
*/
class handler_binlog_reader {
public:
/*
Approximate current position (from which next call to read_binlog_data()
will need to read). Updated by the engine. Used to know which binlog files
the active dump threads are currently reading from, to avoid purging
actively used binlogs.
*/
uint64_t cur_file_no;
uint64_t cur_file_pos;
private:
/* Position and length of any remaining data in buf[]. */
uint32_t buf_data_pos;
uint32_t buf_data_remain;
/* Buffer used when reading data out via read_binlog_data(). */
static constexpr size_t BUF_SIZE= 32768;
uchar *buf;
public:
handler_binlog_reader()
: cur_file_no(~(uint64_t)0), cur_file_pos(~(uint64_t)0),
buf_data_pos(0), buf_data_remain(0)
{
buf= (uchar *)my_malloc(PSI_INSTRUMENT_ME, BUF_SIZE, MYF(0));
}
virtual ~handler_binlog_reader() {
my_free(buf);
};
virtual int read_binlog_data(uchar *buf, uint32_t len) = 0;
virtual bool data_available()= 0;
/*
Wait for data to be available to read, for kill, or for timeout.
Returns true in case of timeout reached, false otherwise.
Caller should check for kill before calling again (to avoid busy-loop).
*/
virtual bool wait_available(THD *thd, const struct timespec *abstime) = 0;
/*
This initializes the current read position to the point of the slave GTID
position passed in as POS. It is permissible to start at a position a bit
earlier in the binlog, only cost is the extra read cost of reading not
needed event data.
If position is found, must return the corresponding binlog state in the
STATE output parameter and initialize cur_file_no and cur_file_pos members.
Returns:
-1 Error
0 The requested GTID position not found, needed binlogs have been purged
1 Ok, position found and returned.
*/
virtual int init_gtid_pos(slave_connection_state *pos,
rpl_binlog_state_base *state) = 0;
/*
Initialize to a legacy-type position (filename, offset). This mostly to
support legacy SHOW BINLOG EVENTS.
*/
virtual int init_legacy_pos(const char *filename, ulonglong offset) = 0;
/*
Can be called after init_gtid_pos() or init_legacy_pos() to make the reader
stop (return EOF) at the end of the binlog file. Used for SHOW BINLOG
EVENTS, which has a file-based interface based on legacy file name.
*/
virtual void enable_single_file() = 0;
int read_log_event(String *packet, uint32_t ev_offset, size_t max_allowed);
};
/* Structure returned by ha_binlog_purge_info(). */
struct handler_binlog_purge_info {
/* The earliest binlog file that is in use by a dump thread. */

66
sql/log_event.cc
View file

@ -1291,6 +1291,72 @@ exit:
}
int handler_binlog_reader::read_log_event(String *packet, uint32_t ev_offset,
size_t max_allowed)
{
uint32_t sofar= 0;
bool header_read= false;
uint32_t target_size= EVENT_LEN_OFFSET + 4;
int res;
if (unlikely(!buf))
return LOG_READ_MEM;
/*
Loop, first reading the "length" field, and then continuing to read data
until a full event has been placed in the packet.
*/
for (;;)
{
if (buf_data_remain <= 0)
{
res= read_binlog_data(buf, BUF_SIZE);
if (res <= 0)
{
res= (res < 0 ? LOG_READ_IO : LOG_READ_EOF);
goto err;
}
buf_data_pos= 0;
buf_data_remain= res;
}
uint32_t amount= std::min(target_size - sofar, buf_data_remain);
packet->append((char *)buf + buf_data_pos, amount);
buf_data_pos+= amount;
buf_data_remain-= amount;
sofar+= amount;
if (target_size == sofar)
{
if (header_read)
break;
else
{
header_read= true;
target_size= uint4korr(&((*packet)[EVENT_LEN_OFFSET + ev_offset]));
if (target_size < LOG_EVENT_MINIMAL_HEADER_LEN)
{
res= LOG_READ_BOGUS;
goto err;
}
else if (target_size > max_allowed)
{
res= LOG_READ_TOO_LARGE;
goto err;
}
/*
Note that here we rely on the fact that all valid events have more
data after the length. This way we avoid conditional for the
(useless) special case where we don't need to read anything more
after having read the first part.
*/
DBUG_ASSERT(LOG_EVENT_MINIMAL_HEADER_LEN > EVENT_LEN_OFFSET+4);
}
}
}
res= 0; /* Success */
err:
return res;
}
/* 2 utility functions for the next method */

42
sql/sql_binlog.cc
View file

@ -57,21 +57,6 @@ static int check_event_type(int type, Relay_log_info *rli)
{
case START_EVENT_V3:
case FORMAT_DESCRIPTION_EVENT:
/*
We need a preliminary FD event in order to parse the FD event,
if we don't already have one.
*/
if (!fd_event)
if (!(rli->relay_log.description_event_for_exec=
new Format_description_log_event(4)))
{
my_error(ER_OUTOFMEMORY, MYF(0), 1);
return 1;
}
/* It is always allowed to execute FD events. */
return 0;
case QUERY_EVENT:
case TABLE_MAP_EVENT:
case WRITE_ROWS_EVENT_V1:
@ -83,19 +68,7 @@ static int check_event_type(int type, Relay_log_info *rli)
case PRE_GA_WRITE_ROWS_EVENT:
case PRE_GA_UPDATE_ROWS_EVENT:
case PRE_GA_DELETE_ROWS_EVENT:
/*
Row events are only allowed if a Format_description_event has
already been seen.
*/
if (fd_event)
return 0;
else
{
my_error(ER_NO_FORMAT_DESCRIPTION_EVENT_BEFORE_BINLOG_STATEMENT,
MYF(0), Log_event::get_type_str((Log_event_type)type));
return 1;
}
break;
return 0;
default:
/*
@ -328,6 +301,19 @@ void mysql_client_binlog_statement(THD* thd)
else if (bytes_decoded == 0)
break; // If no bytes where read, the string contained only whitespace
/*
Create a default format description event.
This is used to read the real Format_description_log_event, or to read
all events if there is none (as happens with --binlog-storage-engine).
*/
if (!rli->relay_log.description_event_for_exec &&
!(rli->relay_log.description_event_for_exec=
new Format_description_log_event(4)))
{
my_error(ER_OUT_OF_RESOURCES, MYF(0));
goto end;
}
DBUG_ASSERT(bytes_decoded > 0);
DBUG_ASSERT(endptr > strptr);
coded_len-= endptr - strptr;

3
storage/innobase/CMakeLists.txt
View file

@ -352,10 +352,8 @@ SET(INNOBASE_SOURCES
include/trx0undo.inl
include/trx0xa.h
include/univ.i
include/ut0bitop.h
include/ut0byte.h
include/ut0byte.inl
include/ut0compr_int.h
include/ut0counter.h
include/ut0dbg.h
include/ut0list.h
@ -425,7 +423,6 @@ SET(INNOBASE_SOURCES
trx/trx0sys.cc
trx/trx0trx.cc
trx/trx0undo.cc
ut/ut0compr_int.cc
ut/ut0dbg.cc
ut/ut0list.cc
ut/ut0mem.cc

4
storage/innobase/fsp/fsp_binlog.cc
View file

@ -22,13 +22,13 @@ InnoDB implementation of binlog.
*******************************************************/
#include <type_traits>
#include "ut0bitop.h"
#include "fsp0fsp.h"
#include "buf0flu.h"
#include "trx0trx.h"
#include "fsp_binlog.h"
#include "innodb_binlog.h"
#include "my_bit.h"
#include "rpl_gtid_base.h"
#include "log.h"
@ -1422,7 +1422,7 @@ fsp_binlog_write_rec(chunk_data_base *chunk_data, mtr_t *mtr, byte chunk_type,
/* Must be a power of two. */
ut_ad(current_binlog_state_interval == 0 ||
current_binlog_state_interval ==
(uint64_t)1 << (63 - nlz(current_binlog_state_interval)));
(uint64_t)1 << (63 - my_nlz(current_binlog_state_interval)));
if (page_no == 1 ||
0 == (page_no & (current_binlog_state_interval - 1))) {

8
storage/innobase/handler/ha_innodb.cc
View file

@ -30,7 +30,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
#define MYSQL_SERVER
#include "univ.i"
#include "ut0bitop.h"
#include "my_bit.h"
/* Include necessary SQL headers */
#include "ha_prototypes.h"
@ -2479,8 +2479,8 @@ innobase_next_autoinc(
operation. The snippet below calculates the product of two numbers
and detects an unsigned integer overflow:
*/
unsigned int m= nlz(need);
unsigned int n= nlz(step);
unsigned int m= my_nlz(need);
unsigned int n= my_nlz(step);
if (m + n <= 8 * sizeof(ulonglong) - 2) {
// The bit width of the original values is too large,
// therefore we are guaranteed to get an overflow.
@ -3704,7 +3704,7 @@ static int innodb_init_params()
if (innodb_binlog_state_interval == 0 ||
innodb_binlog_state_interval !=
(ulonglong)1 << (63 - nlz(innodb_binlog_state_interval)) ||
(ulonglong)1 << (63 - my_nlz(innodb_binlog_state_interval)) ||
innodb_binlog_state_interval % (ulonglong)ibb_page_size) {
ib::error() << "innodb_binlog_state_interval must be a "
"power-of-two multiple of the innodb binlog page size="

4
storage/innobase/handler/innodb_binlog.cc
View file

@ -29,7 +29,6 @@ InnoDB implementation of binlog.
#include <my_global.h>
#include "sql_class.h"
#include "ut0compr_int.h"
#include "innodb_binlog.h"
#include "mtr0log.h"
#include "fsp0fsp.h"
@ -38,8 +37,9 @@ InnoDB implementation of binlog.
#include "small_vector.h"
#include "mysys_err.h"
#include "my_compr_int.h"
#include "rpl_gtid_base.h"
#include "handler.h"
#include "handler_binlog_reader.h"
#include "log.h"

96
storage/innobase/include/ut0bitop.h
View file

@ -1,96 +0,0 @@
/*****************************************************************************
Copyright (c) 2024 Kristian Nielsen.
Copyright (c) 2013, 2023, MariaDB Corporation.
Copyright (c) 2000, 2020, Oracle and/or its affiliates. All Rights Reserved.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
*****************************************************************************/
/******************************************************************//**
@file include/ut0bitop.h
Utilities for fast bitwise operatons.
Created 2024-10-01 Kristian Nielsen <knielsen@knielsen-hq.org>
*******************************************************/
#ifndef INNOBASE_UT0BITOP_H
#define INNOBASE_UT0BITOP_H
/*
The helper function nlz(x) calculates the number of leading zeros
in the binary representation of the number "x", either using a
built-in compiler function or a substitute trick based on the use
of the multiplication operation and a table indexed by the prefix
of the multiplication result:
*/
#ifdef __GNUC__
#define nlz(x) __builtin_clzll(x)
#elif defined(_MSC_VER) && !defined(_M_CEE_PURE) && \
(defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64))
#ifndef __INTRIN_H_
#pragma warning(push, 4)
#pragma warning(disable: 4255 4668)
#include <intrin.h>
#pragma warning(pop)
#endif
__forceinline unsigned int nlz (unsigned long long x)
{
#if defined(_M_IX86) || defined(_M_X64)
unsigned long n;
#ifdef _M_X64
_BitScanReverse64(&n, x);
return (unsigned int) n ^ 63;
#else
unsigned long y = (unsigned long) (x >> 32);
unsigned int m = 31;
if (y == 0)
{
y = (unsigned long) x;
m = 63;
}
_BitScanReverse(&n, y);
return (unsigned int) n ^ m;
#endif
#elif defined(_M_ARM64)
return _CountLeadingZeros64(x);
#endif
}
#else
inline unsigned int nlz (unsigned long long x)
{
static unsigned char table [48] = {
32, 6, 5, 0, 4, 12, 0, 20,
15, 3, 11, 0, 0, 18, 25, 31,
8, 14, 2, 0, 10, 0, 0, 0,
0, 0, 0, 21, 0, 0, 19, 26,
7, 0, 13, 0, 16, 1, 22, 27,
9, 0, 17, 23, 28, 24, 29, 30
};
unsigned int y= (unsigned int) (x >> 32);
unsigned int n= 0;
if (y == 0) {
y= (unsigned int) x;
n= 32;
}
y = y | (y >> 1); // Propagate leftmost 1-bit to the right.
y = y | (y >> 2);
y = y | (y >> 4);
y = y | (y >> 8);
y = y & ~(y >> 16);
y = y * 0x3EF5D037;
return n + table[y >> 26];
}
#endif
#endif /* INNOBASE_UT0BITOP_H */

2
support-files/magic
View file

@ -26,5 +26,5 @@
>3 byte x Version %d
0 belong&0xffffff00 0xfefe0c00 MariaDB GTID index file
>3 byte x Version %d
0 belong&0xffffff00 0xfefe0d00 MariaDB InnoDB new binlog format
0 belong&0xffffff00 0xfefe0d01 MariaDB InnoDB new binlog format
>3 byte x Version %d