From 87ee1e75bcb783386b1f717a02bcbb6983fd0f55 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 13 Dec 2024 15:41:59 +0200 Subject: [PATCH] MDEV-35643 Add support for MySQL 8.0 binlog events MDEV-29533 Crash when MariaDB is replica of MySQL 8.0 MySQL 8.0 has added the following new events in the MySQL binary log PARTIAL_UPDATE_ROWS_EVENT TRANSACTION_PAYLOAD_EVENT HEARTBEAT_LOG_EVENT_V2 - PARTIAL_UPDATE_ROWS_EVENT is used by MySQL to generate update statements using JSON_SET, JSON_REPLACE and JSON_REMOVE to make update of JSON columns more efficient. These events can be disabled by setting 'binlog-row-value-options=""' - TRANSACTION_PAYLOAD_EVENT is used by MySQL to signal that a row event is compressed. It an be disably by setting 'binlog_transaction_compression=0'. - HEARTBEAT_LOG_EVENT_V2 is written to the binary log many times per seconds. It can be ignored by the server. What this patch does: - If PARTIAL_UPDATE_ROWS_EVENT or TRANSACTION_PAYLOAD_EVENT is found, the server will stop with an error message of how to disable the MySQL server to generate such events. - HEARTBEAT_LOG_EVENT_V2 events are ignored. - mariadb-binlog will write the name of the new events. - mariadb-binlog will stop if PARTIAL_UPDATE_ROWS_EVENT or TRANSACTION_PAYLOAD_EVENT is found, unless --force is given. - Fixes a crash in mariadb-binlog if a character set unknown to MariaDB is found. (MDEV-29533) From Kristian Nielsen: - Add test case for MySQL 8.0 to MariaDB replication and fixed a a small typo in post_header_len initialization. Reviewer: knielsen@mariadb.org --- .../std_data/mdev35643_mysql_80_binlog.000001 | Bin 0 -> 4438 bytes .../suite/rpl/r/rpl_from_mysql80.result | 52 ++++++++ mysql-test/suite/rpl/t/rpl_from_mysql80.test | 121 ++++++++++++++++++ sql/log_event.cc | 29 ++++- sql/log_event.h | 8 ++ sql/log_event_client.cc | 7 +- sql/sql_repl.cc | 8 -- 7 files changed, 213 insertions(+), 12 deletions(-) create mode 100644 mysql-test/std_data/mdev35643_mysql_80_binlog.000001 create mode 100644 mysql-test/suite/rpl/r/rpl_from_mysql80.result create mode 100644 mysql-test/suite/rpl/t/rpl_from_mysql80.test diff --git a/mysql-test/std_data/mdev35643_mysql_80_binlog.000001 b/mysql-test/std_data/mdev35643_mysql_80_binlog.000001 new file mode 100644 index 0000000000000000000000000000000000000000..2953c2a277cb81bca21868837725a4ca807d9f1b GIT binary patch literal 4438 zcmeHLdsLH07N2|x7();h6%d4vNO=#ezjn{*oXq!_d+%@N z{$}p|F8oW9oW8d>Cdq^b0LX^@J=g(Y06v$;UC47wTr0^;!b7jTAQ)3%444`K2E0iC zV=j7SGMNOtu~;l;XE1XiU^xQX&u{Mh1)0o8Guyyk1A9K~nznA%qY@u^wAv1N{XHG>=mF#LwbfH zk->!SFaevUSeu~$Gn5(~FmoK30z4EDF7OixNRglaN&%^GCmqFPkWfU1g$ITBg|8ul z1#4VM2^w-G6Ua!v@POrh;g0S+kA+T7q(B%LBoufBrKd}m`IAWE7a$UZldA+GGF7Zd zPIs63{92|*!T{x+b|Ej_uy<&TV&hS_h6HO&!ti%_M|30vL$#$)4|=CXfh5e+Jklm2 z0XmIc_7}iqe|FaYZwSg1E$A)SGw_so*q#O$Sd&n=13H~6sCLJ`SF{cw*GB3TaPAL+FT{!`i3e6KWM{Wo}5UJ0d1@jbM^1fvusCQadxC zVW%9kWANULNCbyTMC`l+^8j;@hrk7%}ngSW!2CF2@`({05)2={th;bJj7h@f;^VP$}3 zq+Fehp%#yXmB>uD6+%5T%n;=w17xHm%N{E(IUx%jLuuGTrS<-3QTqWT)-iMzi?!0< z-v|nwQoH^25}ePr(NR9DzxAjM!8xMP@Pcd(6EwaxrQ0z|?1N?VOTNKL6`G!A#q& z(}~i$@Ce+lS6{vO<8sy}AP1?S-gDHt*5TAQAB386dQU9-b|8r87_u5j&UwFo#A1JI z==6@B!N!ksvrC({O5EM%SO#oVy5yw|dG2;`apSIjy14btkrX@6JICMkif;U|wkBpx zvOFTlZ;@L~QKoGVcbB-a_`3g`$n?=A@{nEg2InRI5SI7!%<=ZStT(Eg2j8&BQqAAr z7G-y7xHz95SJ~J7Tj#ADuQS!sDS?AUX`XS@j_evv-!_+XsiCv=Ex)S1o|NHr)~(Od z@0}=LHN2<(P|n3G{0}diceOaMul_VBSpDlm!frTqV0+H-{$<`#3fo#|rDxOAfig)- zxxwuA3xAPbdGL69SxDU8%7?$vxw*OCholu%kMBLKR4wVa8kOWZ(!Dmy56aue zazgpFM_XdT_h3 zDtLb3(Va^5r?Z@gI0aKxfi_!??#NO{U$YpRDK}6BSQhlvmC;pq9o3faN5t{i{bRl$ zuJ6Fe#@pAbdJd{A&7NF8m4CC&HeOK5U4H0Tvw3CVmRX{9+DzXt()C^ce&%`R`EeJf z&pGhRdlJX)0++WHbS<1 zi@lXRn$ON^Jk8${-RUhXOz0d{nsl(TI&)&q-IIBJasG)caW#ASzS4hQ5GyU)nr3vc zp2_E08QpKrRYxj~?spN~k{-G77bV|!osn%fD!xo>P_tu-EM-ggG&?C{`4WD`W|?bD zvw_T}txI-;2fPbC*A;I#Qkp%+cIy0xI+E|h->kI0X>fDe%}ry+QWr7uS66@aAmL|` zsp;;Ngm1@ZyU*@5m*3Kh#pP3fAAm265L65rY6pM$P7{kP!({(B$4x zUg3NT^)vIZccXHA7QW2du}tS*I23;d;q=ae9UO#>Hc^SCtuys1t&^8hQgn#*&@u{> z^#Wu61!9nJl^|S%>d{bA;QhELXkU<=j2K{Af*JPgsKEiu7i`)=`NZtTCnaoZNzJJ(^4AG zJIT@`yoquJ`Zhr$D59wtJyE71{1^irs4n~Ub~<`Q(B8uMCBp4BptOkypGxhs9_32| zS7ahg9EGS33x=2g6rc+%PpO*00}%8lGzI}nTbH3xTrS|4ISW6>Q)VDt%7OsIxTdhZ zhw3*v_=&K9KyzWYgq>{#*tURuhY(8=7>N0^23nr~xW)cK!T%ct43r9BP@veVMPWd< z<4Sh3Kc-1TrT>G!ysgrmdily8$1YEs8V*>v;J@V<=$!|9Bobinlog_version)); DBUG_DUMP_EVENT_BUF(buf, event_len); + *error= 0; /* Check the integrity; This is needed because handle_slave_io() doesn't check if packet is of proper length. @@ -1237,6 +1241,7 @@ Log_event* Log_event::read_log_event(const uchar *buf, uint event_len, case ANONYMOUS_GTID_LOG_EVENT: case PREVIOUS_GTIDS_LOG_EVENT: case TRANSACTION_CONTEXT_EVENT: + case HEARTBEAT_LOG_EVENT_V2: // MySQL 8.0 case VIEW_CHANGE_EVENT: ev= new Ignorable_log_event(buf, fdle, get_type_str((Log_event_type) event_type)); @@ -1261,6 +1266,21 @@ Log_event* Log_event::read_log_event(const uchar *buf, uint event_len, case START_ENCRYPTION_EVENT: ev= new Start_encryption_log_event(buf, event_len, fdle); break; + case TRANSACTION_PAYLOAD_EVENT: // MySQL 8.0 + *error= + "Found incompatible MySQL 8.0 TRANSACTION_PAYLOAD_EVENT event. " + "You can avoid this event by specifying " + "'binlog_transaction_compression=0' in the MySQL server"; + ev= NULL; + break; + case PARTIAL_UPDATE_ROWS_EVENT: // MySQL 8.0 + *error= + "Found incompatible MySQL 8.0 PARTIAL_UPDATE_ROWS_EVENT event. " + "You can avoid this event by specifying " + "'binlog-row-value-options=\"\"' in the MySQL server"; + ev= NULL; + break; + default: DBUG_PRINT("error",("Unknown event code: %d", (uchar) buf[EVENT_TYPE_OFFSET])); @@ -1303,12 +1323,14 @@ exit: #ifdef MYSQL_CLIENT if (!force_opt) /* then mysqlbinlog dies */ { - *error= "Found invalid event in binary log"; + if (!*error) + *error= "Found invalid event in binary log"; DBUG_RETURN(0); } ev= new Unknown_log_event(buf, fdle); #else - *error= "Found invalid event in binary log"; + if (!*error) + *error= "Found invalid event in binary log"; DBUG_RETURN(0); #endif } @@ -2126,6 +2148,9 @@ Format_description_log_event(uint8 binlog_ver, const char* server_ver) post_header_len[TRANSACTION_CONTEXT_EVENT-1]= 0; post_header_len[VIEW_CHANGE_EVENT-1]= 0; post_header_len[XA_PREPARE_LOG_EVENT-1]= 0; + post_header_len[PARTIAL_UPDATE_ROWS_EVENT-1]= ROWS_HEADER_LEN_V2; + post_header_len[TRANSACTION_PAYLOAD_EVENT-1]= ROWS_HEADER_LEN_V2; + post_header_len[HEARTBEAT_LOG_EVENT_V2-1]= ROWS_HEADER_LEN_V2; post_header_len[WRITE_ROWS_EVENT-1]= ROWS_HEADER_LEN_V2; post_header_len[UPDATE_ROWS_EVENT-1]= ROWS_HEADER_LEN_V2; post_header_len[DELETE_ROWS_EVENT-1]= ROWS_HEADER_LEN_V2; diff --git a/sql/log_event.h b/sql/log_event.h index e4bda624e0c..03902cc286e 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -678,6 +678,14 @@ enum Log_event_type /* not ignored */ XA_PREPARE_LOG_EVENT= 38, + /** + Extension of UPDATE_ROWS_EVENT, allowing partial values according + to binlog_row_value_options. + */ + PARTIAL_UPDATE_ROWS_EVENT = 39, + TRANSACTION_PAYLOAD_EVENT = 40, + HEARTBEAT_LOG_EVENT_V2 = 41, + /* Add new events here - right above this comment! Existing events (except ENUM_END_EVENT) should never change their numbers diff --git a/sql/log_event_client.cc b/sql/log_event_client.cc index 0b431dc5bed..6e62734ae9a 100644 --- a/sql/log_event_client.cc +++ b/sql/log_event_client.cc @@ -1970,7 +1970,7 @@ bool Query_log_event::print_query_header(IO_CACHE* file, print_event_info->auto_increment_offset= auto_increment_offset; } - /* TODO: print the catalog when we feature SET CATALOG */ + /* TODO: print the catalog when we feature USE CATALOG */ if (likely(charset_inited) && (unlikely(!print_event_info->charset_inited || @@ -1984,12 +1984,15 @@ bool Query_log_event::print_query_header(IO_CACHE* file, cs_info->cs_name.str, print_event_info->delimiter)) goto err; } + else if (my_b_printf(file, "# Ignored (Unknown charset) ")) + goto err; + if (my_b_printf(file,"SET " "@@session.character_set_client=%s," "@@session.collation_connection=%d," "@@session.collation_server=%d" "%s\n", - cs_info->cs_name.str, + cs_info ? cs_info->cs_name.str : "Unknown", uint2korr(charset+2), uint2korr(charset+4), print_event_info->delimiter)) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 418e6aafab1..e33ad987fe8 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -434,7 +434,6 @@ static int send_file(THD *thd) Internal to mysql_binlog_send() routine that recalculates checksum for 1. FD event (asserted) that needs additional arrangement prior sending to slave. 2. Start_encryption_log_event whose Ignored flag is set -TODO DBUG_ASSERT can be removed if this function is used for more general cases */ inline void fix_checksum(enum_binlog_checksum_alg checksum_alg, String *packet, @@ -446,13 +445,6 @@ inline void fix_checksum(enum_binlog_checksum_alg checksum_alg, String *packet, /* recalculate the crc for this event */ uint data_len = uint4korr(packet->ptr() + ev_offset + EVENT_LEN_OFFSET); ha_checksum crc; - DBUG_ASSERT((data_len == - LOG_EVENT_MINIMAL_HEADER_LEN + FORMAT_DESCRIPTION_HEADER_LEN + - BINLOG_CHECKSUM_ALG_DESC_LEN + BINLOG_CHECKSUM_LEN) || - (data_len == - LOG_EVENT_MINIMAL_HEADER_LEN + BINLOG_CRYPTO_SCHEME_LENGTH + - BINLOG_KEY_VERSION_LENGTH + BINLOG_NONCE_LENGTH + - BINLOG_CHECKSUM_LEN)); crc= my_checksum(0, (uchar *)packet->ptr() + ev_offset, data_len - BINLOG_CHECKSUM_LEN); int4store(packet->ptr() + ev_offset + data_len - BINLOG_CHECKSUM_LEN, crc);