mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-01-16 03:52:35 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

MDEV-32894 mysqlbinlog flashback support binlog_row_image FULL_NODUP mode

Browse source 
Summary
=======
With FULL_NODUP mode, before image inclues all columns and after
image inclues only the changed columns. flashback will swap the
value of changed columns from after image to before image.
For example:
  BI: c1, c2, c3_old, c4_old
  AI: c3_new, c4_new
flashback will reconstruct the before and after images to
  BI: c1, c2, c3_new, c4_new
  AI: c3_old, c4_old

Implementation
==============
When parsing the before and after image, position and length of
the fields are collected into ai_fields and bi_fields, if it is an
Update_rows_event and the after image doesn't includes all columns.

The changed fields are swapped between bi_fields and ai_fields.
Then it recreates the before image and after image by using
bi_fields and ai_fields. nullbit will be set to 1 if the
field is NULL, otherwise nullbit will be 0.

It also optimized flashback a little bit.
- calc_row_event_length is used instead of print_verbose_one_row
- swap_buff1 and swap_buff2 are removed.
This commit is contained in:
Libing Song 2023-11-24 22:31:18 +08:00 committed by Andrew Hutchings
parent f552febe43
commit 8bf9f21855
5 changed files with 276 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
mysql-test/suite/binlog/r/flashback.result
View file

@ -706,6 +706,60 @@ DROP TABLE t1;
# MDEV-30698 Cover missing test cases for mariadb-binlog options
# --raw [and] --flashback
#
#
# < CASE 8 >
# Verify flashback works well for binlog_row_image full_nodup mode
#
CREATE TABLE t1 (
c01 TINYINT PRIMARY KEY,
c02 SMALLINT,
c03 MEDIUMINT,
c04 INT,
c05 BIGINT,
c06 CHAR(10),
c07 VARCHAR(20),
c08 TEXT,
c09 ENUM('one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight'),
c10 SET('black', 'white', 'red', 'yellow'),
c11 TIMESTAMP(3),
c12 DATETIME(3)
) ENGINE = InnoDB;
INSERT INTO t1 VALUES (1, 1, 1, 1, 1, 'A', 'A', 'A', 'one', 'black',
'2023-11-26 10:00:00.123', '2023-11-26 10:00:00');
INSERT INTO t1 VALUES (2, 1, 1, 1, 1, 'A', 'A', 'A', 'one', 'black',
'2023-11-26 10:00:00.123', '2023-11-26 10:00:00');
INSERT INTO t1 VALUES (3, 1, NULL, 1, 1, 'A', 'A', 'A', 'one', 'black',
'2023-11-26 10:00:00.123', NULL);
INSERT INTO t1 VALUES (4, 1, NULL, 1, 1, 'A', 'A', 'A', 'one', 'black',
'2023-11-26 10:00:00.123', NULL);
FLUSH BINARY LOGS;
# The update includes the cases that
# Value -> Value
# Value -> NULL
# NULL -> value
# and the changed null bits in both first and second null byte
UPDATE t1 SET c02 = NULL, c03 = 2, c09 = 'two',
c10 = NULL, c12 = '2023-11-26 11:00:00';
FLUSH BINARY LOGS;
#
# Data before flashback
#
SELECT * FROM t1;
c01 c02 c03 c04 c05 c06 c07 c08 c09 c10 c11 c12
1 NULL 2 1 1 A A A two NULL 2023-11-26 10:00:00.123 2023-11-26 11:00:00.000
2 NULL 2 1 1 A A A two NULL 2023-11-26 10:00:00.123 2023-11-26 11:00:00.000
3 NULL 2 1 1 A A A two NULL 2023-11-26 10:00:00.123 2023-11-26 11:00:00.000
4 NULL 2 1 1 A A A two NULL 2023-11-26 10:00:00.123 2023-11-26 11:00:00.000
#
# Data after flashback
#
SELECT * FROM t1;
c01 c02 c03 c04 c05 c06 c07 c08 c09 c10 c11 c12
1 1 1 1 1 A A A one black 2023-11-26 10:00:00.123 2023-11-26 10:00:00.000
2 1 1 1 1 A A A one black 2023-11-26 10:00:00.123 2023-11-26 10:00:00.000
3 1 NULL 1 1 A A A one black 2023-11-26 10:00:00.123 NULL
4 1 NULL 1 1 A A A one black 2023-11-26 10:00:00.123 NULL
DROP TABLE t1;
SET binlog_format=statement;
Warnings:
Warning 1105 MariaDB Galera and flashback do not support binlog format: STATEMENT

4
mysql-test/suite/binlog/t/flashback.combinations Normal file
View file

@ -0,0 +1,4 @@
[use_full_mode]
binlog_row_image=FULL
[use_full_nodup_mode]
binlog_row_image=FULL_NODUP

67
mysql-test/suite/binlog/t/flashback.test
View file

@ -372,6 +372,73 @@ DROP TABLE t1;
--error 1 # --raw mode and --flashback mode are not allowed
--exec $MYSQL_BINLOG -vv -B --raw --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000003> $MYSQLTEST_VARDIR/tmp/mysqlbinlog_row_flashback_8.sql
--echo #
--echo # < CASE 8 >
--echo # Verify flashback works well for binlog_row_image full_nodup mode
--echo #
CREATE TABLE t1 (
c01 TINYINT PRIMARY KEY,
c02 SMALLINT,
c03 MEDIUMINT,
c04 INT,
c05 BIGINT,
c06 CHAR(10),
c07 VARCHAR(20),
c08 TEXT,
c09 ENUM('one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight'),
c10 SET('black', 'white', 'red', 'yellow'),
c11 TIMESTAMP(3),
c12 DATETIME(3)
) ENGINE = InnoDB;
INSERT INTO t1 VALUES (1, 1, 1, 1, 1, 'A', 'A', 'A', 'one', 'black',
'2023-11-26 10:00:00.123', '2023-11-26 10:00:00');
INSERT INTO t1 VALUES (2, 1, 1, 1, 1, 'A', 'A', 'A', 'one', 'black',
'2023-11-26 10:00:00.123', '2023-11-26 10:00:00');
INSERT INTO t1 VALUES (3, 1, NULL, 1, 1, 'A', 'A', 'A', 'one', 'black',
'2023-11-26 10:00:00.123', NULL);
INSERT INTO t1 VALUES (4, 1, NULL, 1, 1, 'A', 'A', 'A', 'one', 'black',
'2023-11-26 10:00:00.123', NULL);
--let $checksum_old= `CHECKSUM TABLE t1`
FLUSH BINARY LOGS;
--echo # The update includes the cases that
--echo # Value -> Value
--echo # Value -> NULL
--echo # NULL -> value
--echo # and the changed null bits in both first and second null byte
UPDATE t1 SET c02 = NULL, c03 = 2, c09 = 'two',
c10 = NULL, c12 = '2023-11-26 11:00:00';
--let $master_file= query_get_value("SHOW MASTER STATUS", File, 1)
--let $MYSQLD_DATADIR= `select @@datadir`
FLUSH BINARY LOGS;
--echo #
--echo # Data before flashback
--echo #
SELECT * FROM t1;
--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
--exec $MYSQL_BINLOG -B -vv $MYSQLD_DATADIR/$master_file > $MYSQLTEST_VARDIR/tmp/mysqlbinlog_row_flashback.sql
--exec $MYSQL -e "SET binlog_format= ROW; source $MYSQLTEST_VARDIR/tmp/mysqlbinlog_row_flashback.sql;"
--echo #
--echo # Data after flashback
--echo #
SELECT * FROM t1;
# After flashback, t1's checksum should be same to original checksum
--let $checksum_new = `CHECKSUM TABLE t1`
if ($checksum_new != $checksum_old)
{
--die "After flashback, t1's checksum is different from the original checksum"
}
DROP TABLE t1;
--remove_file $MYSQLTEST_VARDIR/tmp/mysqlbinlog_row_flashback.sql
## Clear
SET binlog_format=statement;
--error ER_FLASHBACK_NOT_SUPPORTED

13
sql/log_event.h
View file

@ -4620,6 +4620,12 @@ public:
#endif
#ifdef MYSQL_CLIENT
struct Field_info
{
const uchar *pos; // Point to a field in before or after image
size_t length; // Length of the field.
};
/* not for direct call, each derived has its own ::print() */
virtual bool print(FILE *file, PRINT_EVENT_INFO *print_event_info)= 0;
void change_to_flashback_event(PRINT_EVENT_INFO *print_event_info, uchar *rows_buff, Log_event_type ev_type);
@ -4628,12 +4634,11 @@ public:
size_t print_verbose_one_row(IO_CACHE *file, table_def *td,
PRINT_EVENT_INFO *print_event_info,
MY_BITMAP *cols_bitmap,
const uchar *ptr, const uchar *prefix,
const my_bool no_fill_output= 0); // if no_fill_output=1, then print result is unnecessary
const uchar *ptr, const uchar *prefix);
size_t calc_row_event_length(table_def *td,
PRINT_EVENT_INFO *print_event_info,
MY_BITMAP *cols_bitmap,
const uchar *value);
const uchar *value,
Field_info *fields);
void count_row_events(PRINT_EVENT_INFO *print_event_info);
#endif

189
sql/log_event_client.cc
View file

@ -951,8 +951,7 @@ size_t
Rows_log_event::print_verbose_one_row(IO_CACHE *file, table_def *td,
PRINT_EVENT_INFO *print_event_info,
MY_BITMAP *cols_bitmap,
const uchar *value, const uchar *prefix,
const my_bool no_fill_output)
const uchar *value, const uchar *prefix)
{
const uchar *value0= value;
const uchar *null_bits= value;
@ -972,7 +971,6 @@ Rows_log_event::print_verbose_one_row(IO_CACHE *file, table_def *td,
value+= (bitmap_bits_set(cols_bitmap) + 7) / 8;
if (!no_fill_output)
if (my_b_printf(file, "%s", prefix))
goto err;
@ -985,7 +983,6 @@ Rows_log_event::print_verbose_one_row(IO_CACHE *file, table_def *td,
if (bitmap_is_set(cols_bitmap, i) == 0)
continue;
if (!no_fill_output)
if (my_b_printf(file, "### @%d=", static_cast<int>(i + 1)))
goto err;
@ -994,7 +991,6 @@ Rows_log_event::print_verbose_one_row(IO_CACHE *file, table_def *td,
size_t fsize= td->calc_field_size((uint)i, (uchar*) value);
if (value + fsize > m_rows_end)
{
if (!no_fill_output)
if (my_b_printf(file, "***Corrupted replication event was detected."
" Not printing the value***\n"))
goto err;
@ -1003,7 +999,6 @@ Rows_log_event::print_verbose_one_row(IO_CACHE *file, table_def *td,
}
}
if (!no_fill_output)
{
size= log_event_print_value(file, print_event_info, is_null? NULL: value,
td->type(i), td->field_metadata(i),
@ -1055,16 +1050,6 @@ Rows_log_event::print_verbose_one_row(IO_CACHE *file, table_def *td,
}
#endif
}
else
{
IO_CACHE tmp_cache;
open_cached_file(&tmp_cache, NULL, NULL, 0, MYF(MY_WME | MY_NABP));
size= log_event_print_value(&tmp_cache, print_event_info,
is_null ? NULL: value,
td->type(i), td->field_metadata(i),
typestr, sizeof(typestr));
close_cached_file(&tmp_cache);
}
if (!size)
goto err;
@ -1072,7 +1057,7 @@ Rows_log_event::print_verbose_one_row(IO_CACHE *file, table_def *td,
if (!is_null)
value+= size;
if (print_event_info->verbose > 1 && !no_fill_output)
if (print_event_info->verbose > 1)
{
if (my_b_write(file, (uchar*)" /* ", 4) ||
my_b_printf(file, "%s ", typestr) ||
@ -1083,7 +1068,6 @@ Rows_log_event::print_verbose_one_row(IO_CACHE *file, table_def *td,
goto err;
}
if (!no_fill_output)
if (my_b_write_byte(file, '\n'))
goto err;
@ -1095,6 +1079,42 @@ err:
return 0;
}
/**
Construct a row image according to field information.
@param[in] cols_bitmap The bitmap of row columns
@param[in] fields Values of columns used to construct a row image.
@buf[out] buf Where the row image stored.
*/
static uchar *fill_row_image(const MY_BITMAP *cols_bitmap,
const Rows_log_event::Field_info *fields,
uchar *buf)
{
uchar *null_bits= buf;
uint nulls_bit_index= 0;
size_t null_bytes= (bitmap_bits_set(cols_bitmap) + 7) / 8;
memset(null_bits, 0, null_bytes);
buf+= null_bytes;;
for (uint i= 0; i < cols_bitmap->n_bits; i ++)
{
if (!bitmap_is_set(cols_bitmap, i)) continue;
if (fields[i].pos) // Field is not null
{
memcpy(buf, fields[i].pos, fields[i].length);
buf+= fields[i].length;
}
else
{
// set the null bit
null_bits[nulls_bit_index / 8]|= (1 << (nulls_bit_index % 8));
}
nulls_bit_index++;
}
return buf;
}
/**
Exchange the SET part and WHERE part for the Update events.
@ -1111,8 +1131,9 @@ void Rows_log_event::change_to_flashback_event(PRINT_EVENT_INFO *print_event_inf
Table_map_log_event *map;
table_def *td;
DYNAMIC_ARRAY rows_arr;
uchar *swap_buff1;
uchar *rows_pos= rows_buff + m_rows_before_size;
Field_info *ai_fields= nullptr;
Field_info *bi_fields= nullptr;
if (!(map= print_event_info->m_table_map.get_table(m_table_id)) ||
!(td= map->create_table_def()))
@ -1124,66 +1145,96 @@ void Rows_log_event::change_to_flashback_event(PRINT_EVENT_INFO *print_event_inf
(void) my_init_dynamic_array(PSI_NOT_INSTRUMENTED, &rows_arr, sizeof(LEX_STRING), 8, 8, MYF(0));
if (get_general_type_code() == UPDATE_ROWS_EVENT ||
get_general_type_code() == UPDATE_ROWS_EVENT_V1)
{
if (!bitmap_is_set_all(&m_cols_ai))
{
ai_fields= (Field_info *) my_malloc(PSI_NOT_INSTRUMENTED,
sizeof(Field_info) * td->size(),
MYF(MY_ZEROFILL));
bi_fields= (Field_info *) my_malloc(PSI_NOT_INSTRUMENTED,
sizeof(Field_info) * td->size(),
MYF(MY_ZEROFILL));
if (!ai_fields || !bi_fields)
{
fprintf(stderr, "\nError: Out of memory. "
"Could not exchange to flashback event.\n");
exit(1);
}
}
}
for (uchar *value= m_rows_buf; value < m_rows_end; )
{
uchar *start_pos= value;
size_t length1= 0;
if (!(length1= print_verbose_one_row(NULL, td, print_event_info,
&m_cols, value,
(const uchar*) "", TRUE)))
if (!(length1= calc_row_event_length(td, &m_cols, value, bi_fields)))
{
fprintf(stderr, "\nError row length: %zu\n", length1);
exit(1);
}
value+= length1;
swap_buff1= (uchar *) my_malloc(PSI_NOT_INSTRUMENTED, length1, MYF(0));
if (!swap_buff1)
{
fprintf(stderr, "\nError: Out of memory. "
"Could not exchange to flashback event.\n");
exit(1);
}
memcpy(swap_buff1, start_pos, length1);
// For Update_event, we have the second part
size_t length2= 0;
if (ev_type == UPDATE_ROWS_EVENT ||
ev_type == UPDATE_ROWS_EVENT_V1)
{
if (!(length2= print_verbose_one_row(NULL, td, print_event_info,
&m_cols, value,
(const uchar*) "", TRUE)))
if (!(length2= calc_row_event_length(td, &m_cols_ai, value, ai_fields)))
{
fprintf(stderr, "\nError row length: %zu\n", length2);
exit(1);
}
value+= length2;
void *swap_buff2= my_malloc(PSI_NOT_INSTRUMENTED, length2, MYF(0));
if (!swap_buff2)
{
fprintf(stderr, "\nError: Out of memory. "
"Could not exchange to flashback event.\n");
exit(1);
}
memcpy(swap_buff2, start_pos + length1, length2); // WHERE part
/* Swap SET and WHERE part */
memcpy(start_pos, swap_buff2, length2);
memcpy(start_pos + length2, swap_buff1, length1);
my_free(swap_buff2);
}
my_free(swap_buff1);
/* Copying one row into a buff, and pushing into the array */
LEX_STRING one_row;
one_row.length= length1 + length2;
one_row.str= (char *) my_malloc(PSI_NOT_INSTRUMENTED, one_row.length, MYF(0));
if (!one_row.str)
{
fprintf(stderr, "\nError: Out of memory. "
"Could not exchange to flashback event.\n");
exit(1);
}
if (length2 != 0) // It has before and after image
{
if (!bi_fields)
{
// Both bi and ai inclues all columns, Swap WHERE and SET Part
memcpy(one_row.str, start_pos + length1, length2);
memcpy(one_row.str+length2, start_pos, length1);
}
else
{
for (uint i= 0; i < (uint)td->size(); i ++)
{
// swap after and before image columns
if (bitmap_is_set(&m_cols, i) && bitmap_is_set(&m_cols_ai, i))
{
Field_info tmp_field= bi_fields[i];
bi_fields[i]= ai_fields[i];
ai_fields[i]= tmp_field;
}
}
// Recreate the before and after image
uchar *pos= (uchar*)one_row.str;
pos= fill_row_image(&m_cols, bi_fields, pos);
pos= fill_row_image(&m_cols_ai, ai_fields, pos);
assert(pos == (uchar*)one_row.str + one_row.length);
}
}
else
{
memcpy(one_row.str, start_pos, one_row.length);
if (one_row.str == NULL || push_dynamic(&rows_arr, (uchar *) &one_row))
}
if (push_dynamic(&rows_arr, (uchar *) &one_row))
{
fprintf(stderr, "\nError: Out of memory. "
"Could not push flashback event into array.\n");
@ -1203,6 +1254,11 @@ void Rows_log_event::change_to_flashback_event(PRINT_EVENT_INFO *print_event_inf
delete_dynamic(&rows_arr);
end:
if (bi_fields)
{
my_free(bi_fields);
my_free(ai_fields);
}
delete td;
}
@ -1323,12 +1379,22 @@ static size_t calc_field_event_length(const uchar *ptr, uint type, uint meta)
return 0;
}
/**
It parses a row image and returns its length and information of
columns if 'fields' is not null.
@param[in] td Table definition of the row image
@param[in] cols_bitmap It marks the columns present in the row image
@param[in] value The row image which will be parsed
@param[out] fields Returns field information of the parsed row image.
@return length of the parsed row image if succeeds, otherwise 0 is returned.
*/
size_t
Rows_log_event::calc_row_event_length(table_def *td,
PRINT_EVENT_INFO *print_event_info,
MY_BITMAP *cols_bitmap,
const uchar *value)
const uchar *value,
Field_info *fields)
{
const uchar *value0= value;
const uchar *null_bits= value;
@ -1361,8 +1427,19 @@ Rows_log_event::calc_row_event_length(table_def *td,
if (!(size= calc_field_event_length(value, td->type(i),
td->field_metadata(i))))
return 0;
if (fields)
{
fields[i].pos= value;
fields[i].length= size;
}
value+= size;
}
else if (fields)
{
fields[i].pos= NULL;
}
null_bit_index++;
}
return value - value0;
@ -1409,8 +1486,7 @@ void Rows_log_event::count_row_events(PRINT_EVENT_INFO *print_event_info)
print_event_info->row_events++;
/* Print the first image */
if (!(length= calc_row_event_length(td, print_event_info,
&m_cols, value)))
if (!(length= calc_row_event_length(td, &m_cols, value, NULL)))
break;
value+= length;
DBUG_ASSERT(value <= m_rows_end);
@ -1418,8 +1494,7 @@ void Rows_log_event::count_row_events(PRINT_EVENT_INFO *print_event_info)
/* Print the second image (for UPDATE only) */
if (row_events == 2)
{
if (!(length= calc_row_event_length(td, print_event_info,
&m_cols_ai, value)))
if (!(length= calc_row_event_length(td, &m_cols_ai, value, NULL)))
break;
value+= length;
DBUG_ASSERT(value <= m_rows_end);