mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-01-27 01:04:19 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
mariadb/mysql-test/main/tmp_space_usage.result
Libing Song 72cc58bb71 MDEV-32014 Rename binlog cache temporary file to binlog file 
for large transaction

Description
===========
When a transaction commits, it copies the binlog events from
binlog cache to binlog file. Very large transactions
(eg. gigabytes) can stall other transactions for a long time
because the data is copied while holding LOCK_log, which blocks
other commits from binlogging.

The solution in this patch is to rename the binlog cache file to
a binlog file instead of copy, if the commiting transaction has
large binlog cache. Rename is a very fast operation, it doesn't
block other transactions a long time.

Design
======
* binlog_large_commit_threshold
  type: ulonglong
  scope: global
  dynamic: yes
  default: 128MB

  Only the binlog cache temporary files large than 128MB are
  renamed to binlog file.

* #binlog_cache_files directory
  To support rename, all binlog cache temporary files are managed
  as normal files now. `#binlog_cache_files` directory is in the same
  directory with binlog files. It is created at server startup if it doesn't
  exist. Otherwise, all files in the directory is deleted at startup.

  The temporary files are named with ML_ prefix and the memorary address
  of the binlog_cache_data object which guarantees it is unique.

* Reserve space
  To supprot rename feature, It must reserve enough space at the
  begin of the binlog cache file. The space is required for
  Format description, Gtid list, checkpoint and Gtid events when
  renaming it to a binlog file.

  Since binlog_cache_data's cache_log is directly accessed by binlog log,
  online alter and wsrep. It is not easy to update all the code. Thus
  binlog cache will not reserve space if it is not session binlog cache or
  wsrep session is enabled.

  - m_file_reserved_bytes
    Stores the bytes reserved at the begin of the cache file.
    It is initialized in write_prepare() and cleared by reset().

    The reserved file header is hide to callers. Thus there is no
    change for callers. E.g.
    - get_byte_position() still get the length of binlog data
      written to the cache, but not the file length.
    - truncate(0) will truncate the file to m_file_reserved_bytes but not 0.

  - write_prepare()
    write_prepare() is called everytime when anything is being written
    into the cache. It will call init_file_reserved_bytes() to  create
    the cache file (if it doesn't exist) and reserve suitable space if
    the data written exceeds buffer's size.

* Binlog_commit_by_rotate
  It is used to encapsulate the code for remaing a binlog cache
  tempoary file to binlog file.
  - should_commit_by_rotate()
    it is called by write_transaction_to_binlog_events() to check if
    a binlog cache should be rename to a binlog file.
  - commit()
    That is the entry to rename a binlog cache and commit the
    transaction. Both rename and commit are protected by LOCK_log,
    Thus not other transactions can write anything into the renamed
    binlog before it.

    Rename happens in a rotation. After the new binlog file is generated,
    replace_binlog_file() is called to:
    - copy data from the new binlog file to its binlog cache file.
    - write gtid event.
    - rename the binlog cache file to binlog file.

    After that the rotation will continue to succeed. Then the transaction
    is committed in a seperated group itself. Its cache file will be
    detached and cache log will be reset before calling
    trx_group_commit_with_engines(). Thus only Xid event be written.
2024-10-17 07:53:59 -06:00

240 lines
9.8 KiB
Text
Raw Blame History

call mtr.add_suppression("Write to binary log failed: .* temporary space limit reached. An incident event is written to binary log.*");
flush status;
flush global status;
show session status like "max_tmp_space_used";
Variable_name Value
Max_tmp_space_used 0
show global status like "max_tmp_space_used";
Variable_name Value
Max_tmp_space_used 0
#
# MDEV-9101 Limit size of total size of created disk temporary files
# and tables.
show session status like "tmp_space_used";
Variable_name Value
Tmp_space_used 0
show global status like "tmp_space_used";
Variable_name Value
Tmp_space_used 0
select @@global.max_tmp_session_space_usage, @@global.max_tmp_total_space_usage;
@@global.max_tmp_session_space_usage @@global.max_tmp_total_space_usage
524288 786432
select @@binlog_stmt_cache_size,@@binlog_format;
@@binlog_stmt_cache_size @@binlog_format
32768 ROW
create table t1 (a int primary key, v varchar(256), c int default(0)) engine=innodb;
insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536;
ERROR HY000: Local temporary space limit reached
set @@max_tmp_session_space_usage=1024*1024*1024;
insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536;
ERROR HY000: Global temporary space limit reached
set @@max_tmp_session_space_usage=default;
set @@binlog_format="statement";
insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536;
insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_65537_to_131072;
select count(*) from t1;
count(*)
131072
create table t2 (a int, b int) engine=innodb select seq as a, 0 as b from seq_1_to_131072;
set @@binlog_format="row";
set @@tmp_memory_table_size=32*1024;
# The following queries should fail because of tmp_space_usage
select * from t1 order by a,v;
ERROR HY000: Local temporary space limit reached
select v,count(*) from t1 group by v limit 2;
ERROR HY000: Local temporary space limit reached
update t1 set v=right(v,2);
ERROR HY000: Local temporary space limit reached
set @@binlog_format="statement";
set @@max_tmp_session_space_usage=65536;
set @@tmp_memory_table_size=0;
update t1,t2 set t1.c=t2.a, t2.b=1 where t1.a=t2.a;
ERROR HY000: Local temporary space limit reached
set @@binlog_format="row";
set @@max_tmp_session_space_usage=default;
drop table t1,t2;
#
# Check max_tmp_total_space_usage & processlist
#
set @@tmp_memory_table_size=1024*1024;
show session status like "tmp_space_used";
Variable_name Value
Tmp_space_used 0
set @@tmp_memory_table_size=0;
flush status;
# session.max_tmp_session_space_usage == global.max_tmp_session_space_usage
connect c1, localhost, root,,;
connection default;
create table t1 (a int primary key, v varchar(256), c int default(0)) engine=innodb;
create table t2 (a int primary key, v varchar(256), c int default(0)) engine=innodb;
begin;
insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_3000;
connection c1;
# information_schema.process_list.tmp_space_used == status.tmp_space_used
insert into t2 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_3000;
ERROR HY000: Global temporary space limit reached
# Test setting tmp_space_usage to 0 to disable quotas
set @save_max_tmp_total_space_usage=@@global.max_tmp_total_space_usage;
set @@global.max_tmp_total_space_usage=0;
set @@max_tmp_session_space_usage=0;
insert into t2 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_3000;
set @@global.max_tmp_total_space_usage=@save_max_tmp_total_space_usage;
set @@max_tmp_session_space_usage=0;
connection default;
insert into t1 (a,v) values(9999990,0);
commit;
select count(*) from t1;
count(*)
3001
disconnect c1;
drop table t1,t2;
#
# Test case from Elena
#
SET @@max_tmp_session_space_usage= 64*1024;
set @@binlog_format="statement";
CREATE OR REPLACE TABLE t1 (a INT, b INT);
INSERT INTO t1 SELECT seq, seq FROM seq_1_to_100000;
ALTER TABLE t1 ORDER BY a, b;
ERROR HY000: Local temporary space limit reached
DROP TABLE t1;
#
# Show that setting max tmp space too low value can stop binary logging
# if non transactional tables are used.
#
set @save_max_tmp_total_space_usage=@@global.max_tmp_total_space_usage;
SET @@global.max_tmp_total_space_usage=64*1024;
set @@binlog_format="row";
create table t1 (a int primary key, v varchar(256)) engine=myisam;
insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536;
Got one of the listed errors
show warnings;
Level Code Message
Error 42 Global temporary space limit reached
Error 1534 Writing one row to the row-based binary log failed
Warning 1196 Some non-transactional changed tables couldn't be rolled back
select count(*) <> 0 from t1;
count(*) <> 0
1
truncate table t1;
alter table t1 engine=innodb;
insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536;
Got one of the listed errors
show warnings;
Level Code Message
Error 42 Global temporary space limit reached
Error 1534 Writing one row to the row-based binary log failed
select count(*) <> 0 from t1;
count(*) <> 0
0
drop table t1;
set @@global.max_tmp_total_space_usage=@save_max_tmp_total_space_usage;
#
# Check updating non transactional table
#
SET max_tmp_session_space_usage= 64*1024;
CREATE TABLE t1 (
a varchar(1024), b varchar(1024), c varchar(1024), d varchar(1024), e varchar(1024), f varchar(1024), g varchar(1024)
) ENGINE=MyISAM;
INSERT INTO t1 VALUES
(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)),
(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)),
(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)),
(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)),
(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)),
(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)),
('x','x','x','x','x','x','x');
UPDATE t1 SET a = '' LIMIT 100;
ERROR HY000: Local temporary space limit reached
DROP TABLE t1;
#
# MDEV-33680 Server hangs or assertion fails upon SELECT with limited
# max_tmp_space_usage
#
set max_tmp_session_space_usage = 1024*1024;
select count(distinct concat(seq,repeat('x',1000))) from seq_1_to_1000;
ERROR HY000: Global temporary space limit reached
#
# MDEV-33751 Assertion `thd' failed in int
# temp_file_size_cb_func(tmp_file_tracking*, int)
#
set @save_max_tmp_total_space_usage=@@global.max_tmp_total_space_usage;
set @@global.max_tmp_total_space_usage=64*1024*1024;
set @@max_tmp_session_space_usage=1179648+65536;
select @@max_tmp_session_space_usage;
@@max_tmp_session_space_usage
1245184
set @save_aria_repair_threads=@@aria_repair_threads;
set @@aria_repair_threads=2;
set @save_max_heap_table_size=@@max_heap_table_size;
set @@max_heap_table_size=16777216;
CREATE TABLE t1 (a CHAR(255),b INT,INDEX (b));
INSERT INTO t1 SELECT SEQ,SEQ FROM seq_1_to_100000;
set @@max_tmp_session_space_usage=1179648;
SELECT * FROM t1 UNION SELECT * FROM t1;
ERROR HY000: Local temporary space limit reached
DROP TABLE t1;
set @@aria_repair_threads=@save_aria_repair_threads;
set @@max_heap_table_size=@save_max_heap_table_size;
set @@global.max_tmp_total_space_usage=@save_max_tmp_total_space_usage;
#
# MDEV-34016 Assertion `info->key_del_used == 0' failed in maria_close
# with limited tmp space
#
set @save_max_tmp_total_space_usage=@@global.max_tmp_total_space_usage;
connect c1, localhost, root,,;
CREATE TABLE t1 (a varchar(1024)) engine=aria CHARSET=latin1;
INSERT INTO t1 VALUES ('this'),('is'),('just'),('a'),('filling'),('for'),(REPEAT('a',500));
set @@global.max_tmp_total_space_usage=2*1024*1024;
SET max_tmp_session_space_usage= 1024*1024, max_heap_table_size= 4*1024*1024;
SELECT DISTINCT a, seq FROM t1 JOIN seq_1_to_600;
ERROR HY000: Got error 200 "Local temporary space limit reached" when merging index
DROP TABLE t1;
connection default;
disconnect c1;
set @@global.max_tmp_total_space_usage=@save_max_tmp_total_space_usage;
#
# MDEV-34054
# Memory leak in Window_func_runner::exec after encountering
# "temporary space limit reached" error
SET max_tmp_session_space_usage= 64*1024;
SELECT MIN(VARIABLE_VALUE) OVER (), NTILE(1) OVER (), MAX(VARIABLE_NAME) OVER () FROM information_schema.SESSION_STATUS;
ERROR HY000: Local temporary space limit reached
#
# MDEV-34060 Unexpected behavior upon reading I_S.ALL_PLUGINS under
# limited tmp space
#
connect c1, localhost, root,,;
set @@binlog_format=row;
CREATE OR REPLACE TABLE t1 (a DATETIME) ENGINE=InnoDB;
BEGIN;
INSERT INTO t1 SELECT NOW() FROM seq_1_to_6000;
SET max_tmp_session_space_usage = 64*1024;
SELECT * FROM information_schema.ALL_PLUGINS LIMIT 2;
ERROR HY000: Local temporary space limit reached
ROLLBACK;
drop table t1;
connection default;
disconnect c1;
#
# MDEV-34142 Server crashes in create_internal_tmp_table with low tmp
# space limit
#
SET MAX_TMP_SESSION_SPACE_USAGE = 128*1024, MAX_HEAP_TABLE_SIZE= 16*1024*1024;
CREATE TABLE t1 (a varchar(1024)) DEFAULT CHARACTER SET utf8mb3;
INSERT INTO t1 SELECT 'x' FROM seq_1_to_50;
SELECT * FROM t1 JOIN seq_1_to_200 INTERSECT ALL SELECT * FROM t1 JOIN seq_1_to_200;
ERROR HY000: Local temporary space limit reached
drop table t1;
#
# MDEV-34149 Corruption-like errors are produced when temporary space
# limit is reached
#
SET MAX_TMP_SESSION_SPACE_USAGE = 400*1024;
Warnings:
Warning 1292 Truncated incorrect max_tmp_session_space_usage value: '409600'
SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_VARIABLES JOIN seq_1_to_100
INTERSECT ALL
SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_VARIABLES JOIN seq_1_to_100;
ERROR HY000: Local temporary space limit reached
# End of 11.5 tests
Reference in a new issue View git blame Copy permalink