mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-01-15 19:42:28 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

MDEV-35469 Heap tables are calling mallocs to often

Browse source 
Heap tables are allocated blocks to store rows according to
my_default_record_cache (mapped to the server global variable
 read_buffer_size).
This causes performance issues when the record length is big
(> 1000 bytes) and the my_default_record_cache is small.

Changed to instead split the default heap allocation to 1/16 of the
allowed space and not use my_default_record_cache anymore when creating
the heap. The allocation is also aligned to be just under a power of 2.

For some test that I have been running, which was using record length=633,
the speed of the query doubled thanks to this change.

Other things:
- Fixed calculation of max_records passed to hp_create() to take
  into account padding between records.
- Updated calculation of memory needed by heap tables. Before we
  did not take into account internal structures needed to access rows.
- Changed block sized for memory_table from 1 to 16384 to get less
  fragmentation. This also avoids a problem where we need 1K
  to manage index and row storage which was not counted for before.
- Moved heap memory usage to a separate test for 32 bit.
- Allocate all data blocks in heap in powers of 2. Change reported
  memory usage for heap to reflect this.

Reviewed-by: Sergei Golubchik <serg@mariadb.org>
This commit is contained in:
Monty 2024-11-21 12:28:57 +02:00
parent f20ee931d8
commit 52c29f3bdc
42 changed files with 316 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
include/heap.h
View file

@ -105,6 +105,7 @@ typedef struct st_heap_block
uint recbuffer; /* Length of one saved record */
ulong records_in_block; /* Records in one heap-block */
ulong last_allocated; /* number of records there is allocated space for */
size_t alloc_size; /* Allocate blocks of this size */
} HP_BLOCK;
struct st_heap_info; /* For reference */

2
include/my_global.h
View file

@ -670,7 +670,7 @@ typedef SOCKET_SIZE_TYPE size_socket;
How much overhead does malloc have. The code often allocates
something like 1024-MALLOC_OVERHEAD bytes
*/
#define MALLOC_OVERHEAD 8
#define MALLOC_OVERHEAD (8+24)
/* get memory in huncs */
#define ONCE_ALLOC_INIT (uint) 4096

4
mysql-test/main/count_distinct.result
View file

@ -97,6 +97,8 @@ drop view v1;
create table t1 (user_id char(64) character set utf8);
insert t1 values(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17);
set @@tmp_table_size = 1024;
Warnings:
Warning 1292 Truncated incorrect tmp_table_size value: '1024'
select count(distinct user_id) from t1;
count(distinct user_id)
17
@ -126,6 +128,8 @@ insert into t1 values
( 2 , 13 ),
( 3 , 14 );
set @@tmp_table_size=1024;
Warnings:
Warning 1292 Truncated incorrect tmp_table_size value: '1024'
select count(distinct a) from t1;
count(distinct a)
10

2
mysql-test/main/derived_opt.result
View file

@ -310,7 +310,7 @@ a char(2) NOT NULL DEFAULT '',
PRIMARY KEY (a)
) ENGINE=MyISAM;
INSERT INTO t4 VALUES ('CD');
set @@tmp_table_size=8192;
set @@tmp_table_size=16384;
EXPLAIN
SELECT * FROM t3 AS tx JOIN t2 AS ty ON (tx.pk = ty.pk)
WHERE

2
mysql-test/main/derived_opt.test
View file

@ -245,7 +245,7 @@ CREATE TABLE t4 (
) ENGINE=MyISAM;
INSERT INTO t4 VALUES ('CD');
set @@tmp_table_size=8192;
set @@tmp_table_size=16384;
--replace_column 9 #
EXPLAIN

2
mysql-test/main/derived_view.result
View file

@ -2183,6 +2183,8 @@ INSERT INTO t3 VALUES ('Miami');
SET @save_optimizer_switch=@@optimizer_switch;
SET optimizer_switch = 'derived_with_keys=on';
SET @@tmp_table_size=1024*4;
Warnings:
Warning 1292 Truncated incorrect tmp_table_size value: '4096'
explain SELECT * FROM (SELECT t1.* FROM t1, t2) AS t JOIN t3 ON t3.a = t.b;
id select_type table type possible_keys key key_len ref rows Extra
1 SIMPLE t3 system NULL NULL NULL NULL 1

2
mysql-test/main/distinct.result
View file

@ -1005,6 +1005,8 @@ DROP TABLE t1;
#
SET @tmp_table_size_save= @@tmp_table_size;
SET @@tmp_table_size= 1024;
Warnings:
Warning 1292 Truncated incorrect tmp_table_size value: '1024'
CREATE TABLE t1 (a INT);
INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8);
INSERT INTO t1 SELECT a+8 FROM t1;

17
mysql-test/main/error_simulation.result
View file

@ -1,15 +1,12 @@
CREATE TABLE t1 (
a varchar(32) character set utf8 collate utf8_bin NOT NULL,
b varchar(32) character set utf8 collate utf8_bin NOT NULL )
a varchar(128) character set utf8 collate utf8_bin NOT NULL,
b varchar(128) character set utf8 collate utf8_bin NOT NULL )
ENGINE=MyISAM DEFAULT CHARSET=utf8;
INSERT INTO t1 VALUES
('AAAAAAAAAA','AAAAAAAAAA'), ('AAAAAAAAAB','AAAAAAAAAB '),
('AAAAAAAAAB','AAAAAAAAAB'), ('AAAAAAAAAC','AAAAAAAAAC'),
('AAAAAAAAAD','AAAAAAAAAD'), ('AAAAAAAAAE','AAAAAAAAAE'),
('AAAAAAAAAF','AAAAAAAAAF'), ('AAAAAAAAAG','AAAAAAAAAG'),
('AAAAAAAAAH','AAAAAAAAAH'), ('AAAAAAAAAI','AAAAAAAAAI'),
('AAAAAAAAAJ','AAAAAAAAAJ'), ('AAAAAAAAAK','AAAAAAAAAK');
set tmp_table_size=1024;
INSERT INTO t1
select concat(repeat("A", 50),char(32+mod(seq,31)),char(32+mod(seq,29))),
concat(repeat("A", 50),char(32+mod(seq,31)),char(32+mod(seq,29)))
from seq_1_to_128;
set tmp_table_size=16384;
SET @saved_dbug = @@SESSION.debug_dbug;
set session debug_dbug="+d,raise_error";
SELECT MAX(a) FROM t1 GROUP BY a,b;

19
mysql-test/main/error_simulation.test
View file

@ -1,23 +1,20 @@
--source include/have_debug.inc
--source include/not_embedded.inc
--source include/have_sequence.inc
#
# Bug #28499: crash for grouping query when tmp_table_size is too small
#
CREATE TABLE t1 (
a varchar(32) character set utf8 collate utf8_bin NOT NULL,
b varchar(32) character set utf8 collate utf8_bin NOT NULL )
a varchar(128) character set utf8 collate utf8_bin NOT NULL,
b varchar(128) character set utf8 collate utf8_bin NOT NULL )
ENGINE=MyISAM DEFAULT CHARSET=utf8;
INSERT INTO t1 VALUES
('AAAAAAAAAA','AAAAAAAAAA'), ('AAAAAAAAAB','AAAAAAAAAB '),
('AAAAAAAAAB','AAAAAAAAAB'), ('AAAAAAAAAC','AAAAAAAAAC'),
('AAAAAAAAAD','AAAAAAAAAD'), ('AAAAAAAAAE','AAAAAAAAAE'),
('AAAAAAAAAF','AAAAAAAAAF'), ('AAAAAAAAAG','AAAAAAAAAG'),
('AAAAAAAAAH','AAAAAAAAAH'), ('AAAAAAAAAI','AAAAAAAAAI'),
('AAAAAAAAAJ','AAAAAAAAAJ'), ('AAAAAAAAAK','AAAAAAAAAK');
INSERT INTO t1
select concat(repeat("A", 50),char(32+mod(seq,31)),char(32+mod(seq,29))),
concat(repeat("A", 50),char(32+mod(seq,31)),char(32+mod(seq,29)))
from seq_1_to_128;
set tmp_table_size=1024;
set tmp_table_size=16384;
# Set debug flag so an error is returned when
# tmp table in query is converted from heap to myisam

2
mysql-test/main/order_by.result
View file

@ -4331,6 +4331,8 @@ CREATE TABLE t1(a VARCHAR(1027), b INT);
INSERT INTO t1 SELECT seq, seq from seq_1_to_34;
SET @save_tmp_memory_table_size= @@tmp_memory_table_size;
SET tmp_memory_table_size= 1056*2;
Warnings:
Warning 1292 Truncated incorrect tmp_memory_table_size value: '2112'
SELECT COUNT(DISTINCT a) FROM t1;
COUNT(DISTINCT a)
34

6
mysql-test/main/set_operation.result
View file

@ -1085,10 +1085,8 @@ select 1 union all select 2 union all select 3 union select 4;
3
4
# test with limited resource
set @@max_heap_table_size= 1024;
Warnings:
Warning 1292 Truncated incorrect max_heap_table_size value: '1024'
set @@tmp_table_size= 1024;
set @@max_heap_table_size= 16384;
set @@tmp_table_size= 16384;
create table t1 (a int, b int);
insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(0,0);
insert into t1 select * from t1;

4
mysql-test/main/set_operation.test
View file

@ -457,8 +457,8 @@ select 1 union all select 2 union all select 3 union select 4;
--echo # test with limited resource
set @@max_heap_table_size= 1024;
set @@tmp_table_size= 1024;
set @@max_heap_table_size= 16384;
set @@tmp_table_size= 16384;
create table t1 (a int, b int);
insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(0,0);

2
mysql-test/main/union.result
View file

@ -874,7 +874,7 @@ insert into t1 select * from t2;
insert into t2 select * from t1;
insert into t1 select * from t2;
insert into t2 select * from t1;
set local tmp_table_size=1024;
set local tmp_table_size=16384;
select count(*) from (select * from t1 union all select * from t2 order by 1) b;
count(*)
21

2
mysql-test/main/union.test
View file

@ -503,7 +503,7 @@ insert into t1 select * from t2;
insert into t2 select * from t1;
insert into t1 select * from t2;
insert into t2 select * from t1;
set local tmp_table_size=1024;
set local tmp_table_size=16384;
select count(*) from (select * from t1 union all select * from t2 order by 1) b;
select count(*) from t1;
select count(*) from t2;

4
mysql-test/main/update.result
View file

@ -491,7 +491,9 @@ a quux
DROP TABLE t1;
connect con1,localhost,root,,test;
connection con1;
set tmp_table_size=1024;
set tmp_table_size=2048;
Warnings:
Warning 1292 Truncated incorrect tmp_table_size value: '2048'
create table t1 (id int, a int, key idx(a));
create table t2 (id int unsigned not null auto_increment primary key, a int);
insert into t2(a) values(1),(2),(3),(4),(5),(6),(7),(8);

2
mysql-test/main/update.test
View file

@ -397,7 +397,7 @@ DROP TABLE t1;
connect (con1,localhost,root,,test);
connection con1;
set tmp_table_size=1024;
set tmp_table_size=2048;
# Create the test tables
create table t1 (id int, a int, key idx(a));

2
mysql-test/main/variables.result
View file

@ -547,7 +547,7 @@ set global table_open_cache=100;
set default_storage_engine=myisam;
set global thread_cache_size=100;
set timestamp=1, timestamp=default;
set tmp_table_size=1024;
set tmp_table_size=16384;
set tx_isolation="READ-COMMITTED";
set wait_timeout=100;
set log_warnings=1;

2
mysql-test/main/variables.test
View file

@ -340,7 +340,7 @@ set global table_open_cache=100;
set default_storage_engine=myisam;
set global thread_cache_size=100;
set timestamp=1, timestamp=default;
set tmp_table_size=1024;
set tmp_table_size=16384;
set tx_isolation="READ-COMMITTED";
set wait_timeout=100;
set log_warnings=1;

52
mysql-test/suite/heap/heap.result
View file

@ -798,58 +798,6 @@ id select_type table type possible_keys key key_len ref rows Extra
3 DERIVED t1 range PRIMARY PRIMARY 4 NULL 1 Using index condition; Using where
DROP TABLE t1,t2,h1;
DROP VIEW v1;
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
1600 2400
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=10000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
16000 24000
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=3000 max_rows=3000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
48000 72000
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
24000 36000
drop table t1;
create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
insert into t1 select rand(100000000);
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1 limit 488;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
16000 24000
insert into t1 select rand(100000000) from t1 limit 1;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
33024 49024
insert into t1 select rand(100000000) from t1 limit 1000;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
49024 73024
insert into t1 select rand(100000000) from t1;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
81024 121024
drop table t1;
CREATE TABLE t1 (id INT);
INSERT INTO t1 VALUES (1);
INSERT INTO t1 VALUES (2);

52
mysql-test/suite/heap/heap.test
View file

@ -567,58 +567,6 @@ DROP TABLE t1,t2,h1;
DROP VIEW v1;
# End of 5.1 tests
#
# Show that MIN_ROWS and MAX_ROWS have an effect on how data_length
# and index_length are allocated.
# Result is different for 32 / 64 bit machines as pointer lengths are different
#
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
insert into t1 values(1);
--replace_result 800 1600 1200 2400
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=10000;
insert into t1 values(1);
--replace_result 8000 16000 12000 24000
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=3000 max_rows=3000;
insert into t1 values(1);
--replace_result 24000 48000 36000 72000
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
insert into t1 values(1);
--replace_result 12000 24000 18000 36000
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;
create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
insert into t1 select rand(100000000);
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1 limit 488;
--replace_result 8000 16000 12000 24000
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
insert into t1 select rand(100000000) from t1 limit 1;
--replace_result 16512 33024 24512 49024
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
insert into t1 select rand(100000000) from t1 limit 1000;
--replace_result 24512 49024 36512 73024
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
insert into t1 select rand(100000000) from t1;
--replace_result 40512 81024 60512 121024
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;
#
# MDEV-5905 Creating tmp. memory table kills the server
#

40
mysql-test/suite/heap/heap_memory_used,32bit.rdiff Normal file
View file

@ -0,0 +1,40 @@
--- suite/heap/heap_memory_used.result
+++ suite/heap/heap_memory_used.reject
@@ -17,13 +17,13 @@
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
-65504 131040
+32736 65504
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
-16352 32736
+16352 16352
drop table t1;
create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
insert into t1 select rand(100000000);
@@ -39,17 +39,17 @@
insert into t1 select rand(100000000) from t1 limit 488;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
-32704 32704
+16352 16352
insert into t1 select rand(100000000) from t1 limit 1;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
-32704 32704
+16352 16352
insert into t1 select rand(100000000) from t1 limit 1000;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
-49056 65408
+32704 32704
insert into t1 select rand(100000000) from t1;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
-81760 114464
+49056 65408
drop table t1;

55
mysql-test/suite/heap/heap_memory_used.result Normal file
View file

@ -0,0 +1,55 @@
#
# Test of heap table memory usage
#
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
16352 16352
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=10000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
16352 16352
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=3000 max_rows=3000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
65504 131040
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
16352 32736
drop table t1;
create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
insert into t1 select rand(100000000);
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1 limit 488;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
32704 32704
insert into t1 select rand(100000000) from t1 limit 1;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
32704 32704
insert into t1 select rand(100000000) from t1 limit 1000;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
49056 65408
insert into t1 select rand(100000000) from t1;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
data_length index_length
81760 114464
drop table t1;

50
mysql-test/suite/heap/heap_memory_used.test Normal file
View file

@ -0,0 +1,50 @@
--echo #
--echo # Test of heap table memory usage
--echo #
--source include/word_size.inc
#
# Show that MIN_ROWS and MAX_ROWS have an effect on how data_length
# and index_length are allocated.
# Result is different for 32 / 64 bit machines as pointer lengths are different
#
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=10000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=3000 max_rows=3000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;
CREATE TABLE t1 (a int, index(a)) engine=heap max_rows=15000;
insert into t1 values(1);
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;
create table t1 (c1 int, index(c1)) engine=heap max_rows=10000;
insert into t1 select rand(100000000);
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1;
insert into t1 select rand(100000000) from t1 limit 488;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
insert into t1 select rand(100000000) from t1 limit 1;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
insert into t1 select rand(100000000) from t1 limit 1000;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
insert into t1 select rand(100000000) from t1;
select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
drop table t1;

2
mysql-test/suite/maria/aria_sort_buffer.result
View file

@ -5,6 +5,8 @@ Note 1105 Cast to unsigned converted negative integer to it's positive complemen
Note 1105 Cast to unsigned converted negative integer to it's positive complement
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '18446744073709551615'
SET SESSION tmp_table_size=65535;
Warnings:
Warning 1292 Truncated incorrect tmp_table_size value: '65535'
CREATE TABLE t1 (a VARCHAR(255));
insert into t1 (a) select seq from seq_1_to_1000;
UPDATE t1 SET a=( (SELECT MAX(a) FROM t1));

2
mysql-test/suite/maria/repair-big-sort.result
View file

@ -2,6 +2,8 @@ SET sql_mode='';
CREATE TEMPORARY TABLE t1 (a tinyINT,b CHAR(1)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
INSERT INTO t1 VALUES (1,1),(3,3),(2,2);
SET SESSION tmp_table_size=True;
Warnings:
Warning 1292 Truncated incorrect tmp_table_size value: '1'
CREATE TABLE t2 (c INT, d DATE) ENGINE=InnoDB PARTITION BY RANGE (YEAR (d)) SUBPARTITION BY HASH (TO_DAYS (d)) (PARTITION p0 VALUES LESS THAN (1990) (SUBPARTITION s0, SUBPARTITION s1), PARTITION p1 VALUES LESS THAN MAXVALUE (SUBPARTITION s4, SUBPARTITION s5));
SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT);
Warnings:

2
mysql-test/suite/sys_vars/r/maximum_basic.result
View file

@ -9,7 +9,7 @@ Warnings:
Warning 1292 Truncated incorrect tmp_table_size value: '40960'
SELECT @@session.tmp_table_size;
@@session.tmp_table_size
8192
16384
SET @@session.max_join_size=40960;
Warnings:
Warning 1292 Truncated incorrect max_join_size value: '40960'

4
mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
View file

@ -3718,7 +3718,7 @@ VARIABLE_TYPE BIGINT UNSIGNED
VARIABLE_COMMENT If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. Same as tmp_table_size.
NUMERIC_MIN_VALUE 0
NUMERIC_MAX_VALUE 18446744073709551615
NUMERIC_BLOCK_SIZE 1
NUMERIC_BLOCK_SIZE 16384
ENUM_VALUE_LIST NULL
READ_ONLY NO
COMMAND_LINE_ARGUMENT REQUIRED
@ -3728,7 +3728,7 @@ VARIABLE_TYPE BIGINT UNSIGNED
VARIABLE_COMMENT Alias for tmp_memory_table_size. If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table.
NUMERIC_MIN_VALUE 0
NUMERIC_MAX_VALUE 18446744073709551615
NUMERIC_BLOCK_SIZE 1
NUMERIC_BLOCK_SIZE 16384
ENUM_VALUE_LIST NULL
READ_ONLY NO
COMMAND_LINE_ARGUMENT REQUIRED

4
mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
View file

@ -4498,7 +4498,7 @@ VARIABLE_TYPE BIGINT UNSIGNED
VARIABLE_COMMENT If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. Same as tmp_table_size.
NUMERIC_MIN_VALUE 0
NUMERIC_MAX_VALUE 18446744073709551615
NUMERIC_BLOCK_SIZE 1
NUMERIC_BLOCK_SIZE 16384
ENUM_VALUE_LIST NULL
READ_ONLY NO
COMMAND_LINE_ARGUMENT REQUIRED
@ -4508,7 +4508,7 @@ VARIABLE_TYPE BIGINT UNSIGNED
VARIABLE_COMMENT Alias for tmp_memory_table_size. If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table.
NUMERIC_MIN_VALUE 0
NUMERIC_MAX_VALUE 18446744073709551615
NUMERIC_BLOCK_SIZE 1
NUMERIC_BLOCK_SIZE 16384
ENUM_VALUE_LIST NULL
READ_ONLY NO
COMMAND_LINE_ARGUMENT REQUIRED

2
mysql-test/suite/sys_vars/r/tmp_disk_table_size_func.result
View file

@ -3,7 +3,7 @@
SET @start_tmp_memory_table_size=@@session.tmp_memory_table_size;
SET @start_tmp_disk_table_size=@@session.tmp_disk_table_size;
set @@session.tmp_memory_table_size=1000;
set @@session.tmp_memory_table_size=16384;
set @@session.tmp_disk_table_size=3000000;
create table t1 (a int primary key, b varchar(2000));
insert into t1 select seq,repeat(char(mod(seq,62)+64),seq) from seq_1_to_2000;

1
mysql-test/suite/sys_vars/r/transaction_alloc_block_size_basic.result
View file

@ -168,7 +168,6 @@ SET @@global.transaction_alloc_block_size = @start_global_value;
SELECT @@global.transaction_alloc_block_size;
@@global.transaction_alloc_block_size
8192
SET @@session.tmp_table_size = @start_session_value;
SELECT @@session.transaction_alloc_block_size;
@@session.transaction_alloc_block_size
1024

2
mysql-test/suite/sys_vars/t/maximum_basic-master.opt
View file

@ -1,5 +1,5 @@
--maximum-auto-increment-increment=8192
--maximum-tmp-table-size=8192
--maximum-tmp-table-size=16384
--maximum-max-join-size=8192
--maximum-use-stat-tables=COMPLEMENTARY
--maximum-sql-mode='REAL_AS_FLOAT,ANSI_QUOTES'

2
mysql-test/suite/sys_vars/t/tmp_disk_table_size_func.test
View file

@ -12,7 +12,7 @@
SET @start_tmp_memory_table_size=@@session.tmp_memory_table_size;
SET @start_tmp_disk_table_size=@@session.tmp_disk_table_size;
set @@session.tmp_memory_table_size=1000;
set @@session.tmp_memory_table_size=16384;
set @@session.tmp_disk_table_size=3000000;
--disable_ps2_protocol

2
mysql-test/suite/sys_vars/t/transaction_alloc_block_size_basic.test
View file

@ -218,11 +218,9 @@ SELECT transaction_alloc_block_size = @@session.transaction_alloc_block_size;
SET @@global.transaction_alloc_block_size = @start_global_value;
SELECT @@global.transaction_alloc_block_size;
SET @@session.tmp_table_size = @start_session_value;
SELECT @@session.transaction_alloc_block_size;
#############################################################
# END OF transaction_alloc_block_size TESTS #
#############################################################

4
mysql-test/suite/versioning/r/partition.result
View file

@ -1117,7 +1117,7 @@ f varchar(45000)
partition by system_time interval 1 year (partition p1 history,
partition pn current);
# fill the table until full
insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
insert into t1 (f) select f from t1;
ERROR HY000: The table 't1' is full
# leave space for exactly one record in current partition
@ -1133,7 +1133,7 @@ f varchar(45000)
) with system versioning engine=memory
partition by system_time interval 1 year (partition p1 history,
partition pn current);
insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
select * into outfile 'load.data' from t1;
load data infile 'load.data' replace into table t1;
load data infile 'load.data' replace into table t1;

4
mysql-test/suite/versioning/t/partition.test
View file

@ -977,7 +977,7 @@ create or replace table t1 (
partition pn current);
--echo # fill the table until full
insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
--error ER_RECORD_FILE_FULL
insert into t1 (f) select f from t1;
--echo # leave space for exactly one record in current partition
@ -995,7 +995,7 @@ create or replace table t1 (
partition by system_time interval 1 year (partition p1 history,
partition pn current);
insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
insert into t1 () values (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),();
--disable_cursor_protocol
--disable_ps2_protocol

2
sql/sql_select.cc
View file

@ -20577,7 +20577,7 @@ bool Create_tmp_table::finalize(THD *thd,
MY_MIN(thd->variables.tmp_memory_table_size,
thd->variables.max_heap_table_size) :
thd->variables.tmp_disk_table_size) /
share->reclength);
MY_ALIGN(share->reclength, sizeof(char*)));
set_if_bigger(share->max_rows,1); // For dummy start options
/*
Push the LIMIT clause to the temporary table creation, so that we

4
sql/sys_vars.cc
View file

@ -4218,7 +4218,7 @@ static Sys_var_ulonglong Sys_tmp_table_size(
"will automatically convert it to an on-disk MyISAM or Aria table.",
SESSION_VAR(tmp_memory_table_size), CMD_LINE(REQUIRED_ARG),
VALID_RANGE(0, (ulonglong)~(intptr)0), DEFAULT(16*1024*1024),
BLOCK_SIZE(1));
BLOCK_SIZE(16384));
static Sys_var_ulonglong Sys_tmp_memory_table_size(
"tmp_memory_table_size",
@ -4227,7 +4227,7 @@ static Sys_var_ulonglong Sys_tmp_memory_table_size(
"Same as tmp_table_size.",
SESSION_VAR(tmp_memory_table_size), CMD_LINE(REQUIRED_ARG),
VALID_RANGE(0, (ulonglong)~(intptr)0), DEFAULT(16*1024*1024),
BLOCK_SIZE(1));
BLOCK_SIZE(16384));
static Sys_var_ulonglong Sys_tmp_disk_table_size(
"tmp_disk_table_size",

14
storage/heap/ha_heap.cc
View file

@ -619,7 +619,7 @@ static int heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table,
case HA_KEY_ALG_UNDEF:
case HA_KEY_ALG_HASH:
keydef[key].algorithm= HA_KEY_ALG_HASH;
mem_per_row+= sizeof(char*) * 2; // = sizeof(HASH_INFO)
mem_per_row+= sizeof(HASH_INFO);
break;
case HA_KEY_ALG_BTREE:
keydef[key].algorithm= HA_KEY_ALG_BTREE;
@ -688,7 +688,6 @@ static int heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table,
}
}
}
mem_per_row+= MY_ALIGN(MY_MAX(share->reclength, sizeof(char*)) + 1, sizeof(char*));
if (table_arg->found_next_number_field)
{
keydef[share->next_number_index].flag|= HA_AUTO_KEY;
@ -696,11 +695,18 @@ static int heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table,
}
hp_create_info->auto_key= auto_key;
hp_create_info->auto_key_type= auto_key_type;
hp_create_info->max_table_size=current_thd->variables.max_heap_table_size;
hp_create_info->max_table_size= MY_MAX(current_thd->variables.max_heap_table_size, sizeof(HP_PTRS));
hp_create_info->with_auto_increment= found_real_auto_increment;
hp_create_info->internal_table= internal_table;
max_rows= (ha_rows) (hp_create_info->max_table_size / mem_per_row);
max_rows= hp_rows_in_memory(share->reclength, mem_per_row,
hp_create_info->max_table_size);
#ifdef GIVE_ERROR_IF_NOT_MEMORY_TO_INSERT_ONE_ROW
/* We do not give the error now but instead give an error on first insert */
if (!max_rows)
return HA_WRONG_CREATE_OPTION;
#endif
if (share->max_rows && share->max_rows < max_rows)
max_rows= share->max_rows;

3
storage/heap/heapdef.h
View file

@ -100,6 +100,9 @@ extern void hp_clear(HP_SHARE *info);
extern void hp_clear_keys(HP_SHARE *info);
extern uint hp_rb_pack_key(HP_KEYDEF *keydef, uchar *key, const uchar *old,
key_part_map keypart_map);
extern ha_rows hp_rows_in_memory(size_t reclength, size_t index_size,
size_t memory_limit);
extern size_t hp_memory_needed_per_row(size_t reclength);
extern mysql_mutex_t THR_LOCK_heap;

2
storage/heap/hp_block.c
View file

@ -78,6 +78,8 @@ int hp_get_new_block(HP_SHARE *info, HP_BLOCK *block, size_t *alloc_length)
*/
*alloc_length= (sizeof(HP_PTRS) * ((i == block->levels) ? i : i - 1) +
(ulonglong)block->records_in_block * block->recbuffer);
/* Alloc in blocks of powers of 2 */
*alloc_length= MY_MAX(*alloc_length, block->alloc_size);
if (!(root=(HP_PTRS*) my_malloc(hp_key_memory_HP_PTRS, *alloc_length,
MYF(MY_WME |
(info->internal ?

99
storage/heap/hp_create.c
View file

@ -15,11 +15,23 @@
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */
#include "heapdef.h"
#include <my_bit.h>
static int keys_compare(void *heap_rb, const void *key1, const void *key2);
static void init_block(HP_BLOCK *block,uint reclength,ulong min_records,
static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records,
ulong max_records);
/*
In how many parts are we going to do allocations of memory and indexes
If we assigne 1M to the heap table memory, we will allocate roughly
(1M/16) bytes per allocaiton
*/
static const int heap_allocation_parts= 16;
/* min block allocation */
static const ulong heap_min_allocation_block= 16384;
/* Create a heap table */
int heap_create(const char *name, HP_CREATE_INFO *create_info,
@ -170,7 +182,8 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info,
share->keydef= (HP_KEYDEF*) (share + 1);
share->key_stat_version= 1;
keyseg= (HA_KEYSEG*) (share->keydef + keys);
init_block(&share->block, visible_offset + 1, min_records, max_records);
init_block(&share->block, hp_memory_needed_per_row(reclength),
min_records, max_records);
/* Fix keys */
memcpy(share->keydef, keydef, (size_t) (sizeof(keydef[0]) * keys));
for (i= 0, keyinfo= share->keydef; i < keys; i++, keyinfo++)
@ -266,44 +279,90 @@ static int keys_compare(void *heap_rb_, const void *key1_,
heap_rb->search_flag, not_used);
}
static void init_block(HP_BLOCK *block, uint reclength, ulong min_records,
/*
Calculate length needed for storing one row
*/
size_t hp_memory_needed_per_row(size_t reclength)
{
/* Data needed for storing record + pointer to records */
reclength= MY_MAX(reclength, sizeof(char*));
/* The + 1 below is for the delete marker at the end of record*/
reclength= MY_ALIGN(reclength+1, sizeof(char*));
return reclength;
}
/*
Calculate the number of rows that fits into a given memory size
*/
ha_rows hp_rows_in_memory(size_t reclength, size_t index_size,
size_t memory_limit)
{
reclength= hp_memory_needed_per_row(reclength);
if ((memory_limit < index_size + reclength + sizeof(HP_PTRS)))
return 0; /* Wrong arguments */
return (ha_rows) ((memory_limit - sizeof(HP_PTRS)) /
(index_size + reclength));
}
static void init_block(HP_BLOCK *block, size_t reclength, ulong min_records,
ulong max_records)
{
ulong i,recbuffer,records_in_block;
ulong i,records_in_block;
ulong recbuffer= (ulong) MY_ALIGN(reclength, sizeof(uchar*));
ulong extra;
ulonglong memory_needed;
size_t alloc_size;
/*
If not min_records and max_records are given, optimize for 1000 rows
*/
if (!min_records)
min_records= MY_MIN(1000, max_records);
min_records= MY_MIN(1000, max_records / heap_allocation_parts);
if (!max_records)
max_records= MY_MAX(min_records, 1000);
min_records= MY_MIN(min_records, max_records);
/*
We don't want too few records_in_block as otherwise the overhead of
of the HP_PTRS block will be too notable
*/
records_in_block= MY_MAX(1000, min_records);
records_in_block= MY_MIN(records_in_block, max_records);
/* If big max_records is given, allocate bigger blocks */
records_in_block= MY_MAX(records_in_block, max_records / 10);
records_in_block= MY_MAX(min_records, max_records / heap_allocation_parts);
/*
Align allocation sizes to power of 2 to get less memory fragmentation from
system alloc().
As long as we have less than 128 allocations, all but one of the
allocations will have an extra HP_PTRS size structure at the start
of the block.
We ensure that the block is not smaller than heap_min_allocation_block
as otherwise we get strange results when max_records <
heap_allocation_parts)
*/
extra= sizeof(HP_PTRS) + MALLOC_OVERHEAD;
/* We don't want too few blocks per row either */
if (records_in_block < 10)
records_in_block= 10;
records_in_block= MY_MIN(10, max_records);
memory_needed= MY_MAX(((ulonglong) records_in_block * recbuffer + extra),
(ulonglong) heap_min_allocation_block);
/* We have to limit memory to INT_MAX32 as my_round_up_to_next_power() is 32 bit */
memory_needed= MY_MIN(memory_needed, (ulonglong) INT_MAX32);
alloc_size= my_round_up_to_next_power((uint32)memory_needed);
records_in_block= (ulong) ((alloc_size - extra)/ recbuffer);
DBUG_PRINT("info", ("records_in_block: %lu" ,records_in_block));
recbuffer= (uint) (reclength + sizeof(uchar**) - 1) & ~(sizeof(uchar**) - 1);
/*
Don't allocate more than my_default_record_cache_size per level.
The + 1 is there to ensure that we get at least 1 row per level (for
the exceptional case of very long rows)
*/
if ((ulonglong) records_in_block*recbuffer >
(my_default_record_cache_size-sizeof(HP_PTRS)*HP_MAX_LEVELS))
records_in_block= (my_default_record_cache_size - sizeof(HP_PTRS) *
HP_MAX_LEVELS) / recbuffer + 1;
block->records_in_block= records_in_block;
block->recbuffer= recbuffer;
block->last_allocated= 0L;
/* All alloctions are done with this size, if possible */
block->alloc_size= alloc_size - MALLOC_OVERHEAD;
for (i= 0; i <= HP_MAX_LEVELS; i++)
block->level_info[i].records_under_level=

5
storage/heap/hp_write.c
View file

@ -145,6 +145,8 @@ static uchar *next_free_record_pos(HP_SHARE *info)
DBUG_PRINT("exit",("Used old position: %p", pos));
DBUG_RETURN(pos);
}
if (!(block_pos=(info->records % info->block.records_in_block)))
{
if ((info->records > info->max_records && info->max_records) ||
(info->data_length + info->index_length >= info->max_table_size))
{
@ -158,8 +160,7 @@ static uchar *next_free_record_pos(HP_SHARE *info)
my_errno=HA_ERR_RECORD_FILE_FULL;
DBUG_RETURN(NULL);
}
if (!(block_pos=(info->records % info->block.records_in_block)))
{
if (hp_get_new_block(info, &info->block,&length))
DBUG_RETURN(NULL);
info->data_length+=length;