mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-04-18 21:25:32 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions1

MDEV-27913 innodb_ft_cache_size max possible value (80000000) is too small for practical purposes

Browse source 
- Make innodb_ft_cache_size & innodb_ft_total_cache_size are dynamic
variable and increase the maximum value of innodb_ft_cache_size to
512MB for 32-bit system and 1 TB for 64-bit system and set
innodb_ft_total_cache_size maximum value to 1 TB for 64-bit system.

- Print warning if the fts cache exceeds the innodb_ft_cache_size
and also unlock the cache if fts cache memory reduces less than
innodb_ft_cache_size.
This commit is contained in:
Thirunarayanan Balathandayuthapani 2022-02-24 19:42:43 +05:30
parent a6f258e47f
commit a76731e1a1
10 changed files with 135 additions and 30 deletions

1
mysql-test/suite/innodb_fts/r/bug_32831765.result
View file

@ -1,3 +1,4 @@
call mtr.add_suppression("InnoDB: Total InnoDB FTS size .* for the table .* exceeds the innodb_ft_cache_size .*");
#
# Bug#32831765 SERVER HITS OOM CONDITION WHEN LOADING TWO
# INNODB TABLES WITH FTS INDEXES

1
mysql-test/suite/innodb_fts/t/bug_32831765.test
View file

@ -2,6 +2,7 @@
--source include/have_debug.inc
--source include/big_test.inc
call mtr.add_suppression("InnoDB: Total InnoDB FTS size .* for the table .* exceeds the innodb_ft_cache_size .*");
--echo #
--echo # Bug#32831765 SERVER HITS OOM CONDITION WHEN LOADING TWO
--echo # INNODB TABLES WITH FTS INDEXES

13
mysql-test/suite/sys_vars/r/innodb_ft_cache_size_basic.result
View file

@ -16,6 +16,15 @@ select * from information_schema.session_variables where variable_name='innodb_f
VARIABLE_NAME VARIABLE_VALUE
INNODB_FT_CACHE_SIZE 8000000
set global innodb_ft_cache_size=1;
ERROR HY000: Variable 'innodb_ft_cache_size' is a read only variable
Warnings:
Warning 1292 Truncated incorrect innodb_ft_cache_size value: '1'
SHOW VARIABLES like 'innodb_ft_cache_size';
Variable_name Value
innodb_ft_cache_size 1600000
set session innodb_ft_cache_size=1;
ERROR HY000: Variable 'innodb_ft_cache_size' is a read only variable
ERROR HY000: Variable 'innodb_ft_cache_size' is a GLOBAL variable and should be set with SET GLOBAL
set global innodb_ft_cache_size=512*1024*1024;
SHOW VARIABLES like 'innodb_ft_cache_size';
Variable_name Value
innodb_ft_cache_size 536870912
set global innodb_ft_cache_size=default;

13
mysql-test/suite/sys_vars/r/innodb_ft_total_cache_size_basic.result
View file

@ -16,6 +16,15 @@ select * from information_schema.session_variables where variable_name='innodb_f
VARIABLE_NAME VARIABLE_VALUE
INNODB_FT_TOTAL_CACHE_SIZE 640000000
set global innodb_ft_total_cache_size=1;
ERROR HY000: Variable 'innodb_ft_total_cache_size' is a read only variable
Warnings:
Warning 1292 Truncated incorrect innodb_ft_total_cache_size value: '1'
set session innodb_ft_total_cache_size=1;
ERROR HY000: Variable 'innodb_ft_total_cache_size' is a read only variable
ERROR HY000: Variable 'innodb_ft_total_cache_size' is a GLOBAL variable and should be set with SET GLOBAL
SHOW VARIABLES like 'innodb_ft_total_cache_size';
Variable_name Value
innodb_ft_total_cache_size 32000000
set global innodb_ft_total_cache_size=512*1024*1024;
show variables like 'innodb_ft_total_cache_size';
Variable_name Value
innodb_ft_total_cache_size 536870912
set global innodb_ft_total_cache_size=default;

11
mysql-test/suite/sys_vars/t/innodb_ft_cache_size_basic.test
View file

@ -18,11 +18,12 @@ select * from information_schema.global_variables where variable_name='innodb_ft
select * from information_schema.session_variables where variable_name='innodb_ft_cache_size';
--enable_warnings
#
# show that it's read-only
#
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
set global innodb_ft_cache_size=1;
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
SHOW VARIABLES like 'innodb_ft_cache_size';
--error ER_GLOBAL_VARIABLE
set session innodb_ft_cache_size=1;
set global innodb_ft_cache_size=512*1024*1024;
SHOW VARIABLES like 'innodb_ft_cache_size';
set global innodb_ft_cache_size=default;

10
mysql-test/suite/sys_vars/t/innodb_ft_total_cache_size_basic.test
View file

@ -18,11 +18,11 @@ select * from information_schema.global_variables where variable_name='innodb_ft
select * from information_schema.session_variables where variable_name='innodb_ft_total_cache_size';
--enable_warnings
#
# show that it's read-only
#
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
set global innodb_ft_total_cache_size=1;
--error ER_INCORRECT_GLOBAL_LOCAL_VAR
--error ER_GLOBAL_VARIABLE
set session innodb_ft_total_cache_size=1;
SHOW VARIABLES like 'innodb_ft_total_cache_size';
set global innodb_ft_total_cache_size=512*1024*1024;
show variables like 'innodb_ft_total_cache_size';
set global innodb_ft_total_cache_size=default;

37
storage/innobase/fts/fts0fts.cc
View file

@ -53,14 +53,26 @@ by looking up the key word in the obsolete table names */
/** This is maximum FTS cache for each table and would be
a configurable variable */
ulong fts_max_cache_size;
size_t fts_max_cache_size;
static size_t fts_get_max_cache()
{
#if UNIV_WORD_SIZE == 4
return my_atomic_load32_explicit(&fts_max_cache_size,
MY_MEMORY_ORDER_RELAXED);
#else
return my_atomic_load64_explicit(
reinterpret_cast<int64*>(&fts_max_cache_size),
MY_MEMORY_ORDER_RELAXED);
#endif
}
/** Whether the total memory used for FTS cache is exhausted, and we will
need a sync to free some memory */
bool fts_need_sync = false;
/** Variable specifying the total memory allocated for FTS cache */
ulong fts_max_total_cache_size;
size_t fts_max_total_cache_size;
/** This is FTS result cache limit for each query and would be
a configurable variable */
@ -3381,7 +3393,7 @@ fts_add_doc_from_tuple(
rw_lock_x_unlock(&table->fts->cache->lock);
if (cache->total_size > fts_max_cache_size / 5
if (cache->total_size > fts_get_max_cache() / 5
|| fts_need_sync) {
fts_sync(cache->sync, true, false);
}
@ -3546,7 +3558,7 @@ fts_add_doc_by_id(
&& (fts_need_sync
|| (cache->total_size
- cache->total_size_at_sync)
> fts_max_cache_size / 10);
> fts_get_max_cache() / 10);
if (need_sync) {
cache->total_size_at_sync =
cache->total_size;
@ -4284,7 +4296,7 @@ fts_sync(
ulint i;
dberr_t error = DB_SUCCESS;
fts_cache_t* cache = sync->table->fts->cache;
size_t fts_cache_size= 0;
rw_lock_x_lock(&cache->lock);
/* Check if cache is being synced.
@ -4309,11 +4321,17 @@ fts_sync(
fts_sync_begin(sync);
begin_sync:
if (cache->total_size > fts_max_cache_size) {
fts_cache_size= fts_get_max_cache();
if (cache->total_size > fts_cache_size) {
/* Avoid the case: sync never finish when
insert/update keeps comming. */
ut_ad(sync->unlock_cache);
sync->unlock_cache = false;
ib::warn() << "Total InnoDB FTS size "
<< cache->total_size << " for the table "
<< cache->sync->table->name
<< " exceeds the innodb_ft_cache_size "
<< fts_cache_size;
}
for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
@ -4336,6 +4354,13 @@ begin_sync:
if (error != DB_SUCCESS) {
goto end_sync;
}
if (!sync->unlock_cache
&& cache->total_size < fts_get_max_cache()) {
/* Reset the unlock cache if the value
is less than innodb_ft_cache_size */
sync->unlock_cache = true;
}
}
DBUG_EXECUTE_IF("fts_instrument_sync_interrupted",

15
storage/innobase/fts/fts0opt.cc
View file

@ -2734,6 +2734,19 @@ static ulint fts_optimize_how_many()
return(n_tables);
}
/** @return innodb_ft_total_cache_size */
static size_t fts_get_max_total_cache_size()
{
#if UNIV_WORD_SIZE == 4
return my_atomic_load32_explicit(
&fts_max_total_cache_size, MY_MEMORY_ORDER_RELAXED);
#else
return my_atomic_load64_explicit((volatile int64 *)
reinterpret_cast<int64*>(&fts_max_total_cache_size),
MY_MEMORY_ORDER_RELAXED);
#endif
}
/**********************************************************************//**
Check if the total memory used by all FTS table exceeds the maximum limit.
@return true if a sync is needed, false otherwise */
@ -2761,7 +2774,7 @@ static bool fts_is_sync_needed()
total_memory += slot->table->fts->cache->total_size;
}
if (total_memory > fts_max_total_cache_size) {
if (total_memory > fts_get_max_total_cache_size()) {
return(true);
}
}

60
storage/innobase/handler/ha_innodb.cc
View file

@ -800,6 +800,36 @@ innodb_stopword_table_validate(
for update function */
struct st_mysql_value* value); /*!< in: incoming string */
static
void innodb_ft_cache_size_update(THD*, st_mysql_sys_var*, void*, const void* save)
{
#if UNIV_WORD_SIZE == 4
my_atomic_store32_explicit(
&fts_max_cache_size, *static_cast<const size_t*>(save),
MY_MEMORY_ORDER_RELAXED);
#else
my_atomic_store64_explicit(
reinterpret_cast<int64*>(&fts_max_cache_size),
*static_cast<const size_t*>(save),
MY_MEMORY_ORDER_RELAXED);
#endif
}
static
void innodb_ft_total_cache_size_update(THD*, st_mysql_sys_var*, void*, const void* save)
{
#if UNIV_WORD_SIZE == 4
my_atomic_store32_explicit(
&fts_max_total_cache_size, *static_cast<const size_t*>(save),
MY_MEMORY_ORDER_RELAXED);
#else
my_atomic_store64_explicit(
reinterpret_cast<int64*>(&fts_max_total_cache_size),
*static_cast<const size_t*>(save),
MY_MEMORY_ORDER_RELAXED);
#endif
}
static bool is_mysql_datadir_path(const char *path);
/** Validate passed-in "value" is a valid directory name.
@ -19787,15 +19817,31 @@ static MYSQL_SYSVAR_STR(ft_aux_table, innodb_ft_aux_table,
"FTS internal auxiliary table to be checked",
innodb_ft_aux_table_validate, NULL, NULL);
static MYSQL_SYSVAR_ULONG(ft_cache_size, fts_max_cache_size,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"InnoDB Fulltext search cache size in bytes",
NULL, NULL, 8000000, 1600000, 80000000, 0);
#if UNIV_WORD_SIZE == 4
static MYSQL_SYSVAR_ULONG(ft_total_cache_size, fts_max_total_cache_size,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
static MYSQL_SYSVAR_SIZE_T(ft_cache_size, fts_max_cache_size,
PLUGIN_VAR_RQCMDARG,
"InnoDB Fulltext search cache size in bytes",
NULL, innodb_ft_cache_size_update, 8000000, 1600000, 1U << 29, 0);
static MYSQL_SYSVAR_SIZE_T(ft_total_cache_size, fts_max_total_cache_size,
PLUGIN_VAR_RQCMDARG,
"Total memory allocated for InnoDB Fulltext Search cache",
NULL, NULL, 640000000, 32000000, 1600000000, 0);
NULL, innodb_ft_total_cache_size_update, 640000000, 32000000, 1600000000, 0);
#else
static MYSQL_SYSVAR_SIZE_T(ft_cache_size, fts_max_cache_size,
PLUGIN_VAR_RQCMDARG,
"InnoDB Fulltext search cache size in bytes",
NULL, innodb_ft_cache_size_update, 8000000, 1600000, 1ULL << 40, 0);
static MYSQL_SYSVAR_SIZE_T(ft_total_cache_size, fts_max_total_cache_size,
PLUGIN_VAR_RQCMDARG,
"Total memory allocated for InnoDB Fulltext Search cache",
NULL, innodb_ft_total_cache_size_update, 640000000, 32000000, 1ULL << 40, 0);
#endif
static MYSQL_SYSVAR_SIZE_T(ft_result_cache_limit, fts_result_cache_limit,
PLUGIN_VAR_RQCMDARG,

4
storage/innobase/include/fts0fts.h
View file

@ -352,10 +352,10 @@ struct fts_stopword_t;
extern const char* fts_default_stopword[];
/** Variable specifying the maximum FTS cache size for each table */
extern ulong fts_max_cache_size;
extern size_t fts_max_cache_size;
/** Variable specifying the total memory allocated for FTS cache */
extern ulong fts_max_total_cache_size;
extern size_t fts_max_total_cache_size;
/** Variable specifying the FTS result cache limit for each query */
extern size_t fts_result_cache_limit;