From 0b612619efb57734c2e1d10452ca25f24267fc69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2024 14:36:54 +0200 Subject: [PATCH 01/10] MDEV-33098: Fix some instrumentation for innodb.doublewrite_debug buf_flush_page_cleaner(): A continue or break inside DBUG_EXECUTE_IF actually is a no-op. Use an explicit call to _db_keyword_() to actually avoid advancing the checkpoint. buf_flush_list_now_set(): Invoke os_aio_wait_until_no_pending_writes() to ensure that the page write to the system tablespace is completed. --- mysql-test/suite/innodb/t/doublewrite_debug.test | 11 ----------- storage/innobase/buf/buf0flu.cc | 6 +++--- storage/innobase/handler/ha_innodb.cc | 3 +++ 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/mysql-test/suite/innodb/t/doublewrite_debug.test b/mysql-test/suite/innodb/t/doublewrite_debug.test index b8eb81c645c..1bff8b4e07f 100644 --- a/mysql-test/suite/innodb/t/doublewrite_debug.test +++ b/mysql-test/suite/innodb/t/doublewrite_debug.test @@ -61,17 +61,6 @@ set global innodb_fil_make_page_dirty_debug = 0; set global innodb_buf_flush_list_now = 1; --let CLEANUP_IF_CHECKPOINT=drop table t1, unexpected_checkpoint; -# Occasionally, a checkpoint would occur on the MSAN builder. -# We do not know the reason, because the failure can only be reproduced if there is -# enough load in that environment. -# Therefore, we allow the test to be skipped when run on MSAN. -# In other environments, we want the test to fail if a checkpoint occurs, -# so that we would catch it if it starts to happen regularly. -if (`select count(*) from information_schema.system_variables where variable_name='have_sanitizer' and global_value like "MSAN%"`) -{ ---let CLEANUP_IF_CHECKPOINT=drop table t1; -} - --source ../include/no_checkpoint_end.inc --echo # Make the 1st page (page_no=0) and 2nd page (page_no=1) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index c0c743ede57..e181d311a7c 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -2333,9 +2333,9 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*) do { - DBUG_EXECUTE_IF("ib_log_checkpoint_avoid", continue;); - DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", continue;); - + IF_DBUG(if (_db_keyword_(nullptr, "ib_log_checkpoint_avoid", 1) || + _db_keyword_(nullptr, "ib_log_checkpoint_avoid_hard", 1)) + continue,); if (!recv_recovery_is_on() && !srv_startup_is_before_trx_rollback_phase && srv_operation <= SRV_OPERATION_EXPORT_RESTORED) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 777ea1ba86e..9b4f8b37cb5 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -18313,7 +18313,10 @@ buf_flush_list_now_set(THD*, st_mysql_sys_var*, void*, const void* save) if (s) buf_flush_sync(); else + { while (buf_flush_list_space(fil_system.sys_space, nullptr)); + os_aio_wait_until_no_pending_writes(); + } mysql_mutex_lock(&LOCK_global_system_variables); } From 022ae42155588b63245f2e6fbd0d795d2b7cf5ea Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 14 Dec 2023 15:17:59 +0100 Subject: [PATCH 02/10] MDEV-11777 REGEXP_REPLACE converts utf8mb4 supplementary characters to '?' use utf8mb4 with PCRE2, not utf8mb3 --- mysql-test/main/func_regexp_pcre.result | 9 +++++++++ mysql-test/main/func_regexp_pcre.test | 8 ++++++++ sql/item_cmpfunc.cc | 2 +- sql/item_cmpfunc.h | 2 +- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/func_regexp_pcre.result b/mysql-test/main/func_regexp_pcre.result index 9e51ced961c..36f2fced1db 100644 --- a/mysql-test/main/func_regexp_pcre.result +++ b/mysql-test/main/func_regexp_pcre.result @@ -895,3 +895,12 @@ REGEXP_INSTR('a_kollision', 'o([lm])\\1') 4 SELECT a FROM (SELECT "aa" a) t WHERE a REGEXP '[0-9]'; a +# +# MDEV-11777 REGEXP_REPLACE converts utf8mb4 supplementary characters to '?' +# +select hex(regexp_replace(cast(x'F09F9881' as char character set 'utf8mb4'), _utf8mb4'a', _utf8mb4'b')) as Text; +Text +F09F9881 +# +# End of 10.6 tests +# diff --git a/mysql-test/main/func_regexp_pcre.test b/mysql-test/main/func_regexp_pcre.test index e6e356f4a8c..8c2408f5763 100644 --- a/mysql-test/main/func_regexp_pcre.test +++ b/mysql-test/main/func_regexp_pcre.test @@ -470,3 +470,11 @@ SELECT REGEXP_INSTR('a_kollision', 'o([lm])\\1'); # SELECT a FROM (SELECT "aa" a) t WHERE a REGEXP '[0-9]'; --enable_service_connection + +--echo # +--echo # MDEV-11777 REGEXP_REPLACE converts utf8mb4 supplementary characters to '?' +--echo # +select hex(regexp_replace(cast(x'F09F9881' as char character set 'utf8mb4'), _utf8mb4'a', _utf8mb4'b')) as Text; +--echo # +--echo # End of 10.6 tests +--echo # diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index b978267f0b7..f8fd28aebb5 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -6096,7 +6096,7 @@ void Regexp_processor_pcre::init(CHARSET_INFO *data_charset, int extra_flags) // Convert text data to utf-8. m_library_charset= data_charset == &my_charset_bin ? - &my_charset_bin : &my_charset_utf8mb3_general_ci; + &my_charset_bin : &my_charset_utf8mb4_general_ci; m_conversion_is_needed= (data_charset != &my_charset_bin) && !my_charset_same(data_charset, m_library_charset); diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index af8586695ec..f393635d1e4 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -3057,7 +3057,7 @@ public: m_pcre(NULL), m_pcre_match_data(NULL), m_conversion_is_needed(true), m_is_const(0), m_library_flags(0), - m_library_charset(&my_charset_utf8mb3_general_ci) + m_library_charset(&my_charset_utf8mb4_general_ci) {} int default_regex_flags(); void init(CHARSET_INFO *data_charset, int extra_flags); From c44cac91ab9ea449cb57eabf593cca4a5cfe2deb Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 27 Dec 2023 17:04:43 +0100 Subject: [PATCH 03/10] MDEV-33031 Assertion failure upon reading from performance schema with binlog enabled need to protect access to thread-local cache_mngr with LOCK_thd_data technically only access from different threads has to be protected, but this is the SHOW STATUS code path, so the difference is neglectable --- .../suite/perfschema/r/rpl_threads.result | 6 +++++ .../suite/perfschema/t/rpl_threads.test | 6 +++++ sql/log.cc | 23 ++++++++++--------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/mysql-test/suite/perfschema/r/rpl_threads.result b/mysql-test/suite/perfschema/r/rpl_threads.result index c756b4d9046..3e9cf22f715 100644 --- a/mysql-test/suite/perfschema/r/rpl_threads.result +++ b/mysql-test/suite/perfschema/r/rpl_threads.result @@ -58,4 +58,10 @@ select NAME, TYPE, PROCESSLIST_COMMAND, PROCESSLIST_STATE from performance_schema.threads where PROCESSLIST_ID = @slave_sql_pid; NAME TYPE PROCESSLIST_COMMAND PROCESSLIST_STATE +# +# MDEV-33031 Assertion failure upon reading from performance schema with binlog enabled +# +select variable_name, variable_value from performance_schema.status_by_thread +where variable_name like '%impossible%'; +variable_name variable_value include/rpl_end.inc diff --git a/mysql-test/suite/perfschema/t/rpl_threads.test b/mysql-test/suite/perfschema/t/rpl_threads.test index a5ca51a94a4..fcecf775722 100644 --- a/mysql-test/suite/perfschema/t/rpl_threads.test +++ b/mysql-test/suite/perfschema/t/rpl_threads.test @@ -81,5 +81,11 @@ select NAME, TYPE, PROCESSLIST_COMMAND, PROCESSLIST_STATE from performance_schema.threads where PROCESSLIST_ID = @slave_sql_pid; +--echo # +--echo # MDEV-33031 Assertion failure upon reading from performance schema with binlog enabled +--echo # +select variable_name, variable_value from performance_schema.status_by_thread +where variable_name like '%impossible%'; # should not crash + --source include/rpl_end.inc diff --git a/sql/log.cc b/sql/log.cc index fa15ccfeb4c..2e55949a722 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -11826,14 +11826,21 @@ set_binlog_snapshot_file(const char *src) void TC_LOG_BINLOG::set_status_variables(THD *thd) { - binlog_cache_mngr *cache_mngr; + bool have_snapshot= false; if (thd && opt_bin_log) - cache_mngr= (binlog_cache_mngr*) thd_get_ha_data(thd, binlog_hton); - else - cache_mngr= 0; + { + mysql_mutex_lock(&thd->LOCK_thd_data); + auto cache_mngr= (binlog_cache_mngr*) thd_get_ha_data(thd, binlog_hton); + have_snapshot= cache_mngr && cache_mngr->last_commit_pos_file[0]; + if (have_snapshot) + { + set_binlog_snapshot_file(cache_mngr->last_commit_pos_file); + binlog_snapshot_position= cache_mngr->last_commit_pos_offset; + } + mysql_mutex_unlock(&thd->LOCK_thd_data); + } - bool have_snapshot= (cache_mngr && cache_mngr->last_commit_pos_file[0] != 0); mysql_mutex_lock(&LOCK_commit_ordered); binlog_status_var_num_commits= this->num_commits; binlog_status_var_num_group_commits= this->num_group_commits; @@ -11848,12 +11855,6 @@ TC_LOG_BINLOG::set_status_variables(THD *thd) binlog_status_group_commit_trigger_timeout= this->group_commit_trigger_timeout; binlog_status_group_commit_trigger_lock_wait= this->group_commit_trigger_lock_wait; mysql_mutex_unlock(&LOCK_prepare_ordered); - - if (have_snapshot) - { - set_binlog_snapshot_file(cache_mngr->last_commit_pos_file); - binlog_snapshot_position= cache_mngr->last_commit_pos_offset; - } } From b3065af6e6a944776f5410e89fffea5aecd7320f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 28 Dec 2023 15:19:42 +0100 Subject: [PATCH 04/10] cleanup: spider status variables reduce code duplication --- storage/spider/spd_param.cc | 49 ++++++++++++------------------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc index 90be2c045b9..2da262cd2bc 100644 --- a/storage/spider/spd_param.cc +++ b/storage/spider/spd_param.cc @@ -116,73 +116,56 @@ extern volatile ulonglong spider_mon_table_cache_version_req; MYSQL_SYSVAR_NAME(param_name).def_val; \ } -#ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS -static int spider_direct_update(THD *thd, SHOW_VAR *var, char *buff) +static int spider_trx_status_var(THD *thd, SHOW_VAR *var, char *buff, + ulonglong SPIDER_TRX::*counter) { int error_num = 0; SPIDER_TRX *trx; DBUG_ENTER("spider_direct_update"); var->type = SHOW_LONGLONG; if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->direct_update_count; + var->value = (char *) &(trx->*counter); DBUG_RETURN(error_num); } + +#ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS +static int spider_direct_update(THD *thd, SHOW_VAR *var, char *buff) +{ + DBUG_ENTER("spider_direct_update"); + DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_update_count)); +} + static int spider_direct_delete(THD *thd, SHOW_VAR *var, char *buff) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_direct_delete"); - var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->direct_delete_count; - DBUG_RETURN(error_num); + DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_delete_count)); } #endif static int spider_direct_order_limit(THD *thd, SHOW_VAR *var, char *buff) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_direct_order_limit"); - var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->direct_order_limit_count; - DBUG_RETURN(error_num); + DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_order_limit_count)); } static int spider_direct_aggregate(THD *thd, SHOW_VAR *var, char *buff) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_direct_aggregate"); - var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->direct_aggregate_count; - DBUG_RETURN(error_num); + DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_aggregate_count)); } static int spider_parallel_search(THD *thd, SHOW_VAR *var, char *buff) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_parallel_search"); - var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->parallel_search_count; - DBUG_RETURN(error_num); + DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::parallel_search_count)); } #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) static int spider_hs_result_free(THD *thd, SHOW_VAR *var, char *buff) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_hs_result_free"); - var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &trx->hs_result_free_count; - DBUG_RETURN(error_num); + DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::hs_result_free_count)); } #endif From 23e107d751ced72ca81d8f362a9ee56aaf34a032 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 28 Dec 2023 15:23:40 +0100 Subject: [PATCH 05/10] MDEV-33031 Assertion failure upon reading from performance schema with binlog enabled same assertion with spider. spider status variables didn't expect to be queried from a different thread without LOCK_thd_data. And they didn't expect to be queried under LOCK_thd_data either (because spider_get_trx() calls thd_set_ha_data()). --- .../mysql-test/spider/bugfix/r/perfschema.result | 11 +++++++++++ .../mysql-test/spider/bugfix/t/perfschema.opt | 1 + .../mysql-test/spider/bugfix/t/perfschema.test | 15 +++++++++++++++ storage/spider/spd_param.cc | 14 +++++++++----- 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/perfschema.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/perfschema.opt create mode 100644 storage/spider/mysql-test/spider/bugfix/t/perfschema.test diff --git a/storage/spider/mysql-test/spider/bugfix/r/perfschema.result b/storage/spider/mysql-test/spider/bugfix/r/perfschema.result new file mode 100644 index 00000000000..9ce2e38f40b --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/perfschema.result @@ -0,0 +1,11 @@ +# +# MDEV-33031 Assertion failure upon reading from performance schema with binlog enabled +# +connect foo,localhost,root; +select variable_name, variable_value from performance_schema.status_by_thread +where variable_name like '%spider_direct_aggregate%'; +variable_name variable_value +Spider_direct_aggregate 0 +Spider_direct_aggregate 0 +disconnect foo; +connection default; diff --git a/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt b/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt new file mode 100644 index 00000000000..611d08f0c78 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/perfschema.opt @@ -0,0 +1 @@ +--performance-schema diff --git a/storage/spider/mysql-test/spider/bugfix/t/perfschema.test b/storage/spider/mysql-test/spider/bugfix/t/perfschema.test new file mode 100644 index 00000000000..2f1a961a513 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/perfschema.test @@ -0,0 +1,15 @@ +disable_query_log; +source ../../include/init_spider.inc; +enable_query_log; + +--echo # +--echo # MDEV-33031 Assertion failure upon reading from performance schema with binlog enabled +--echo # +connect foo,localhost,root; +select variable_name, variable_value from performance_schema.status_by_thread +where variable_name like '%spider_direct_aggregate%'; +disconnect foo; +connection default; + +disable_query_log; +source ../../include/deinit_spider.inc; diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc index 2da262cd2bc..308857d153a 100644 --- a/storage/spider/spd_param.cc +++ b/storage/spider/spd_param.cc @@ -116,16 +116,20 @@ extern volatile ulonglong spider_mon_table_cache_version_req; MYSQL_SYSVAR_NAME(param_name).def_val; \ } +extern handlerton *spider_hton_ptr; static int spider_trx_status_var(THD *thd, SHOW_VAR *var, char *buff, ulonglong SPIDER_TRX::*counter) { - int error_num = 0; - SPIDER_TRX *trx; DBUG_ENTER("spider_direct_update"); var->type = SHOW_LONGLONG; - if ((trx = spider_get_trx(thd, TRUE, &error_num))) - var->value = (char *) &(trx->*counter); - DBUG_RETURN(error_num); + var->value= buff; + if (thd != current_thd) + mysql_mutex_lock(&thd->LOCK_thd_data); + SPIDER_TRX *trx = (SPIDER_TRX*)thd_get_ha_data(thd, spider_hton_ptr); + *(ulonglong*)buff= trx ? trx->*counter : 0; + if (thd != current_thd) + mysql_mutex_unlock(&thd->LOCK_thd_data); + DBUG_RETURN(0); } From 0a122637b51c8024ab3ef1b7db30336fd935becc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 9 Jan 2024 15:54:36 +0100 Subject: [PATCH 06/10] cleanup: change a function, that always return 0, to void --- storage/spider/spd_param.cc | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc index 308857d153a..7e87511ba4a 100644 --- a/storage/spider/spd_param.cc +++ b/storage/spider/spd_param.cc @@ -117,7 +117,7 @@ extern volatile ulonglong spider_mon_table_cache_version_req; } extern handlerton *spider_hton_ptr; -static int spider_trx_status_var(THD *thd, SHOW_VAR *var, char *buff, +static void spider_trx_status_var(THD *thd, SHOW_VAR *var, char *buff, ulonglong SPIDER_TRX::*counter) { DBUG_ENTER("spider_direct_update"); @@ -129,7 +129,7 @@ static int spider_trx_status_var(THD *thd, SHOW_VAR *var, char *buff, *(ulonglong*)buff= trx ? trx->*counter : 0; if (thd != current_thd) mysql_mutex_unlock(&thd->LOCK_thd_data); - DBUG_RETURN(0); + DBUG_VOID_RETURN; } @@ -137,39 +137,45 @@ static int spider_trx_status_var(THD *thd, SHOW_VAR *var, char *buff, static int spider_direct_update(THD *thd, SHOW_VAR *var, char *buff) { DBUG_ENTER("spider_direct_update"); - DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_update_count)); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_update_count); + DBUG_RETURN(0); } static int spider_direct_delete(THD *thd, SHOW_VAR *var, char *buff) { DBUG_ENTER("spider_direct_delete"); - DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_delete_count)); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_delete_count); + DBUG_RETURN(0); } #endif static int spider_direct_order_limit(THD *thd, SHOW_VAR *var, char *buff) { DBUG_ENTER("spider_direct_order_limit"); - DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_order_limit_count)); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_order_limit_count); + DBUG_RETURN(0); } static int spider_direct_aggregate(THD *thd, SHOW_VAR *var, char *buff) { DBUG_ENTER("spider_direct_aggregate"); - DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_aggregate_count)); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::direct_aggregate_count); + DBUG_RETURN(0); } static int spider_parallel_search(THD *thd, SHOW_VAR *var, char *buff) { DBUG_ENTER("spider_parallel_search"); - DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::parallel_search_count)); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::parallel_search_count); + DBUG_RETURN(0); } #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) static int spider_hs_result_free(THD *thd, SHOW_VAR *var, char *buff) { DBUG_ENTER("spider_hs_result_free"); - DBUG_RETURN(spider_trx_status_var(thd, var, buff, &SPIDER_TRX::hs_result_free_count)); + spider_trx_status_var(thd, var, buff, &SPIDER_TRX::hs_result_free_count); + DBUG_RETURN(0); } #endif From c6c2a2b8d463e0a103997aba81f18c37fcdc0597 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 2 Jan 2024 22:23:26 +0100 Subject: [PATCH 07/10] MDEV-33150 double-locking of LOCK_thd_kill in performance_schema.session_status perfschema thread walker needs to take thread's LOCK_thd_kill to prevent the thread from disappearing why it's being looked at. But there's no need to lock it for the current thread. In fact, it was harmful as some code down the stack might take LOCK_thd_kill (e.g. set_killed() does it, and my_malloc_size_cb_func() calls set_killed()). And it caused a bunch of mutexes being locked under LOCK_thd_kill, which created problems later when my_malloc_size_cb_func() called set_killed() at some unspecified point under some random mutexes. --- .../perfschema/r/misc_session_status.result | 20 ++++++++++++++++++ .../perfschema/t/misc_session_status.test | 18 ++++++++++++++++ storage/perfschema/pfs_variable.cc | 21 ++++++++++++------- storage/perfschema/pfs_variable.h | 8 +++++-- 4 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 mysql-test/suite/perfschema/r/misc_session_status.result create mode 100644 mysql-test/suite/perfschema/t/misc_session_status.test diff --git a/mysql-test/suite/perfschema/r/misc_session_status.result b/mysql-test/suite/perfschema/r/misc_session_status.result new file mode 100644 index 00000000000..3ce472fc617 --- /dev/null +++ b/mysql-test/suite/perfschema/r/misc_session_status.result @@ -0,0 +1,20 @@ +# +# MDEV-33150 double-locking of LOCK_thd_kill in performance_schema.session_status +# +set @old_innodb_io_capacity=@@global.innodb_io_capacity; +set @old_innodb_io_capacity_max=@@global.innodb_io_capacity_max; +select * from performance_schema.session_status limit 0; +VARIABLE_NAME VARIABLE_VALUE +set max_session_mem_used=32768; +select * from performance_schema.session_status; +ERROR HY000: The MariaDB server is running with the --max-session-mem-used=32768 option so it cannot execute this statement +set global innodb_io_capacity_max=100; +Warnings: +Warning 1210 Setting innodb_io_capacity_max 100 lower than innodb_io_capacity 200. +Warning 1210 Setting innodb_io_capacity to 100 +set max_session_mem_used=default; +set global innodb_io_capacity=@old_innodb_io_capacity; +Warnings: +Warning 1210 Setting innodb_io_capacity to 200 higher than innodb_io_capacity_max 100 +Warning 1210 Setting innodb_max_io_capacity to 400 +set global innodb_io_capacity_max=@old_innodb_io_capacity_max; diff --git a/mysql-test/suite/perfschema/t/misc_session_status.test b/mysql-test/suite/perfschema/t/misc_session_status.test new file mode 100644 index 00000000000..ea662ce6738 --- /dev/null +++ b/mysql-test/suite/perfschema/t/misc_session_status.test @@ -0,0 +1,18 @@ +--source include/not_embedded.inc +--source include/have_perfschema.inc +--echo # +--echo # MDEV-33150 double-locking of LOCK_thd_kill in performance_schema.session_status +--echo # +source include/have_innodb.inc; +set @old_innodb_io_capacity=@@global.innodb_io_capacity; +set @old_innodb_io_capacity_max=@@global.innodb_io_capacity_max; +select * from performance_schema.session_status limit 0; # discover the table +set max_session_mem_used=32768; +--error ER_OPTION_PREVENTS_STATEMENT +# this used to crash, when OOM happened under LOCK_thd_kill +select * from performance_schema.session_status; +# this used to cause mutex lock order violation when OOM happened under LOCK_global_system_variables +set global innodb_io_capacity_max=100; +set max_session_mem_used=default; +set global innodb_io_capacity=@old_innodb_io_capacity; +set global innodb_io_capacity_max=@old_innodb_io_capacity_max; diff --git a/storage/perfschema/pfs_variable.cc b/storage/perfschema/pfs_variable.cc index aa07686139b..e2193cfb8a6 100644 --- a/storage/perfschema/pfs_variable.cc +++ b/storage/perfschema/pfs_variable.cc @@ -254,7 +254,8 @@ int PFS_system_variable_cache::do_materialize_all(THD *unsafe_thd) } /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -354,7 +355,8 @@ int PFS_system_variable_cache::do_materialize_session(PFS_thread *pfs_thread) } /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -407,7 +409,8 @@ int PFS_system_variable_cache::do_materialize_session(PFS_thread *pfs_thread, ui } /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -458,7 +461,8 @@ int PFS_system_variable_cache::do_materialize_session(THD *unsafe_thd) } /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -990,7 +994,8 @@ int PFS_status_variable_cache::do_materialize_all(THD* unsafe_thd) manifest(m_safe_thd, m_show_var_array.front(), status_vars, "", false, false); /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -1036,7 +1041,8 @@ int PFS_status_variable_cache::do_materialize_session(THD* unsafe_thd) manifest(m_safe_thd, m_show_var_array.front(), status_vars, "", false, true); /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; @@ -1078,7 +1084,8 @@ int PFS_status_variable_cache::do_materialize_session(PFS_thread *pfs_thread) manifest(m_safe_thd, m_show_var_array.front(), status_vars, "", false, true); /* Release lock taken in get_THD(). */ - mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); + if (m_safe_thd != current_thd) + mysql_mutex_unlock(&m_safe_thd->LOCK_thd_kill); m_materialized= true; ret= 0; diff --git a/storage/perfschema/pfs_variable.h b/storage/perfschema/pfs_variable.h index fda8dc692d9..7dc248269b3 100644 --- a/storage/perfschema/pfs_variable.h +++ b/storage/perfschema/pfs_variable.h @@ -211,8 +211,12 @@ public: if (thd != m_unsafe_thd) return false; - /* Hold this lock to keep THD during materialization. */ - mysql_mutex_lock(&thd->LOCK_thd_kill); + /* + Hold this lock to keep THD during materialization. + But don't lock current_thd (to be able to use set_killed() later + */ + if (thd != current_thd) + mysql_mutex_lock(&thd->LOCK_thd_kill); return true; } void set_unsafe_thd(THD *unsafe_thd) { m_unsafe_thd= unsafe_thd; } From 4cbf75dd3386088158b5858f7c5c0e1bb9498783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2024 07:51:37 +0200 Subject: [PATCH 08/10] MDEV-33137: Assertion end_lsn == page_lsn failed in recv_recover_page trx_purge_free_segment(), trx_purge_truncate_rseg_history(): Do not claim that the blocks will be modified in the mini-transaction, because that will not always be the case. Whenever there is a modification, mtr_t::set_modified() will flag it. The debug assertion that failed in recovery is checking that all changes to data pages are covered by log records. Due to these incorrect calls, we would unnecessarily write unmodified data pages, which is something that commit 05fa4558e0e82302ece981deabce764491464eb2 aims to avoid. The incorrect calls had originally been added in commit de31ca6a212118bd29876f964497050e446bda02 (MDEV-32820) and commit 86767bcc0f121db3ad83a74647a642754a0ce57f (MDEV-29593). Reviewed by: Vladislav Lesin Tested by: Elena Stepanova --- storage/innobase/trx/trx0purge.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index b500be548db..efbf2396991 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -378,8 +378,8 @@ static void trx_purge_free_segment(buf_block_t *rseg_hdr, buf_block_t *block, ut_ad(rseg_hdr->page.id() == rseg_hdr_id); block->page.lock.x_lock(); ut_ad(block->page.id() == id); - mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_MODIFY); - mtr.memo_push(block, MTR_MEMO_PAGE_X_MODIFY); + mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_FIX); + mtr.memo_push(block, MTR_MEMO_PAGE_X_FIX); } while (!fseg_free_step(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + @@ -502,7 +502,7 @@ loop: mtr.start(); rseg_hdr->page.lock.x_lock(); ut_ad(rseg_hdr->page.id() == rseg.page_id()); - mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_MODIFY); + mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_FIX); goto loop; } From 593278f92790ffefcd189c6c005e622b18d111ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2024 11:52:26 +0200 Subject: [PATCH 09/10] MDEV-32050 fixup: Remove srv_purge_rseg_truncate_frequency --- storage/innobase/handler/ha_innodb.cc | 4 +++- storage/innobase/include/srv0srv.h | 3 --- storage/innobase/srv/srv0srv.cc | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 9f255048394..e3b18599c99 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -19349,8 +19349,10 @@ static MYSQL_SYSVAR_ULONGLONG(max_undo_log_size, srv_max_undo_log_size, 10 << 20, 10 << 20, 1ULL << (32 + UNIV_PAGE_SIZE_SHIFT_MAX), 0); +static ulong innodb_purge_rseg_truncate_frequency; + static MYSQL_SYSVAR_ULONG(purge_rseg_truncate_frequency, - srv_purge_rseg_truncate_frequency, + innodb_purge_rseg_truncate_frequency, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_DEPRECATED, "Deprecated parameter with no effect", NULL, NULL, 128, 1, 128, 0); diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index ce91020464c..291969a9668 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -263,9 +263,6 @@ extern unsigned long long srv_max_undo_log_size; extern uint srv_n_fil_crypt_threads; extern uint srv_n_fil_crypt_threads_started; -/** Rate at which UNDO records should be purged. */ -extern ulong srv_purge_rseg_truncate_frequency; - /** Enable or Disable Truncate of UNDO tablespace. */ extern my_bool srv_undo_log_truncate; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 5fdcd91b63b..7923b14d69d 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -107,9 +107,6 @@ segment). It is quite possible that some of the tablespaces doesn't host any of the rollback-segment based on configuration used. */ ulint srv_undo_tablespaces_active; -/** Rate at which UNDO records should be purged. */ -ulong srv_purge_rseg_truncate_frequency; - /** Enable or Disable Truncate of UNDO tablespace. Note: If enabled then UNDO tablespace will be selected for truncate. While Server waits for undo-tablespace to truncate if user disables From 3613fb2aa85abd67313bd974253755358f430f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2024 11:53:00 +0200 Subject: [PATCH 10/10] MDEV-33112 innodb_undo_log_truncate=ON is blocking page write When innodb_undo_log_truncate=ON causes an InnoDB undo tablespace to be truncated, we must guarantee that the undo tablespace will be rebuilt atomically: After mtr_t::commit_shrink() has durably written the mini-transaction that rebuilds the undo tablespace, we must not write any old pages to the tablespace. To guarantee this, in trx_purge_truncate_history() we used to traverse the entire buf_pool.flush_list in order to acquire exclusive latches on all pages for the undo tablespace that reside in the buffer pool, so that those pages cannot be written and will be evicted during mtr_t::commit_shrink(). But, this traversal may interfere with the page writing activity of buf_flush_page_cleaner(). It would be better to lazily discard the old pages of the truncated undo tablespace. fil_space_t::is_being_truncated, fil_space_t::clear_stopping(): Remove. fil_space_t::create_lsn: A new field, identifying the LSN of the latest rebuild of a tablespace. buf_page_t::flush(), buf_flush_try_neighbors(): Evict pages whose FIL_PAGE_LSN is below fil_space_t::create_lsn. mtr_t::commit_shrink(): Update fil_space_t::create_lsn and fil_space_t::size right before the log is durably written and the tablespace file is being truncated. fsp_page_create(), trx_purge_truncate_history(): Simplify the logic. Reviewed by: Thirunarayanan Balathandayuthapani, Vladislav Lesin Performance tested by: Axel Schwenke Correctness tested by: Matthias Leich --- storage/innobase/buf/buf0flu.cc | 51 ++++++++----- storage/innobase/fil/fil0fil.cc | 15 ++-- storage/innobase/fsp/fsp0fsp.cc | 77 ++------------------ storage/innobase/include/fil0fil.h | 21 +++--- storage/innobase/include/mtr0mtr.h | 5 +- storage/innobase/include/srv0srv.h | 2 +- storage/innobase/mtr/mtr0mtr.cc | 38 ++++++---- storage/innobase/trx/trx0purge.cc | 111 ++++------------------------- 8 files changed, 94 insertions(+), 226 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index e886f0672e0..f07ecedf535 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -792,16 +792,20 @@ bool buf_page_t::flush(bool evict, fil_space_t *space) ut_ad(space->referenced()); const auto s= state(); - ut_a(s >= FREED); + + const lsn_t lsn= + mach_read_from_8(my_assume_aligned<8> + (FIL_PAGE_LSN + (zip.data ? zip.data : frame))); + ut_ad(lsn + ? lsn >= oldest_modification() || oldest_modification() == 2 + : space->purpose != FIL_TYPE_TABLESPACE); if (s < UNFIXED) { + ut_a(s >= FREED); if (UNIV_LIKELY(space->purpose == FIL_TYPE_TABLESPACE)) { - const lsn_t lsn= - mach_read_from_8(my_assume_aligned<8> - (FIL_PAGE_LSN + (zip.data ? zip.data : frame))); - ut_ad(lsn >= oldest_modification()); + freed: if (lsn > log_sys.get_flushed_lsn()) { mysql_mutex_unlock(&buf_pool.mutex); @@ -813,6 +817,12 @@ bool buf_page_t::flush(bool evict, fil_space_t *space) return false; } + if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + { + ut_ad(space->purpose == FIL_TYPE_TABLESPACE); + goto freed; + } + ut_d(const auto f=) zip.fix.fetch_add(WRITE_FIX - UNFIXED); ut_ad(f >= UNFIXED); ut_ad(f < READ_FIX); @@ -907,16 +917,9 @@ bool buf_page_t::flush(bool evict, fil_space_t *space) if ((s & LRU_MASK) == REINIT || !space->use_doublewrite()) { - if (UNIV_LIKELY(space->purpose == FIL_TYPE_TABLESPACE)) - { - const lsn_t lsn= - mach_read_from_8(my_assume_aligned<8>(FIL_PAGE_LSN + - (write_frame ? write_frame - : frame))); - ut_ad(lsn >= oldest_modification()); - if (lsn > log_sys.get_flushed_lsn()) - log_write_up_to(lsn, true); - } + if (UNIV_LIKELY(space->purpose == FIL_TYPE_TABLESPACE) && + lsn > log_sys.get_flushed_lsn()) + log_write_up_to(lsn, true); space->io(IORequest{type, this, slot}, physical_offset(), size, write_frame, this); } @@ -1096,11 +1099,25 @@ static ulint buf_flush_try_neighbors(fil_space_t *space, bool contiguous, bool evict, ulint n_flushed, ulint n_to_flush) { - mysql_mutex_unlock(&buf_pool.mutex); - ut_ad(space->id == page_id.space()); ut_ad(bpage->id() == page_id); + { + const lsn_t lsn= + mach_read_from_8(my_assume_aligned<8> + (FIL_PAGE_LSN + + (bpage->zip.data ? bpage->zip.data : bpage->frame))); + ut_ad(lsn >= bpage->oldest_modification()); + if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + { + ut_a(!bpage->flush(evict, space)); + mysql_mutex_unlock(&buf_pool.mutex); + return 0; + } + } + + mysql_mutex_unlock(&buf_pool.mutex); + ulint count= 0; page_id_t id= page_id; page_id_t high= buf_flush_check_neighbors(*space, id, contiguous, evict); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 35acb78ea3b..34748199c0d 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -558,7 +558,7 @@ fil_space_extend_must_retry( ut_ad(UT_LIST_GET_LAST(space->chain) == node); ut_ad(size >= FIL_IBD_FILE_INITIAL_SIZE); ut_ad(node->space == space); - ut_ad(space->referenced() || space->is_being_truncated); + ut_ad(space->referenced()); *success = space->size >= size; @@ -647,8 +647,7 @@ fil_space_extend_must_retry( default: ut_ad(space->purpose == FIL_TYPE_TABLESPACE || space->purpose == FIL_TYPE_IMPORT); - if (space->purpose == FIL_TYPE_TABLESPACE - && !space->is_being_truncated) { + if (space->purpose == FIL_TYPE_TABLESPACE) { goto do_flush; } break; @@ -733,12 +732,10 @@ bool fil_space_extend(fil_space_t *space, uint32_t size) bool success= false; const bool acquired= space->acquire(); mysql_mutex_lock(&fil_system.mutex); - if (acquired || space->is_being_truncated) - { + if (acquired) while (fil_space_extend_must_retry(space, UT_LIST_GET_LAST(space->chain), size, &success)) mysql_mutex_lock(&fil_system.mutex); - } mysql_mutex_unlock(&fil_system.mutex); if (acquired) space->release(); @@ -3058,11 +3055,9 @@ fil_space_validate_for_mtr_commit( ut_ad(!is_predefined_tablespace(space->id)); /* We are serving mtr_commit(). While there is an active - mini-transaction, we should have !space->stop_new_ops. This is + mini-transaction, we should have !space->is_stopping(). This is guaranteed by meta-data locks or transactional locks. */ - ut_ad(!space->is_stopping() - || space->is_being_truncated /* fil_truncate_prepare() */ - || space->referenced()); + ut_ad(!space->is_stopping() || space->referenced()); } #endif /* UNIV_DEBUG */ diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 93d558ebc4b..2c7188e4988 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1034,77 +1034,11 @@ static buf_block_t* fsp_page_create(fil_space_t *space, page_no_t offset, mtr_t *mtr) { - buf_block_t *block; - - if (UNIV_UNLIKELY(space->is_being_truncated)) - { - const page_id_t page_id{space->id, offset}; - uint32_t state; - block= mtr->get_already_latched(page_id, MTR_MEMO_PAGE_X_FIX); - if (block) - goto have_latch; - else - { - buf_pool_t::hash_chain &chain= - buf_pool.page_hash.cell_get(page_id.fold()); - mysql_mutex_lock(&buf_pool.mutex); - block= reinterpret_cast - (buf_pool.page_hash.get(page_id, chain)); - if (!block) - { - mysql_mutex_unlock(&buf_pool.mutex); - goto create; - } - } - - if (!mtr->have_x_latch(*block)) - { - const bool got{block->page.lock.x_lock_try()}; - mysql_mutex_unlock(&buf_pool.mutex); - if (!got) - { - block->page.lock.x_lock(); - const page_id_t id{block->page.id()}; - if (UNIV_UNLIKELY(id != page_id)) - { - ut_ad(id.is_corrupted()); - block->page.lock.x_unlock(); - goto create; - } - } - state= block->page.fix() + 1; - mtr->memo_push(block, MTR_MEMO_PAGE_X_FIX); - } - else - { - mysql_mutex_unlock(&buf_pool.mutex); - have_latch: - state= block->page.state(); - } - - ut_ad(state > buf_page_t::FREED); - ut_ad(state < buf_page_t::READ_FIX); - ut_ad((state & buf_page_t::LRU_MASK) != buf_page_t::IBUF_EXIST); - ut_ad(block->page.lock.x_lock_count() == 1); - ut_ad(block->page.frame); -#ifdef BTR_CUR_HASH_ADAPT - ut_ad(!block->index); -#endif - - block->page.set_reinit(state < buf_page_t::UNFIXED - ? buf_page_t::FREED - : (state & buf_page_t::LRU_MASK)); - } - else - { - create: - buf_block_t *free_block= buf_LRU_get_free_block(false); - block= buf_page_create(space, static_cast(offset), - space->zip_size(), mtr, free_block); - if (UNIV_UNLIKELY(block != free_block)) - buf_pool.free_block(free_block); - } - + buf_block_t *free_block= buf_LRU_get_free_block(false); + buf_block_t *block= buf_page_create(space, static_cast(offset), + space->zip_size(), mtr, free_block); + if (UNIV_UNLIKELY(block != free_block)) + buf_pool.free_block(free_block); fsp_init_file_page(space, block, mtr); return block; } @@ -1799,7 +1733,6 @@ page_alloc: ut_d(const auto x = block->page.lock.x_lock_count()); ut_ad(x || block->page.lock.not_recursive()); - ut_ad(x == 1 || space->is_being_truncated); ut_ad(x <= 2); ut_ad(!fil_page_get_type(block->page.frame)); mtr->write<1>(*block, FIL_PAGE_TYPE + 1 + block->page.frame, diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 8c822f5871c..5bebc304a27 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -362,8 +362,6 @@ struct fil_space_t final Protected by log_sys.mutex. If and only if this is nonzero, the tablespace will be in named_spaces. */ - /** whether undo tablespace truncation is in progress */ - bool is_being_truncated; fil_type_t purpose;/*!< purpose */ UT_LIST_BASE_NODE_T(fil_node_t) chain; /*!< base node for the file chain */ @@ -442,6 +440,8 @@ private: /** LSN of freeing last page; protected by freed_range_mutex */ lsn_t last_freed_lsn; + /** LSN of undo tablespace creation or 0; protected by latch */ + lsn_t create_lsn; public: /** @return whether doublewrite buffering is needed */ inline bool use_doublewrite() const; @@ -449,6 +449,12 @@ public: /** @return whether a page has been freed */ inline bool is_freed(uint32_t page); + /** Set create_lsn. */ + inline void set_create_lsn(lsn_t lsn); + + /** @return the latest tablespace rebuild LSN, or 0 */ + lsn_t get_create_lsn() const { return create_lsn; } + /** Apply freed_ranges to the file. @param writable whether the file is writable @return number of pages written or hole-punched */ @@ -526,9 +532,6 @@ public: /** Note that operations on the tablespace must stop. */ inline void set_stopping(); - /** Note that operations on the tablespace can resume after truncation */ - inline void clear_stopping(); - /** Drop the tablespace and wait for any pending operations to cease @param id tablespace identifier @param detached_handle pointer to file to be closed later, or nullptr @@ -1625,14 +1628,6 @@ inline void fil_space_t::set_stopping() #endif } -inline void fil_space_t::clear_stopping() -{ - mysql_mutex_assert_owner(&fil_system.mutex); - static_assert(STOPPING_WRITES == 1U << 30, "compatibility"); - ut_d(auto n=) n_pending.fetch_sub(STOPPING_WRITES, std::memory_order_relaxed); - ut_ad((n & STOPPING) == STOPPING_WRITES); -} - /** Flush pending writes from the file system cache to the file. */ template inline void fil_space_t::flush() { diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index dbd8ae76be0..3828a31cdb5 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -89,8 +89,9 @@ struct mtr_t { { auto s= m_memo.size(); rollback_to_savepoint(s - 1, s); } /** Commit a mini-transaction that is shrinking a tablespace. - @param space tablespace that is being shrunk */ - ATTRIBUTE_COLD void commit_shrink(fil_space_t &space); + @param space tablespace that is being shrunk + @param size new size in pages */ + ATTRIBUTE_COLD void commit_shrink(fil_space_t &space, uint32_t size); /** Commit a mini-transaction that is deleting or renaming a file. @param space tablespace that is being renamed or deleted diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 291969a9668..a45aa0d14dd 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -271,7 +271,7 @@ extern my_bool srv_undo_log_truncate; extern my_bool srv_prefix_index_cluster_optimization; /** Default size of UNDO tablespace (10MiB for innodb_page_size=16k) */ -constexpr ulint SRV_UNDO_TABLESPACE_SIZE_IN_PAGES= (10U << 20) / +constexpr uint32_t SRV_UNDO_TABLESPACE_SIZE_IN_PAGES= (10U << 20) / UNIV_PAGE_SIZE_DEF; extern char* srv_log_group_home_dir; diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 08adb9cfab8..6bdab147e20 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -258,9 +258,21 @@ void mtr_t::rollback_to_savepoint(ulint begin, ulint end) m_memo.erase(m_memo.begin() + begin, m_memo.begin() + end); } +/** Set create_lsn. */ +inline void fil_space_t::set_create_lsn(lsn_t lsn) +{ +#ifndef SUX_LOCK_GENERIC + ut_ad(latch.is_write_locked()); +#endif + /* Concurrent log_checkpoint_low() must be impossible. */ + mysql_mutex_assert_owner(&log_sys.mutex); + create_lsn= lsn; +} + /** Commit a mini-transaction that is shrinking a tablespace. -@param space tablespace that is being shrunk */ -void mtr_t::commit_shrink(fil_space_t &space) +@param space tablespace that is being shrunk +@param size new size in pages */ +void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) { ut_ad(is_active()); ut_ad(!is_inside_ibuf()); @@ -278,16 +290,23 @@ void mtr_t::commit_shrink(fil_space_t &space) const lsn_t start_lsn= do_write().first; ut_d(m_log.erase()); + fil_node_t *file= UT_LIST_GET_LAST(space.chain); mysql_mutex_lock(&log_sys.flush_order_mutex); + mysql_mutex_lock(&fil_system.mutex); + ut_ad(file->is_open()); + space.size= file->size= size; + space.set_create_lsn(m_commit_lsn); + mysql_mutex_unlock(&fil_system.mutex); + + space.clear_freed_ranges(); + /* Durably write the reduced FSP_SIZE before truncating the data file. */ log_write_and_flush(); os_file_truncate(space.chain.start->name, space.chain.start->handle, - os_offset_t{space.size} << srv_page_size_shift, true); + os_offset_t{size} << srv_page_size_shift, true); - space.clear_freed_ranges(); - - const page_id_t high{space.id, space.size}; + const page_id_t high{space.id, size}; for (mtr_memo_slot_t &slot : m_memo) { @@ -331,13 +350,6 @@ void mtr_t::commit_shrink(fil_space_t &space) mysql_mutex_unlock(&log_sys.flush_order_mutex); - mysql_mutex_lock(&fil_system.mutex); - ut_ad(space.is_being_truncated); - ut_ad(space.is_stopping_writes()); - space.clear_stopping(); - space.is_being_truncated= false; - mysql_mutex_unlock(&fil_system.mutex); - release(); release_resources(); srv_stats.log_write_requests.inc(); diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index efbf2396991..230036c8499 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -41,6 +41,7 @@ Created 3/26/1996 Heikki Tuuri #include "dict0load.h" #include #include +#include "log.h" /** Maximum allowable purge history length. <=0 means 'infinite'. */ ulong srv_max_purge_lag = 0; @@ -669,16 +670,9 @@ not_free: rseg.latch.rd_unlock(); } - ib::info() << "Truncating " << file->name; + sql_print_information("InnoDB: Truncating %s", file->name); trx_purge_cleanse_purge_queue(space); - log_free_check(); - - mtr_t mtr; - mtr.start(); - mtr.x_lock_space(&space); - const auto space_id= space.id; - /* Lock all modified pages of the tablespace. During truncation, we do not want any writes to the file. @@ -688,86 +682,12 @@ not_free: discarding the to-be-trimmed pages without flushing would break crash recovery. */ - rescan: if (UNIV_UNLIKELY(srv_shutdown_state != SRV_SHUTDOWN_NONE) && srv_fast_shutdown) - { - fast_shutdown: - mtr.commit(); return; - } - - mysql_mutex_lock(&buf_pool.flush_list_mutex); - for (buf_page_t *bpage= UT_LIST_GET_LAST(buf_pool.flush_list); bpage; ) - { - ut_ad(bpage->oldest_modification()); - ut_ad(bpage->in_file()); - - buf_page_t *prev= UT_LIST_GET_PREV(list, bpage); - - if (bpage->oldest_modification() > 2 && bpage->id().space() == space_id) - { - ut_ad(bpage->frame); - bpage->fix(); - { - /* Try to acquire an exclusive latch while the cache line is - fresh after fix(). */ - const bool got_lock{bpage->lock.x_lock_try()}; - buf_pool.flush_hp.set(prev); - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!got_lock) - bpage->lock.x_lock(); - } - -#ifdef BTR_CUR_HASH_ADAPT - /* There is no AHI on undo tablespaces. */ - ut_ad(!reinterpret_cast(bpage)->index); -#endif - ut_ad(!bpage->is_io_fixed()); - ut_ad(bpage->id().space() == space_id); - - if (bpage->oldest_modification() > 2 && - !mtr.have_x_latch(*reinterpret_cast(bpage))) - mtr.memo_push(reinterpret_cast(bpage), - MTR_MEMO_PAGE_X_FIX); - else - { - bpage->unfix(); - bpage->lock.x_unlock(); - } - - mysql_mutex_lock(&buf_pool.flush_list_mutex); - - if (prev != buf_pool.flush_hp.get()) - { - /* The functions buf_pool_t::release_freed_page() or - buf_do_flush_list_batch() may be right now holding - buf_pool.mutex and waiting to acquire - buf_pool.flush_list_mutex. Ensure that they can proceed, - to avoid extreme waits. */ - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - mysql_mutex_lock(&buf_pool.mutex); - mysql_mutex_unlock(&buf_pool.mutex); - goto rescan; - } - } - - bpage= prev; - } - - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - - if (UNIV_UNLIKELY(srv_shutdown_state != SRV_SHUTDOWN_NONE) && - srv_fast_shutdown) - goto fast_shutdown; - - /* Re-initialize tablespace, in a single mini-transaction. */ - const ulint size= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES; /* Adjust the tablespace metadata. */ mysql_mutex_lock(&fil_system.mutex); - space.set_stopping(); - space.is_being_truncated= true; if (space.crypt_data) { space.reacquire(); @@ -778,26 +698,20 @@ not_free: else mysql_mutex_unlock(&fil_system.mutex); - for (auto i= 6000; space.referenced(); - std::this_thread::sleep_for(std::chrono::milliseconds(10))) - { - if (!--i) - { - mtr.commit(); - ib::error() << "Failed to freeze UNDO tablespace " << file->name; - return; - } - } + /* Re-initialize tablespace, in a single mini-transaction. */ + const uint32_t size= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES; + log_free_check(); + + mtr_t mtr; + mtr.start(); + mtr.x_lock_space(&space); /* Associate the undo tablespace with mtr. During mtr::commit_shrink(), InnoDB can use the undo tablespace object to clear all freed ranges */ mtr.set_named_space(&space); mtr.trim_pages(page_id_t(space.id, size)); ut_a(fsp_header_init(&space, size, &mtr) == DB_SUCCESS); - mysql_mutex_lock(&fil_system.mutex); - space.size= file->size= size; - mysql_mutex_unlock(&fil_system.mutex); for (auto &rseg : trx_sys.rseg_array) { @@ -823,7 +737,7 @@ not_free: rseg.reinit(rblock->page.id().page_no()); } - mtr.commit_shrink(space); + mtr.commit_shrink(space, size); /* No mutex; this is only updated by the purge coordinator. */ export_vars.innodb_undo_truncations++; @@ -840,11 +754,12 @@ not_free: purge_sys.next_stored= false; } - DBUG_EXECUTE_IF("ib_undo_trunc", ib::info() << "ib_undo_trunc"; + DBUG_EXECUTE_IF("ib_undo_trunc", + sql_print_information("InnoDB: ib_undo_trunc"); log_buffer_flush_to_disk(); DBUG_SUICIDE();); - ib::info() << "Truncated " << file->name; + sql_print_information("InnoDB: Truncated %s", file->name); purge_sys.truncate.last= purge_sys.truncate.current; ut_ad(&space == purge_sys.truncate.current); purge_sys.truncate.current= nullptr;