mirror of
https://github.com/MariaDB/server.git
synced 2025-01-17 04:22:27 +01:00
33f15dc7ac
SHOW PROCESSLIST, SHOW BINLOGS Problem: A deadlock was occurring when 4 threads were involved in acquiring locks in the following way Thread 1: Dump thread ( Slave is reconnecting, so on Master, a new dump thread is trying kill zombie dump threads. It acquired thread's LOCK_thd_data and it is about to acquire mysys_var->current_mutex ( which LOCK_log) Thread 2: Application thread is executing show binlogs and acquired LOCK_log and it is about to acquire LOCK_index. Thread 3: Application thread is executing Purge binary logs and acquired LOCK_index and it is about to acquire LOCK_thread_count. Thread 4: Application thread is executing show processlist and acquired LOCK_thread_count and it is about to acquire zombie dump thread's LOCK_thd_data. Deadlock Cycle: Thread 1 -> Thread 2 -> Thread 3-> Thread 4 ->Thread 1 The same above deadlock was observed even when thread 4 is executing 'SELECT * FROM information_schema.processlist' command and acquired LOCK_thread_count and it is about to acquire zombie dump thread's LOCK_thd_data. Analysis: There are four locks involved in the deadlock. LOCK_log, LOCK_thread_count, LOCK_index and LOCK_thd_data. LOCK_log, LOCK_thread_count, LOCK_index are global mutexes where as LOCK_thd_data is local to a thread. We can divide these four locks in two groups. Group 1 consists of LOCK_log and LOCK_index and the order should be LOCK_log followed by LOCK_index. Group 2 consists of other two mutexes LOCK_thread_count, LOCK_thd_data and the order should be LOCK_thread_count followed by LOCK_thd_data. Unfortunately, there is no specific predefined lock order defined to follow in the MySQL system when it comes to locks across these two groups. In the above problematic example, there is no problem in the way we are acquiring the locks if you see each thread individually. But If you combine all 4 threads, they end up in a deadlock. Fix: Since everything seems to be fine in the way threads are taking locks, In this patch We are changing the duration of the locks in Thread 4 to break the deadlock. i.e., before the patch, Thread 4 ('show processlist' command) mysqld_list_processes() function acquires LOCK_thread_count for the complete duration of the function and it also acquires/releases each thread's LOCK_thd_data. LOCK_thread_count is used to protect addition and deletion of threads in global threads list. While show process list is looping through all the existing threads, it will be a problem if a thread is exited but there is no problem if a new thread is added to the system. Hence a new mutex is introduced "LOCK_thd_remove" which will protect deletion of a thread from global threads list. All threads which are getting exited should acquire LOCK_thd_remove followed by LOCK_thread_count. (It should take LOCK_thread_count also because other places of the code still thinks that exit thread is protected with LOCK_thread_count. In this fix, we are changing only 'show process list' query logic ) (Eg: unlink_thd logic will be protected with LOCK_thd_remove). Logic of mysqld_list_processes(or file_schema_processlist) will now be protected with 'LOCK_thd_remove' instead of 'LOCK_thread_count'. Now the new locking order after this patch is: LOCK_thd_remove -> LOCK_thd_data -> LOCK_log -> LOCK_index -> LOCK_thread_count
198 lines
5.3 KiB
C++
198 lines
5.3 KiB
C++
/* Copyright (c) 2007, 2014, Oracle and/or its affiliates. All rights reserved.
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; version 2 of the License.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
|
|
|
|
/*
|
|
Implementation for the thread scheduler
|
|
*/
|
|
|
|
#ifdef USE_PRAGMA_INTERFACE
|
|
#pragma implementation
|
|
#endif
|
|
|
|
#include <sql_priv.h>
|
|
#include "unireg.h" // REQUIRED: for other includes
|
|
#include "scheduler.h"
|
|
#include "sql_connect.h" // init_new_connection_handler_thread
|
|
#include "scheduler.h"
|
|
#include "sql_callback.h"
|
|
|
|
/*
|
|
End connection, in case when we are using 'no-threads'
|
|
*/
|
|
|
|
static bool no_threads_end(THD *thd, bool put_in_cache)
|
|
{
|
|
unlink_thd(thd);
|
|
mysql_mutex_unlock(&LOCK_thread_count);
|
|
mysql_mutex_unlock(&LOCK_thd_remove);
|
|
return 1; // Abort handle_one_connection
|
|
}
|
|
|
|
static scheduler_functions one_thread_scheduler_functions=
|
|
{
|
|
1, // max_threads
|
|
NULL, // init
|
|
init_new_connection_handler_thread, // init_new_connection_thread
|
|
#ifndef EMBEDDED_LIBRARY
|
|
handle_connection_in_main_thread, // add_connection
|
|
#else
|
|
NULL, // add_connection
|
|
#endif // EMBEDDED_LIBRARY
|
|
NULL, // thd_wait_begin
|
|
NULL, // thd_wait_end
|
|
NULL, // post_kill_notification
|
|
no_threads_end, // end_thread
|
|
NULL, // end
|
|
};
|
|
|
|
#ifndef EMBEDDED_LIBRARY
|
|
static scheduler_functions one_thread_per_connection_scheduler_functions=
|
|
{
|
|
0, // max_threads
|
|
NULL, // init
|
|
init_new_connection_handler_thread, // init_new_connection_thread
|
|
create_thread_to_handle_connection, // add_connection
|
|
NULL, // thd_wait_begin
|
|
NULL, // thd_wait_end
|
|
NULL, // post_kill_notification
|
|
one_thread_per_connection_end, // end_thread
|
|
NULL, // end
|
|
};
|
|
#endif // EMBEDDED_LIBRARY
|
|
|
|
|
|
scheduler_functions *thread_scheduler= NULL;
|
|
|
|
/** @internal
|
|
Helper functions to allow mysys to call the thread scheduler when
|
|
waiting for locks.
|
|
*/
|
|
|
|
/**@{*/
|
|
extern "C"
|
|
{
|
|
static void scheduler_wait_lock_begin(void) {
|
|
MYSQL_CALLBACK(thread_scheduler,
|
|
thd_wait_begin, (current_thd, THD_WAIT_TABLE_LOCK));
|
|
}
|
|
|
|
static void scheduler_wait_lock_end(void) {
|
|
MYSQL_CALLBACK(thread_scheduler, thd_wait_end, (current_thd));
|
|
}
|
|
|
|
static void scheduler_wait_sync_begin(void) {
|
|
MYSQL_CALLBACK(thread_scheduler,
|
|
thd_wait_begin, (current_thd, THD_WAIT_TABLE_LOCK));
|
|
}
|
|
|
|
static void scheduler_wait_sync_end(void) {
|
|
MYSQL_CALLBACK(thread_scheduler, thd_wait_end, (current_thd));
|
|
}
|
|
};
|
|
/**@}*/
|
|
|
|
/**
|
|
Common scheduler init function.
|
|
|
|
The scheduler is either initialized by calling
|
|
one_thread_scheduler() or one_thread_per_connection_scheduler() in
|
|
mysqld.cc, so this init function will always be called.
|
|
*/
|
|
static void scheduler_init() {
|
|
thr_set_lock_wait_callback(scheduler_wait_lock_begin,
|
|
scheduler_wait_lock_end);
|
|
thr_set_sync_wait_callback(scheduler_wait_sync_begin,
|
|
scheduler_wait_sync_end);
|
|
}
|
|
|
|
/*
|
|
Initialize scheduler for --thread-handling=one-thread-per-connection
|
|
*/
|
|
|
|
#ifndef EMBEDDED_LIBRARY
|
|
void one_thread_per_connection_scheduler()
|
|
{
|
|
scheduler_init();
|
|
one_thread_per_connection_scheduler_functions.max_threads= max_connections;
|
|
thread_scheduler= &one_thread_per_connection_scheduler_functions;
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
Initailize scheduler for --thread-handling=no-threads
|
|
*/
|
|
|
|
void one_thread_scheduler()
|
|
{
|
|
scheduler_init();
|
|
thread_scheduler= &one_thread_scheduler_functions;
|
|
}
|
|
|
|
|
|
/*
|
|
Initialize scheduler for --thread-handling=one-thread-per-connection
|
|
*/
|
|
|
|
/*
|
|
thd_scheduler keeps the link between THD and events.
|
|
It's embedded in the THD class.
|
|
*/
|
|
|
|
thd_scheduler::thd_scheduler()
|
|
: m_psi(NULL), data(NULL)
|
|
{
|
|
}
|
|
|
|
|
|
thd_scheduler::~thd_scheduler()
|
|
{
|
|
}
|
|
|
|
static scheduler_functions *saved_thread_scheduler;
|
|
static uint saved_thread_handling;
|
|
|
|
extern "C"
|
|
int my_thread_scheduler_set(scheduler_functions *scheduler)
|
|
{
|
|
DBUG_ASSERT(scheduler != 0);
|
|
|
|
if (scheduler == NULL)
|
|
return 1;
|
|
|
|
saved_thread_scheduler= thread_scheduler;
|
|
saved_thread_handling= thread_handling;
|
|
thread_scheduler= scheduler;
|
|
// Scheduler loaded dynamically
|
|
thread_handling= SCHEDULER_TYPES_COUNT;
|
|
return 0;
|
|
}
|
|
|
|
|
|
extern "C"
|
|
int my_thread_scheduler_reset()
|
|
{
|
|
DBUG_ASSERT(saved_thread_scheduler != NULL);
|
|
|
|
if (saved_thread_scheduler == NULL)
|
|
return 1;
|
|
|
|
thread_scheduler= saved_thread_scheduler;
|
|
thread_handling= saved_thread_handling;
|
|
saved_thread_scheduler= 0;
|
|
return 0;
|
|
}
|
|
|
|
|
|
|