2024-04-18 14:41:30 +02:00
|
|
|
/* Copyright 2018-2024 Codership Oy <info@codership.com>
|
2019-01-23 12:30:00 +01:00
|
|
|
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
|
|
it under the terms of the GNU General Public License as published by
|
|
|
|
the Free Software Foundation; version 2 of the License.
|
|
|
|
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
GNU General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
|
|
along with this program; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
|
|
|
|
#include "mariadb.h"
|
|
|
|
|
|
|
|
#include "mysql/service_wsrep.h"
|
|
|
|
#include "wsrep/key.hpp"
|
|
|
|
#include "wsrep_thd.h"
|
|
|
|
#include "wsrep_trans_observer.h"
|
|
|
|
#include "sql_class.h"
|
|
|
|
#include "debug_sync.h"
|
2019-03-15 06:09:13 +01:00
|
|
|
#include "log.h"
|
2019-01-23 12:30:00 +01:00
|
|
|
|
|
|
|
extern "C" my_bool wsrep_on(const THD *thd)
|
|
|
|
{
|
|
|
|
return my_bool(WSREP(thd));
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" void wsrep_thd_LOCK(const THD *thd)
|
|
|
|
{
|
|
|
|
mysql_mutex_lock(&thd->LOCK_thd_data);
|
|
|
|
}
|
|
|
|
|
MDEV-29293 MariaDB stuck on starting commit state
This commit contains a merge from 10.5-MDEV-29293-squash
into 10.6.
Although the bug MDEV-29293 was not reproducible with 10.6,
the fix contains several improvements for wsrep KILL query and
BF abort handling, and addresses the following issues:
* MDEV-30307 KILL command issued inside a transaction is
problematic for galera replication:
This commit will remove KILL TOI replication, so Galera side
transaction context is not lost during KILL.
* MDEV-21075 KILL QUERY maintains nodes data consistency but
breaks GTID sequence: This is fixed as well as KILL does not
use TOI, and thus does not change GTID state.
* MDEV-30372 Assertion in wsrep-lib state: This was caused by
BF abort or KILL when local transaction was in the middle
of group commit. This commit disables THD::killed handling
during commit, so the problem is avoided.
* MDEV-30963 Assertion failure !lock.was_chosen_as_deadlock_victim
in trx0trx.h:1065: The assertion happened when the victim was
BF aborted via MDL while it was committing. This commit changes
MDL BF aborts so that transactions which are committing cannot
be BF aborted via MDL. The RQG grammar attached in the issue
could not reproduce the crash anymore.
Original commit message from 10.5 fix:
MDEV-29293 MariaDB stuck on starting commit state
The problem seems to be a deadlock between KILL command execution
and BF abort issued by an applier, where:
* KILL has locked victim's LOCK_thd_kill and LOCK_thd_data.
* Applier has innodb side global lock mutex and victim trx mutex.
* KILL is calling innobase_kill_query, and is blocked by innodb
global lock mutex.
* Applier is in wsrep_innobase_kill_one_trx and is blocked by
victim's LOCK_thd_kill.
The fix in this commit removes the TOI replication of KILL command
and makes KILL execution less intrusive operation. Aborting the
victim happens now by using awake_no_mutex() and ha_abort_transaction().
If the KILL happens when the transaction is committing, the
KILL operation is postponed to happen after the statement
has completed in order to avoid KILL to interrupt commit
processing.
Notable changes in this commit:
* wsrep client connections's error state may remain sticky after
client connection is closed. This error message will then pop
up for the next client session issuing first SQL statement.
This problem raised with test galera.galera_bf_kill.
The fix is to reset wsrep client error state, before a THD is
reused for next connetion.
* Release THD locks in wsrep_abort_transaction when locking
innodb mutexes. This guarantees same locking order as with applier
BF aborting.
* BF abort from MDL was changed to do BF abort on server/wsrep-lib
side first, and only then do the BF abort on InnoDB side. This
removes the need to call back from InnoDB for BF aborts which originate
from MDL and simplifies the locking.
* Removed wsrep_thd_set_wsrep_aborter() from service_wsrep.h.
The manipulation of the wsrep_aborter can be done solely on
server side. Moreover, it is now debug only variable and
could be excluded from optimized builds.
* Remove LOCK_thd_kill from wsrep_thd_LOCK/UNLOCK to allow more
fine grained locking for SR BF abort which may require locking
of victim LOCK_thd_kill. Added explicit call for
wsrep_thd_kill_LOCK/UNLOCK where appropriate.
* Wsrep-lib was updated to version which allows external
locking for BF abort calls.
Changes to MTR tests:
* Disable galera_bf_abort_group_commit. This test is going to
be removed (MDEV-30855).
* Make galera_var_retry_autocommit result more readable by echoing
cases and expectations into result. Only one expected result for
reap to verify that server returns expected status for query.
* Record galera_gcache_recover_manytrx as result file was incomplete.
Trivial change.
* Make galera_create_table_as_select more deterministic:
Wait until CTAS execution has reached MDL wait for multi-master
conflict case. Expected error from multi-master conflict is
ER_QUERY_INTERRUPTED. This is because CTAS does not yet have open
wsrep transaction when it is waiting for MDL, query gets interrupted
instead of BF aborted. This should be addressed in separate task.
* A new test galera_bf_abort_registering to check that registering trx gets
BF aborted through MDL.
* A new test galera_kill_group_commit to verify correct behavior
when KILL is executed while the transaction is committing.
Co-authored-by: Seppo Jaakola <seppo.jaakola@iki.fi>
Co-authored-by: Jan Lindström <jan.lindstrom@galeracluster.com>
Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>
2023-04-19 15:51:55 +02:00
|
|
|
extern "C" int wsrep_thd_TRYLOCK(const THD *thd)
|
|
|
|
{
|
|
|
|
return mysql_mutex_trylock(&thd->LOCK_thd_data);
|
|
|
|
}
|
|
|
|
|
2019-01-23 12:30:00 +01:00
|
|
|
extern "C" void wsrep_thd_UNLOCK(const THD *thd)
|
|
|
|
{
|
|
|
|
mysql_mutex_unlock(&thd->LOCK_thd_data);
|
|
|
|
}
|
|
|
|
|
2021-02-12 17:44:22 +01:00
|
|
|
extern "C" void wsrep_thd_kill_LOCK(const THD *thd)
|
|
|
|
{
|
|
|
|
mysql_mutex_lock(&thd->LOCK_thd_kill);
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" void wsrep_thd_kill_UNLOCK(const THD *thd)
|
|
|
|
{
|
|
|
|
mysql_mutex_unlock(&thd->LOCK_thd_kill);
|
|
|
|
}
|
|
|
|
|
2019-01-23 12:30:00 +01:00
|
|
|
extern "C" const char* wsrep_thd_client_state_str(const THD *thd)
|
|
|
|
{
|
|
|
|
return wsrep::to_c_string(thd->wsrep_cs().state());
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" const char* wsrep_thd_client_mode_str(const THD *thd)
|
|
|
|
{
|
|
|
|
return wsrep::to_c_string(thd->wsrep_cs().mode());
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" const char* wsrep_thd_transaction_state_str(const THD *thd)
|
|
|
|
{
|
|
|
|
return wsrep::to_c_string(thd->wsrep_cs().transaction().state());
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" const char *wsrep_thd_query(const THD *thd)
|
|
|
|
{
|
2019-10-02 12:24:34 +02:00
|
|
|
if (!thd)
|
|
|
|
return "NULL";
|
|
|
|
|
|
|
|
switch(thd->lex->sql_command)
|
2019-11-01 14:23:18 +01:00
|
|
|
{
|
2019-10-02 12:24:34 +02:00
|
|
|
// Mask away some security related details from error log
|
2019-11-01 14:23:18 +01:00
|
|
|
case SQLCOM_CREATE_USER:
|
|
|
|
return "CREATE USER";
|
|
|
|
case SQLCOM_GRANT:
|
|
|
|
return "GRANT";
|
|
|
|
case SQLCOM_REVOKE:
|
|
|
|
return "REVOKE";
|
|
|
|
case SQLCOM_SET_OPTION:
|
|
|
|
if (thd->lex->definer)
|
2019-10-02 12:24:34 +02:00
|
|
|
return "SET PASSWORD";
|
2019-11-01 14:23:18 +01:00
|
|
|
/* fallthrough */
|
|
|
|
default:
|
2019-10-02 12:24:34 +02:00
|
|
|
return (thd->query() ? thd->query() : "NULL");
|
2019-11-01 14:23:18 +01:00
|
|
|
}
|
|
|
|
return "NULL";
|
2019-01-23 12:30:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" query_id_t wsrep_thd_transaction_id(const THD *thd)
|
|
|
|
{
|
|
|
|
return thd->wsrep_cs().transaction().id().get();
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" long long wsrep_thd_trx_seqno(const THD *thd)
|
|
|
|
{
|
|
|
|
const wsrep::client_state& cs= thd->wsrep_cs();
|
|
|
|
if (cs.mode() == wsrep::client_state::m_toi)
|
|
|
|
{
|
|
|
|
return cs.toi_meta().seqno().get();
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
return cs.transaction().ws_meta().seqno().get();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" void wsrep_thd_self_abort(THD *thd)
|
|
|
|
{
|
|
|
|
thd->wsrep_cs().bf_abort(wsrep::seqno(0));
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" const char* wsrep_get_sr_table_name()
|
|
|
|
{
|
|
|
|
return wsrep_sr_table_name_full;
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" my_bool wsrep_get_debug()
|
|
|
|
{
|
|
|
|
return wsrep_debug;
|
|
|
|
}
|
|
|
|
|
2021-04-14 22:40:46 +02:00
|
|
|
/*
|
|
|
|
Test if this connection is a true local (user) connection and not
|
|
|
|
a replication or wsrep applier thread.
|
|
|
|
|
|
|
|
Note that this is only usable for galera (as there are other kinds
|
|
|
|
of system threads, and only if WSREP_NNULL() is tested by the caller.
|
|
|
|
*/
|
2019-01-23 12:30:00 +01:00
|
|
|
extern "C" my_bool wsrep_thd_is_local(const THD *thd)
|
|
|
|
{
|
MDEV-21096 async slave crash with gtid_log_pos table access (#1413)
The original crash happened when async replication IO thread was updating mysql.gtid_slave_pos table. Operations on this table should remain node local, but it appears that protection (THD::wsrep_ignore_table flag) to prevent wsrep replication for this table mas missing for innodb write_row() and update_row().
It was somewhat difficult to reproduce the issue, because mtr seems to create the affected table mysql.gtid_log_pos as of Aria engine type, and Aria engine operations will not be replicated anyhow. It looks, though, that in release installation, mysql.gtid_slave_pos table is of InnoDB engine.
It was possible to trigger somewhat related problem by running test galera.galera_as_slave_gtid with configuration: gtid_pos_auto_engines=InnoDB. However, this test mode, causes earlier crash when replication background thread creates aditional table: mysql.gtid_slave_pos_InnoDB, and this table create triggered wsrep TOI replication, which also failed for assertion. Actually, async replication IO and background threads should not replicate anything to cluster.
This pull request contains new test galera.galera_as_slave_gtid_auto_engine, which basically just runs galera.galera_as_slave_gtid with configuration of gtid_pos_auto_engines=InnoDB.
Test galera.galera_as_slave_gtid is also modified for better code reuse.
Actual fix for MDEV-21096 is in storage/innobase/handler/ha_innodb.cc, where THD::wsrep_ignore_table flag is now honored before wsrep key population.
There is additional fix in sql/service_wsrep.cc where async replication IO and background threads are marked as non-local. This fences these threads out of wsrep replication altogether. Note that this change, actually makes the use of THD::wsrep_ignore-table redundant. We may want to refactor THD::wsrep_ignore_table out in the future, if there is no other use case for it in sight.
2019-11-25 10:19:33 +01:00
|
|
|
/*
|
2021-04-14 22:40:46 +02:00
|
|
|
async replication IO and background threads have nothing to
|
|
|
|
replicate in the cluster, marking them as non-local here to
|
|
|
|
prevent write set population and replication
|
MDEV-21096 async slave crash with gtid_log_pos table access (#1413)
The original crash happened when async replication IO thread was updating mysql.gtid_slave_pos table. Operations on this table should remain node local, but it appears that protection (THD::wsrep_ignore_table flag) to prevent wsrep replication for this table mas missing for innodb write_row() and update_row().
It was somewhat difficult to reproduce the issue, because mtr seems to create the affected table mysql.gtid_log_pos as of Aria engine type, and Aria engine operations will not be replicated anyhow. It looks, though, that in release installation, mysql.gtid_slave_pos table is of InnoDB engine.
It was possible to trigger somewhat related problem by running test galera.galera_as_slave_gtid with configuration: gtid_pos_auto_engines=InnoDB. However, this test mode, causes earlier crash when replication background thread creates aditional table: mysql.gtid_slave_pos_InnoDB, and this table create triggered wsrep TOI replication, which also failed for assertion. Actually, async replication IO and background threads should not replicate anything to cluster.
This pull request contains new test galera.galera_as_slave_gtid_auto_engine, which basically just runs galera.galera_as_slave_gtid with configuration of gtid_pos_auto_engines=InnoDB.
Test galera.galera_as_slave_gtid is also modified for better code reuse.
Actual fix for MDEV-21096 is in storage/innobase/handler/ha_innodb.cc, where THD::wsrep_ignore_table flag is now honored before wsrep key population.
There is additional fix in sql/service_wsrep.cc where async replication IO and background threads are marked as non-local. This fences these threads out of wsrep replication altogether. Note that this change, actually makes the use of THD::wsrep_ignore-table redundant. We may want to refactor THD::wsrep_ignore_table out in the future, if there is no other use case for it in sight.
2019-11-25 10:19:33 +01:00
|
|
|
|
2021-04-14 22:40:46 +02:00
|
|
|
async replication SQL thread, applies client transactions from
|
|
|
|
mariadb master and will be replicated into cluster
|
|
|
|
*/
|
MDEV-21096 async slave crash with gtid_log_pos table access (#1413)
The original crash happened when async replication IO thread was updating mysql.gtid_slave_pos table. Operations on this table should remain node local, but it appears that protection (THD::wsrep_ignore_table flag) to prevent wsrep replication for this table mas missing for innodb write_row() and update_row().
It was somewhat difficult to reproduce the issue, because mtr seems to create the affected table mysql.gtid_log_pos as of Aria engine type, and Aria engine operations will not be replicated anyhow. It looks, though, that in release installation, mysql.gtid_slave_pos table is of InnoDB engine.
It was possible to trigger somewhat related problem by running test galera.galera_as_slave_gtid with configuration: gtid_pos_auto_engines=InnoDB. However, this test mode, causes earlier crash when replication background thread creates aditional table: mysql.gtid_slave_pos_InnoDB, and this table create triggered wsrep TOI replication, which also failed for assertion. Actually, async replication IO and background threads should not replicate anything to cluster.
This pull request contains new test galera.galera_as_slave_gtid_auto_engine, which basically just runs galera.galera_as_slave_gtid with configuration of gtid_pos_auto_engines=InnoDB.
Test galera.galera_as_slave_gtid is also modified for better code reuse.
Actual fix for MDEV-21096 is in storage/innobase/handler/ha_innodb.cc, where THD::wsrep_ignore_table flag is now honored before wsrep key population.
There is additional fix in sql/service_wsrep.cc where async replication IO and background threads are marked as non-local. This fences these threads out of wsrep replication altogether. Note that this change, actually makes the use of THD::wsrep_ignore-table redundant. We may want to refactor THD::wsrep_ignore_table out in the future, if there is no other use case for it in sight.
2019-11-25 10:19:33 +01:00
|
|
|
return (
|
|
|
|
thd->system_thread != SYSTEM_THREAD_SLAVE_BACKGROUND &&
|
|
|
|
thd->system_thread != SYSTEM_THREAD_SLAVE_IO &&
|
|
|
|
thd->wsrep_cs().mode() == wsrep::client_state::m_local);
|
2019-01-23 12:30:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" my_bool wsrep_thd_is_applying(const THD *thd)
|
|
|
|
{
|
|
|
|
return thd->wsrep_cs().mode() == wsrep::client_state::m_high_priority;
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" my_bool wsrep_thd_is_toi(const THD *thd)
|
|
|
|
{
|
|
|
|
return thd->wsrep_cs().mode() == wsrep::client_state::m_toi;
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" my_bool wsrep_thd_is_local_toi(const THD *thd)
|
|
|
|
{
|
|
|
|
return thd->wsrep_cs().mode() == wsrep::client_state::m_toi &&
|
|
|
|
thd->wsrep_cs().toi_mode() == wsrep::client_state::m_local;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" my_bool wsrep_thd_is_in_rsu(const THD *thd)
|
|
|
|
{
|
|
|
|
return thd->wsrep_cs().mode() == wsrep::client_state::m_rsu;
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" my_bool wsrep_thd_is_BF(const THD *thd, my_bool sync)
|
|
|
|
{
|
|
|
|
my_bool status = FALSE;
|
|
|
|
if (thd && WSREP(thd))
|
|
|
|
{
|
|
|
|
if (sync) mysql_mutex_lock(&thd->LOCK_thd_data);
|
|
|
|
status = (wsrep_thd_is_applying(thd) || wsrep_thd_is_toi(thd));
|
|
|
|
if (sync) mysql_mutex_unlock(&thd->LOCK_thd_data);
|
|
|
|
}
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" my_bool wsrep_thd_is_SR(const THD *thd)
|
|
|
|
{
|
MDEV-34836: TOI on parent table must BF abort SR in progress on a child
Applied SR transaction on the child table was not BF aborted by TOI running
on the parent table for several reasons:
Although SR correctly collected FK-referenced keys to parent, TOI in Galera
disregards common certification index and simply sets itself to depend on
the latest certified write set seqno.
Since this write set was the fragment of SR transaction, TOI was allowed to
run in parallel with SR presuming it would BF abort the latter.
At the same time, DML transactions in the server don't grab MDL locks on
FK-referenced tables, thus parent table wasn't protected by an MDL lock from
SR and it couldn't provoke MDL lock conflict for TOI to BF abort SR transaction.
In InnoDB, DDL transactions grab shared MDL locks on child tables, which is not
enough to trigger MDL conflict in Galera.
InnoDB-level Wsrep patch didn't contain correct conflict resolution logic due to
the fact that it was believed MDL locking should always produce conflicts correctly.
The fix brings conflict resolution rules similar to MDL-level checks to InnoDB,
thus accounting for the problematic case.
Apart from that, wsrep_thd_is_SR() is patched to return true only for executing
SR transactions. It should be safe as any other SR state is either the same as
for any single write set (thus making the two logically equivalent), or it reflects
an SR transaction as being aborting or prepared, which is handled separately in
BF-aborting logic, and for regular execution path it should not matter at all.
Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>
2024-09-09 12:30:42 +02:00
|
|
|
return thd && thd->wsrep_cs().transaction().is_streaming() &&
|
|
|
|
thd->wsrep_cs().transaction().state() == wsrep::transaction::s_executing;
|
2019-01-23 12:30:00 +01:00
|
|
|
}
|
|
|
|
|
2024-11-07 08:04:16 +01:00
|
|
|
extern "C" void wsrep_handle_SR_rollback(THD *bf_thd __attribute__((unused)),
|
2019-01-23 12:30:00 +01:00
|
|
|
THD *victim_thd)
|
|
|
|
{
|
2024-11-07 08:04:16 +01:00
|
|
|
/*
|
|
|
|
We should always be in victim_thd context, either client session is
|
|
|
|
rolling back or rollbacker thread should be in control.
|
|
|
|
*/
|
2019-01-23 12:30:00 +01:00
|
|
|
DBUG_ASSERT(victim_thd);
|
2024-11-07 08:04:16 +01:00
|
|
|
DBUG_ASSERT(current_thd == victim_thd);
|
2020-02-24 17:04:43 +01:00
|
|
|
DBUG_ASSERT(wsrep_thd_is_SR(victim_thd));
|
2019-01-23 12:30:00 +01:00
|
|
|
|
2024-11-07 08:04:16 +01:00
|
|
|
/* Defensive measure to avoid crash in production. */
|
|
|
|
if (!victim_thd) return;
|
2021-10-21 13:49:51 +02:00
|
|
|
|
2024-11-07 08:04:16 +01:00
|
|
|
WSREP_DEBUG("Handle SR rollback, for deadlock: thd %llu trx_id %" PRIu64 " frags %zu conf %s",
|
2019-01-23 12:30:00 +01:00
|
|
|
victim_thd->thread_id,
|
|
|
|
victim_thd->wsrep_trx_id(),
|
|
|
|
victim_thd->wsrep_sr().fragments_certified(),
|
|
|
|
wsrep_thd_transaction_state_str(victim_thd));
|
2019-08-30 07:42:24 +02:00
|
|
|
|
2024-11-07 08:04:16 +01:00
|
|
|
DEBUG_SYNC(victim_thd, "wsrep_before_SR_rollback");
|
2021-10-21 13:49:51 +02:00
|
|
|
|
2024-11-07 08:04:16 +01:00
|
|
|
wsrep_thd_self_abort(victim_thd);
|
2019-01-23 12:30:00 +01:00
|
|
|
}
|
|
|
|
|
2019-12-04 08:21:14 +01:00
|
|
|
extern "C" my_bool wsrep_thd_bf_abort(THD *bf_thd, THD *victim_thd,
|
2019-01-23 12:30:00 +01:00
|
|
|
my_bool signal)
|
|
|
|
{
|
2021-02-07 17:48:58 +01:00
|
|
|
mysql_mutex_assert_owner(&victim_thd->LOCK_thd_kill);
|
MDEV-29293 MariaDB stuck on starting commit state
This commit contains a merge from 10.5-MDEV-29293-squash
into 10.6.
Although the bug MDEV-29293 was not reproducible with 10.6,
the fix contains several improvements for wsrep KILL query and
BF abort handling, and addresses the following issues:
* MDEV-30307 KILL command issued inside a transaction is
problematic for galera replication:
This commit will remove KILL TOI replication, so Galera side
transaction context is not lost during KILL.
* MDEV-21075 KILL QUERY maintains nodes data consistency but
breaks GTID sequence: This is fixed as well as KILL does not
use TOI, and thus does not change GTID state.
* MDEV-30372 Assertion in wsrep-lib state: This was caused by
BF abort or KILL when local transaction was in the middle
of group commit. This commit disables THD::killed handling
during commit, so the problem is avoided.
* MDEV-30963 Assertion failure !lock.was_chosen_as_deadlock_victim
in trx0trx.h:1065: The assertion happened when the victim was
BF aborted via MDL while it was committing. This commit changes
MDL BF aborts so that transactions which are committing cannot
be BF aborted via MDL. The RQG grammar attached in the issue
could not reproduce the crash anymore.
Original commit message from 10.5 fix:
MDEV-29293 MariaDB stuck on starting commit state
The problem seems to be a deadlock between KILL command execution
and BF abort issued by an applier, where:
* KILL has locked victim's LOCK_thd_kill and LOCK_thd_data.
* Applier has innodb side global lock mutex and victim trx mutex.
* KILL is calling innobase_kill_query, and is blocked by innodb
global lock mutex.
* Applier is in wsrep_innobase_kill_one_trx and is blocked by
victim's LOCK_thd_kill.
The fix in this commit removes the TOI replication of KILL command
and makes KILL execution less intrusive operation. Aborting the
victim happens now by using awake_no_mutex() and ha_abort_transaction().
If the KILL happens when the transaction is committing, the
KILL operation is postponed to happen after the statement
has completed in order to avoid KILL to interrupt commit
processing.
Notable changes in this commit:
* wsrep client connections's error state may remain sticky after
client connection is closed. This error message will then pop
up for the next client session issuing first SQL statement.
This problem raised with test galera.galera_bf_kill.
The fix is to reset wsrep client error state, before a THD is
reused for next connetion.
* Release THD locks in wsrep_abort_transaction when locking
innodb mutexes. This guarantees same locking order as with applier
BF aborting.
* BF abort from MDL was changed to do BF abort on server/wsrep-lib
side first, and only then do the BF abort on InnoDB side. This
removes the need to call back from InnoDB for BF aborts which originate
from MDL and simplifies the locking.
* Removed wsrep_thd_set_wsrep_aborter() from service_wsrep.h.
The manipulation of the wsrep_aborter can be done solely on
server side. Moreover, it is now debug only variable and
could be excluded from optimized builds.
* Remove LOCK_thd_kill from wsrep_thd_LOCK/UNLOCK to allow more
fine grained locking for SR BF abort which may require locking
of victim LOCK_thd_kill. Added explicit call for
wsrep_thd_kill_LOCK/UNLOCK where appropriate.
* Wsrep-lib was updated to version which allows external
locking for BF abort calls.
Changes to MTR tests:
* Disable galera_bf_abort_group_commit. This test is going to
be removed (MDEV-30855).
* Make galera_var_retry_autocommit result more readable by echoing
cases and expectations into result. Only one expected result for
reap to verify that server returns expected status for query.
* Record galera_gcache_recover_manytrx as result file was incomplete.
Trivial change.
* Make galera_create_table_as_select more deterministic:
Wait until CTAS execution has reached MDL wait for multi-master
conflict case. Expected error from multi-master conflict is
ER_QUERY_INTERRUPTED. This is because CTAS does not yet have open
wsrep transaction when it is waiting for MDL, query gets interrupted
instead of BF aborted. This should be addressed in separate task.
* A new test galera_bf_abort_registering to check that registering trx gets
BF aborted through MDL.
* A new test galera_kill_group_commit to verify correct behavior
when KILL is executed while the transaction is committing.
Co-authored-by: Seppo Jaakola <seppo.jaakola@iki.fi>
Co-authored-by: Jan Lindström <jan.lindstrom@galeracluster.com>
Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>
2023-04-19 15:51:55 +02:00
|
|
|
mysql_mutex_assert_owner(&victim_thd->LOCK_thd_data);
|
2019-01-23 12:30:00 +01:00
|
|
|
my_bool ret= wsrep_bf_abort(bf_thd, victim_thd);
|
|
|
|
/*
|
|
|
|
Send awake signal if victim was BF aborted or does not
|
|
|
|
have wsrep on. Note that this should never interrupt RSU
|
|
|
|
as RSU has paused the provider.
|
|
|
|
*/
|
|
|
|
if ((ret || !wsrep_on(victim_thd)) && signal)
|
2020-05-14 08:17:14 +02:00
|
|
|
{
|
2020-05-19 10:12:26 +02:00
|
|
|
victim_thd->wsrep_aborter= bf_thd->thread_id;
|
2023-01-16 17:02:16 +01:00
|
|
|
victim_thd->awake_no_mutex(KILL_QUERY_HARD);
|
2020-05-19 10:12:26 +02:00
|
|
|
} else {
|
2022-06-17 14:16:23 +02:00
|
|
|
WSREP_DEBUG("wsrep_thd_bf_abort skipped awake, signal %d", signal);
|
2020-05-14 08:17:14 +02:00
|
|
|
}
|
2019-01-23 12:30:00 +01:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" my_bool wsrep_thd_skip_locking(const THD *thd)
|
|
|
|
{
|
|
|
|
return thd && thd->wsrep_skip_locking;
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" my_bool wsrep_thd_order_before(const THD *left, const THD *right)
|
|
|
|
{
|
2024-05-13 07:55:36 +02:00
|
|
|
my_bool before= (wsrep_thd_is_BF(left, false) &&
|
|
|
|
wsrep_thd_is_BF(right, false) &&
|
|
|
|
wsrep_thd_trx_seqno(left) < wsrep_thd_trx_seqno(right));
|
|
|
|
|
|
|
|
WSREP_DEBUG("wsrep_thd_order_before: %s thread=%llu seqno=%llu query=%s "
|
|
|
|
"%s %s thread=%llu, seqno=%llu query=%s",
|
|
|
|
(wsrep_thd_is_BF(left, false) ? "BF" : "def"),
|
|
|
|
thd_get_thread_id(left),
|
|
|
|
wsrep_thd_trx_seqno(left),
|
|
|
|
wsrep_thd_query(left),
|
|
|
|
(before ? " TRUE " : " FALSE "),
|
|
|
|
(wsrep_thd_is_BF(right, false) ? "BF" : "def"),
|
|
|
|
thd_get_thread_id(right),
|
|
|
|
wsrep_thd_trx_seqno(right),
|
|
|
|
wsrep_thd_query(right));
|
|
|
|
|
|
|
|
return before;
|
2019-01-23 12:30:00 +01:00
|
|
|
}
|
|
|
|
|
2024-04-18 14:41:30 +02:00
|
|
|
/** Check if wsrep transaction is aborting state.
|
|
|
|
|
|
|
|
Calling function should make sure that wsrep transaction state
|
|
|
|
can't change during this function.
|
|
|
|
|
|
|
|
This function is called from
|
|
|
|
wsrep_abort_thd where we hold THD::LOCK_thd_data
|
|
|
|
wsrep_handle_mdl_conflict we hold THD::LOCK_thd_data
|
|
|
|
wsrep_assert_no_bf_bf_wait we hold lock_sys.latch
|
|
|
|
innobase_kill_query we hold THD::LOCK_thd_data (THD::awake_no_mutex)
|
|
|
|
|
|
|
|
@param thd thread handle
|
|
|
|
|
|
|
|
@return true if wsrep transaction is aborting
|
|
|
|
@return false if not
|
|
|
|
|
|
|
|
*/
|
2019-01-23 12:30:00 +01:00
|
|
|
extern "C" my_bool wsrep_thd_is_aborting(const MYSQL_THD thd)
|
|
|
|
{
|
2021-09-17 14:20:02 +02:00
|
|
|
const wsrep::client_state& cs(thd->wsrep_cs());
|
|
|
|
const enum wsrep::transaction::state tx_state(cs.transaction().state());
|
2024-04-18 14:41:30 +02:00
|
|
|
|
2021-09-17 14:20:02 +02:00
|
|
|
switch (tx_state)
|
2019-01-23 12:30:00 +01:00
|
|
|
{
|
|
|
|
case wsrep::transaction::s_must_abort:
|
|
|
|
return (cs.state() == wsrep::client_state::s_exec ||
|
|
|
|
cs.state() == wsrep::client_state::s_result);
|
|
|
|
case wsrep::transaction::s_aborting:
|
|
|
|
return true;
|
|
|
|
default:
|
2024-04-18 14:41:30 +02:00
|
|
|
break;
|
2019-01-23 12:30:00 +01:00
|
|
|
}
|
2021-09-17 14:20:02 +02:00
|
|
|
|
2019-01-23 12:30:00 +01:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline enum wsrep::key::type
|
|
|
|
map_key_type(enum Wsrep_service_key_type type)
|
|
|
|
{
|
|
|
|
switch (type)
|
|
|
|
{
|
|
|
|
case WSREP_SERVICE_KEY_SHARED: return wsrep::key::shared;
|
|
|
|
case WSREP_SERVICE_KEY_REFERENCE: return wsrep::key::reference;
|
|
|
|
case WSREP_SERVICE_KEY_UPDATE: return wsrep::key::update;
|
|
|
|
case WSREP_SERVICE_KEY_EXCLUSIVE: return wsrep::key::exclusive;
|
|
|
|
}
|
|
|
|
return wsrep::key::exclusive;
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" int wsrep_thd_append_key(THD *thd,
|
|
|
|
const struct wsrep_key* key,
|
|
|
|
int n_keys,
|
|
|
|
enum Wsrep_service_key_type key_type)
|
|
|
|
{
|
|
|
|
Wsrep_client_state& client_state(thd->wsrep_cs());
|
|
|
|
DBUG_ASSERT(client_state.transaction().active());
|
|
|
|
int ret= 0;
|
|
|
|
for (int i= 0; i < n_keys && ret == 0; ++i)
|
|
|
|
{
|
|
|
|
wsrep::key wsrep_key(map_key_type(key_type));
|
|
|
|
for (size_t kp= 0; kp < key[i].key_parts_num; ++kp)
|
|
|
|
{
|
|
|
|
wsrep_key.append_key_part(key[i].key_parts[kp].ptr, key[i].key_parts[kp].len);
|
|
|
|
}
|
|
|
|
ret= client_state.append_key(wsrep_key);
|
|
|
|
}
|
2019-04-01 13:23:05 +02:00
|
|
|
/*
|
|
|
|
In case of `wsrep_gtid_mode` when WS will be replicated, we need to set
|
|
|
|
`server_id` for events that are going to be written in IO, and in case of
|
|
|
|
manual SET gtid_seq_no=X we are ignoring value.
|
|
|
|
*/
|
|
|
|
if (!ret && wsrep_gtid_mode && !thd->slave_thread && !wsrep_thd_is_applying(thd))
|
|
|
|
{
|
|
|
|
thd->variables.server_id= wsrep_gtid_server.server_id;
|
|
|
|
thd->variables.gtid_seq_no= 0;
|
|
|
|
}
|
2019-01-23 12:30:00 +01:00
|
|
|
return ret;
|
|
|
|
}
|
2019-03-15 06:09:13 +01:00
|
|
|
|
|
|
|
extern "C" void wsrep_commit_ordered(THD *thd)
|
|
|
|
{
|
|
|
|
if (wsrep_is_active(thd) &&
|
2019-04-01 13:23:05 +02:00
|
|
|
(thd->wsrep_trx().state() == wsrep::transaction::s_committing ||
|
|
|
|
thd->wsrep_trx().state() == wsrep::transaction::s_ordered_commit))
|
2019-03-15 06:09:13 +01:00
|
|
|
{
|
2019-04-01 13:23:05 +02:00
|
|
|
wsrep_gtid_server.signal_waiters(thd->wsrep_current_gtid_seqno, false);
|
|
|
|
if (wsrep_thd_is_local(thd))
|
|
|
|
{
|
|
|
|
thd->wsrep_last_written_gtid_seqno= thd->wsrep_current_gtid_seqno;
|
|
|
|
}
|
2020-06-23 14:19:36 +02:00
|
|
|
if (thd->wsrep_trx().state() != wsrep::transaction::s_ordered_commit &&
|
|
|
|
!wsrep_commit_will_write_binlog(thd))
|
2019-04-01 13:23:05 +02:00
|
|
|
{
|
2020-05-31 09:28:59 +02:00
|
|
|
DEBUG_SYNC(thd, "before_wsrep_ordered_commit");
|
2019-04-01 13:23:05 +02:00
|
|
|
thd->wsrep_cs().ordered_commit();
|
|
|
|
}
|
2019-03-15 06:09:13 +01:00
|
|
|
}
|
|
|
|
}
|
2019-08-28 08:19:24 +02:00
|
|
|
|
|
|
|
extern "C" my_bool wsrep_thd_has_ignored_error(const THD *thd)
|
|
|
|
{
|
|
|
|
return thd->wsrep_has_ignored_error;
|
|
|
|
}
|
|
|
|
|
|
|
|
extern "C" void wsrep_thd_set_ignored_error(THD *thd, my_bool val)
|
|
|
|
{
|
|
|
|
thd->wsrep_has_ignored_error= val;
|
|
|
|
}
|
2019-10-02 12:24:34 +02:00
|
|
|
|
|
|
|
extern "C" ulong wsrep_OSU_method_get(const MYSQL_THD thd)
|
|
|
|
{
|
|
|
|
if (thd)
|
|
|
|
return(thd->variables.wsrep_OSU_method);
|
|
|
|
else
|
|
|
|
return(global_system_variables.wsrep_OSU_method);
|
|
|
|
}
|
2020-05-19 10:12:26 +02:00
|
|
|
|
2020-09-21 11:29:00 +02:00
|
|
|
extern "C" void wsrep_report_bf_lock_wait(const THD *thd,
|
|
|
|
unsigned long long trx_id)
|
|
|
|
{
|
|
|
|
if (thd)
|
|
|
|
{
|
|
|
|
WSREP_ERROR("Thread %s trx_id: %llu thread: %ld "
|
|
|
|
"seqno: %lld client_state: %s client_mode: %s transaction_mode: %s "
|
|
|
|
"applier: %d toi: %d local: %d "
|
|
|
|
"query: %s",
|
|
|
|
wsrep_thd_is_BF(thd, false) ? "BF" : "normal",
|
|
|
|
trx_id,
|
|
|
|
thd_get_thread_id(thd),
|
|
|
|
wsrep_thd_trx_seqno(thd),
|
|
|
|
wsrep_thd_client_state_str(thd),
|
|
|
|
wsrep_thd_client_mode_str(thd),
|
|
|
|
wsrep_thd_transaction_state_str(thd),
|
|
|
|
wsrep_thd_is_applying(thd),
|
|
|
|
wsrep_thd_is_toi(thd),
|
|
|
|
wsrep_thd_is_local(thd),
|
|
|
|
wsrep_thd_query(thd));
|
|
|
|
}
|
|
|
|
}
|
2021-12-09 17:12:20 +01:00
|
|
|
|
|
|
|
extern "C" void wsrep_thd_set_PA_unsafe(THD *thd)
|
|
|
|
{
|
|
|
|
if (thd && thd->wsrep_cs().mark_transaction_pa_unsafe())
|
|
|
|
{
|
|
|
|
WSREP_DEBUG("session does not have active transaction, can not mark as PA unsafe");
|
|
|
|
}
|
|
|
|
}
|
2022-12-05 15:03:32 +01:00
|
|
|
|
2023-11-21 14:43:11 +01:00
|
|
|
extern "C" uint32 wsrep_get_domain_id()
|
|
|
|
{
|
|
|
|
return wsrep_gtid_domain_id;
|
|
|
|
}
|
2024-01-02 16:37:58 +01:00
|
|
|
|
2022-12-05 15:03:32 +01:00
|
|
|
extern "C" my_bool wsrep_thd_is_local_transaction(const THD *thd)
|
|
|
|
{
|
|
|
|
return (wsrep_thd_is_local(thd) &&
|
|
|
|
thd->wsrep_cs().transaction().active());
|
|
|
|
}
|