From fdf2d407707faf05b8b7d67662a70cc5537d15aa Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 24 Mar 2017 12:07:07 +0100 Subject: [PATCH] MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Implement auto-creation of mysql.gtid_slave_pos* tables with needed engines, if listed in --gtid-pos-auto-engines. Uses an asynchronous approach to minimise locking overhead. The list of available tables is extended with a flag. Extra entries are added for --gtid-pos-auto-engines tables that do not exist yet, marked as not existing but ready for auto-creation. If record_gtid() needs a table marked for auto-creation, it sends a request to the slave background thread to create the table, and continues to use an existing table for the current and immediately coming transactions. As soon as the slave background thread has made the new table available, it will be used for all subsequent relevant transactions in record_gtid(). This asynchronous approach also avoids a lot of complex issues around trying to do DDL in the middle of an on-going transaction. --- mysql-test/suite/rpl/r/rpl_mdev12179.result | 195 +++++++++++++++++- mysql-test/suite/rpl/t/rpl_mdev12179.test | 199 +++++++++++++++++- scripts/mysql_system_tables.sql | 2 + sql/rpl_gtid.cc | 21 +- sql/rpl_gtid.h | 10 +- sql/rpl_rli.cc | 83 +++++++- sql/slave.cc | 216 +++++++++++++++++++- sql/slave.h | 3 + 8 files changed, 702 insertions(+), 27 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_mdev12179.result b/mysql-test/suite/rpl/r/rpl_mdev12179.result index c7a791faeb6..8bf8f183181 100644 --- a/mysql-test/suite/rpl/r/rpl_mdev12179.result +++ b/mysql-test/suite/rpl/r/rpl_mdev12179.result @@ -1,4 +1,5 @@ include/rpl_init.inc [topology=1->2] +connection server_2; SET GLOBAL gtid_pos_auto_engines="innodb"; ERROR HY000: This operation cannot be performed as you have a running slave ''; run STOP SLAVE '' first include/stop_slave.inc @@ -35,11 +36,13 @@ SELECT @@gtid_pos_auto_engines; @@gtid_pos_auto_engines include/start_slave.inc +connection server_1; CREATE TABLE t1 (a INT PRIMARY KEY); INSERT INTO t1 VALUES (1); SELECT * FROM t1 ORDER BY a; a 1 +connection server_2; SELECT * FROM t1 ORDER BY a; a 1 @@ -50,6 +53,7 @@ ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; TRUNCATE mysql.gtid_slave_pos; SET sql_log_bin=1; +connection server_1; INSERT INTO t1 VALUES (2); INSERT INTO t1 VALUES (3); SELECT * FROM t1 ORDER BY a; @@ -58,20 +62,203 @@ a 2 3 include/save_master_gtid.inc +*** Restart server with --gtid-pos-auto-engines=innodb,myisam *** +connection server_2; include/sync_with_master_gtid.inc SELECT * FROM t1 ORDER BY a; a 1 2 3 +*** Verify no new gtid_slave_pos* tables are created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos MyISAM +gtid_slave_pos_innodb InnoDB SELECT @@gtid_pos_auto_engines; @@gtid_pos_auto_engines InnoDB,MyISAM include/stop_slave.inc -SET GLOBAL gtid_pos_auto_engines=""; -include/start_slave.inc SET sql_log_bin=0; -DROP TABLE mysql.gtid_slave_pos_innodb; +INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; +DROP TABLE mysql.gtid_slave_pos; +RENAME TABLE mysql.gtid_slave_pos_innodb TO mysql.gtid_slave_pos; SET sql_log_bin=1; -DROP TABLE t1; +connection server_1; +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (4); +INSERT INTO t2 VALUES (1); +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +SELECT * FROM t2 ORDER BY a; +a +1 +include/save_master_gtid.inc +*** Restart server with --gtid-pos-auto-engines=myisam,innodb *** +connection server_2; +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +SELECT * FROM t2 ORDER BY a; +a +1 +*** Verify that no new gtid_slave_pos* tables are auto-created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos InnoDB +include/stop_slave.inc +SET sql_log_bin=0; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; +SET sql_log_bin=1; +connection server_1; +INSERT INTO t1 VALUES (5); +INSERT INTO t2 VALUES (2); +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +include/save_master_gtid.inc +connection server_2; +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +*** Verify that mysql.gtid_slave_pos_InnoDB is auto-created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos MyISAM +gtid_slave_pos_InnoDB InnoDB +include/stop_slave.inc +SET sql_log_bin=0; +INSERT INTO mysql.gtid_slave_pos SELECT * FROM mysql.gtid_slave_pos_InnoDB; +DROP TABLE mysql.gtid_slave_pos_InnoDB; +SET sql_log_bin=1; +connection server_1; +INSERT INTO t1 VALUES (6); +INSERT INTO t2 VALUES (3); +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +6 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +3 +include/save_master_gtid.inc +*** Restart server without --gtid-pos-auto-engines *** +connection server_2; +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +6 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +3 +*** Verify that no mysql.gtid_slave_pos* table is auto-created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos MyISAM +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +domain_id max(seq_no) +0 11 +include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines="innodb"; +include/start_slave.inc +connection server_1; +INSERT INTO t1 VALUES (7); +INSERT INTO t2 VALUES (4); +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +6 +7 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +3 +4 +include/save_master_gtid.inc +connection server_2; +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +6 +7 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +3 +4 +*** Verify that mysql.gtid_slave_pos_InnoDB is auto-created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos MyISAM +gtid_slave_pos_InnoDB InnoDB +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +domain_id max(seq_no) +0 13 +include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines=""; +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_InnoDB; +SET sql_log_bin=1; +include/start_slave.inc +connection server_1; +DROP TABLE t1, t2; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_mdev12179.test b/mysql-test/suite/rpl/t/rpl_mdev12179.test index a4b344de3f1..b4609c2ed7a 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev12179.test +++ b/mysql-test/suite/rpl/t/rpl_mdev12179.test @@ -63,6 +63,7 @@ SELECT * FROM t1 ORDER BY a; # Let the slave mysqld server start again. # As we are restarting, also take the opportunity to test --gtid-pos-auto-engines +--echo *** Restart server with --gtid-pos-auto-engines=innodb,myisam *** --append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect restart: --skip-slave-start=0 --gtid-pos-auto-engines=innodb,myisam EOF @@ -74,17 +75,203 @@ EOF --source include/sync_with_master_gtid.inc SELECT * FROM t1 ORDER BY a; +--echo *** Verify no new gtid_slave_pos* tables are created *** +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; + SELECT @@gtid_pos_auto_engines; --source include/stop_slave.inc -SET GLOBAL gtid_pos_auto_engines=""; ---source include/start_slave.inc - ---connection server_2 SET sql_log_bin=0; -DROP TABLE mysql.gtid_slave_pos_innodb; +INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; +DROP TABLE mysql.gtid_slave_pos; +RENAME TABLE mysql.gtid_slave_pos_innodb TO mysql.gtid_slave_pos; SET sql_log_bin=1; +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +--shutdown_server 30 +--source include/wait_until_disconnected.inc + --connection server_1 -DROP TABLE t1; +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (4); +INSERT INTO t2 VALUES (1); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +--source include/save_master_gtid.inc + +--echo *** Restart server with --gtid-pos-auto-engines=myisam,innodb *** +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 --gtid-pos-auto-engines=myisam,innodb +EOF + +--connection server_2 +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Verify that no new gtid_slave_pos* tables are auto-created *** +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; + + +--source include/stop_slave.inc +SET sql_log_bin=0; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; +SET sql_log_bin=1; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +--shutdown_server 30 +--source include/wait_until_disconnected.inc + +--connection server_1 +INSERT INTO t1 VALUES (5); +INSERT INTO t2 VALUES (2); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +--source include/save_master_gtid.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +--echo *** Restart server with --gtid-pos-auto-engines=innodb *** +restart: --skip-slave-start=0 --gtid-pos-auto-engines=innodb +EOF + +--connection server_2 +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Verify that mysql.gtid_slave_pos_InnoDB is auto-created *** +# Note, the create happens asynchronously, so wait for it. +let $wait_condition= + SELECT EXISTS (SELECT * FROM information_schema.tables + WHERE table_schema='mysql' AND table_name='gtid_slave_pos_InnoDB'); +--source include/wait_condition.inc +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; + + +--source include/stop_slave.inc +SET sql_log_bin=0; +INSERT INTO mysql.gtid_slave_pos SELECT * FROM mysql.gtid_slave_pos_InnoDB; +DROP TABLE mysql.gtid_slave_pos_InnoDB; +SET sql_log_bin=1; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +--shutdown_server 30 +--source include/wait_until_disconnected.inc + +--connection server_1 +INSERT INTO t1 VALUES (6); +INSERT INTO t2 VALUES (3); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +--source include/save_master_gtid.inc + +--echo *** Restart server without --gtid-pos-auto-engines *** +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 +EOF + +--connection server_2 +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Verify that no mysql.gtid_slave_pos* table is auto-created *** +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; + +--source include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines="innodb"; +--source include/start_slave.inc + +--connection server_1 +INSERT INTO t1 VALUES (7); +INSERT INTO t2 VALUES (4); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Verify that mysql.gtid_slave_pos_InnoDB is auto-created *** +let $wait_condition= + SELECT EXISTS (SELECT * FROM information_schema.tables + WHERE table_schema='mysql' AND table_name='gtid_slave_pos_InnoDB'); +--source include/wait_condition.inc +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; + +# Check that the auto-created InnoDB table starts being used without +# needing slave restart. The auto-create happens asynchronously, so it +# is non-deterministic when it will start being used. But we can wait +# for it to happen. + +--let $count=300 +--let $done=0 +--let $old_silent= $keep_include_silent +--let $keep_include_silent= 1 +--disable_query_log +while (!$done) +{ + --connection server_1 + INSERT INTO t2(a) SELECT 1+MAX(a) FROM t2; + --source include/save_master_gtid.inc + + --connection server_2 + --source include/sync_with_master_gtid.inc + --let $done=`SELECT COUNT(*) > 0 FROM mysql.gtid_slave_pos_InnoDB` + if (!$done) + { + dec $count; + if (!$count) + { + SELECT * FROM mysql.gtid_slave_pos_InnoDB; + --die Timeout waiting for mysql.gtid_slave_pos_InnoDB to be used + } + real_sleep 0.1; + } +} +--enable_query_log +--let $keep_include_silent=$old_silent +# Note that at this point, the contents of table t2, as well as the GTID +# position, is non-deterministic. + + +#--connection server_2 +--source include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines=""; +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_InnoDB; +SET sql_log_bin=1; +--source include/start_slave.inc + +--connection server_1 +DROP TABLE t1, t2; --source include/rpl_end.inc diff --git a/scripts/mysql_system_tables.sql b/scripts/mysql_system_tables.sql index 7b614163f46..60c1ac97955 100644 --- a/scripts/mysql_system_tables.sql +++ b/scripts/mysql_system_tables.sql @@ -224,6 +224,8 @@ CREATE TABLE IF NOT EXISTS column_stats (db_name varchar(64) NOT NULL, table_nam CREATE TABLE IF NOT EXISTS index_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, index_name varchar(64) NOT NULL, prefix_arity int(11) unsigned NOT NULL, avg_frequency decimal(12,4) DEFAULT NULL, PRIMARY KEY (db_name,table_name,index_name,prefix_arity) ) ENGINE=MyISAM CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Indexes'; +-- Note: This definition must be kept in sync with the one used in +-- build_gtid_pos_create_query() in sql/slave.cc SET @cmd= "CREATE TABLE IF NOT EXISTS gtid_slave_pos ( domain_id INT UNSIGNED NOT NULL, sub_id BIGINT UNSIGNED NOT NULL, diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index ac3c621a5c1..af70f281d8c 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -26,6 +26,7 @@ #include "key.h" #include "rpl_gtid.h" #include "rpl_rli.h" +#include "slave.h" const LEX_STRING rpl_gtid_slave_state_table_name= @@ -494,8 +495,20 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) { if (table_entry->table_hton == trx_hton) { - *out_tablename= table_entry->table_name; - return; + if (likely(table_entry->state == GTID_POS_AVAILABLE)) + { + *out_tablename= table_entry->table_name; + return; + } + /* + This engine is marked to automatically create the table. + We cannot easily do this here (possibly in the middle of a + transaction). But we can request the slave background thread + to create it, and in a short while it should become available + for following transactions. + */ + slave_background_gtid_pos_create_request(table_entry); + break; } table_entry= table_entry->next; } @@ -1240,7 +1253,8 @@ rpl_slave_state::add_gtid_pos_table(rpl_slave_state::gtid_pos_table *entry) struct rpl_slave_state::gtid_pos_table * -rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, void *hton) +rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, void *hton, + rpl_slave_state::gtid_pos_table_state state) { struct gtid_pos_table *p; char *allocated_str; @@ -1258,6 +1272,7 @@ rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, void *hton) p->table_hton= hton; p->table_name.str= allocated_str; p->table_name.length= table_name->length; + p->state= state; return p; } diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index b4f80fe3ddf..e6a3ea1a667 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -157,6 +157,12 @@ struct rpl_slave_state }; /* Descriptor for mysql.gtid_slave_posXXX table in specific engine. */ + enum gtid_pos_table_state { + GTID_POS_AUTO_CREATE, + GTID_POS_CREATE_REQUESTED, + GTID_POS_CREATE_IN_PROGRESS, + GTID_POS_AVAILABLE + }; struct gtid_pos_table { struct gtid_pos_table *next; /* @@ -167,6 +173,7 @@ struct rpl_slave_state */ void *table_hton; LEX_STRING table_name; + uint8 state; }; /* Mapping from domain_id to its element. */ @@ -232,7 +239,8 @@ struct rpl_slave_state void set_gtid_pos_tables_list(gtid_pos_table *new_list, gtid_pos_table *default_entry); void add_gtid_pos_table(gtid_pos_table *entry); - struct gtid_pos_table *alloc_gtid_pos_table(LEX_STRING *table_name, void *hton); + struct gtid_pos_table *alloc_gtid_pos_table(LEX_STRING *table_name, + void *hton, rpl_slave_state::gtid_pos_table_state state); void free_gtid_pos_tables(struct gtid_pos_table *list); }; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 74e4be524b9..2cda87925d1 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1690,7 +1690,9 @@ process_gtid_pos_table(THD *thd, LEX_STRING *table_name, void *hton, entry= entry->next; } - if (!(p= rpl_global_gtid_slave_state->alloc_gtid_pos_table(table_name, hton))) + p= rpl_global_gtid_slave_state->alloc_gtid_pos_table(table_name, + hton, rpl_slave_state::GTID_POS_AVAILABLE); + if (!p) return 1; p->next= data->table_list; data->table_list= p; @@ -1700,6 +1702,59 @@ process_gtid_pos_table(THD *thd, LEX_STRING *table_name, void *hton, } +/* + Put tables corresponding to @@gtid_pos_auto_engines at the end of the list, + marked to be auto-created if needed. +*/ +static int +gtid_pos_auto_create_tables(rpl_slave_state::gtid_pos_table **list_ptr) +{ + plugin_ref *auto_engines; + int err= 0; + mysql_mutex_lock(&LOCK_global_system_variables); + for (auto_engines= opt_gtid_pos_auto_plugins; + !err && auto_engines && *auto_engines; + ++auto_engines) + { + void *hton= plugin_hton(*auto_engines); + char buf[FN_REFLEN+1]; + LEX_STRING table_name; + char *p; + rpl_slave_state::gtid_pos_table *entry, **next_ptr; + + /* See if this engine is already in the list. */ + next_ptr= list_ptr; + entry= *list_ptr; + while (entry) + { + if (entry->table_hton == hton) + break; + next_ptr= &entry->next; + entry= entry->next; + } + if (entry) + continue; + + /* Add an auto-create entry for this engine at end of list. */ + p= strmake(buf, rpl_gtid_slave_state_table_name.str, FN_REFLEN); + p= strmake(p, "_", FN_REFLEN - (p - buf)); + p= strmake(p, plugin_name(*auto_engines)->str, FN_REFLEN - (p - buf)); + table_name.str= buf; + table_name.length= p - buf; + entry= rpl_global_gtid_slave_state->alloc_gtid_pos_table + (&table_name, hton, rpl_slave_state::GTID_POS_AUTO_CREATE); + if (!entry) + { + err= 1; + break; + } + *next_ptr= entry; + } + mysql_mutex_unlock(&LOCK_global_system_variables); + return err; +} + + static int load_gtid_state_cb(THD *thd, LEX_STRING *table_name, void *arg) { @@ -1746,6 +1801,18 @@ rpl_load_gtid_slave_state(THD *thd) if ((err= scan_all_gtid_slave_pos_table(thd, load_gtid_state_cb, &cb_data))) goto end; + if (!cb_data.default_entry) + { + /* + If the mysql.gtid_slave_pos table does not exist, but at least one other + table is available, arbitrarily pick the first in the list to use as + default. + */ + cb_data.default_entry= cb_data.table_list; + } + if ((err= gtid_pos_auto_create_tables(&cb_data.table_list))) + goto end; + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); if (rpl_global_gtid_slave_state->loaded) { @@ -1757,18 +1824,10 @@ rpl_load_gtid_slave_state(THD *thd) { my_error(ER_NO_SUCH_TABLE, MYF(0), "mysql", rpl_gtid_slave_state_table_name.str); + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); err= 1; goto end; } - else if (!cb_data.default_entry) - { - /* - If the mysql.gtid_slave_pos table does not exist, but at least one other - table is available, arbitrarily pick the first in the list to use as - default. - */ - cb_data.default_entry= cb_data.table_list; - } for (i= 0; i < array.elements; ++i) { @@ -1878,7 +1937,7 @@ find_gtid_slave_pos_tables(THD *thd) err= 1; goto end; } - else if (!cb_data.default_entry) + if (!cb_data.default_entry) { /* If the mysql.gtid_slave_pos table does not exist, but at least one other @@ -1887,6 +1946,8 @@ find_gtid_slave_pos_tables(THD *thd) */ cb_data.default_entry= cb_data.table_list; } + if ((err= gtid_pos_auto_create_tables(&cb_data.table_list))) + goto end; any_running= any_slave_sql_running(); mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); diff --git a/sql/slave.cc b/sql/slave.cc index fbdb78b5c5d..443e5b7a5f1 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -59,6 +59,7 @@ #include "rpl_tblmap.h" #include "debug_sync.h" #include "rpl_parallel.h" +#include "sql_show.h" #define FLAGSTR(V,F) ((V)&(F)?#F" ":"") @@ -279,15 +280,178 @@ static void init_slave_psi_keys(void) #endif /* HAVE_PSI_INTERFACE */ +/* + Note: This definition needs to be kept in sync with the one in + mysql_system_tables.sql which is used by mysql_create_db. +*/ +static const char gtid_pos_table_definition1[]= + "CREATE TABLE "; +static const char gtid_pos_table_definition2[]= + " (domain_id INT UNSIGNED NOT NULL, " + "sub_id BIGINT UNSIGNED NOT NULL, " + "server_id INT UNSIGNED NOT NULL, " + "seq_no BIGINT UNSIGNED NOT NULL, " + "PRIMARY KEY (domain_id, sub_id)) CHARSET=latin1 " + "COMMENT='Replication slave GTID position' " + "ENGINE="; + +/* + Build a query string + CREATE TABLE mysql.gtid_slave_pos_ ... ENGINE= +*/ +static bool +build_gtid_pos_create_query(THD *thd, String *query, + LEX_STRING *table_name, + LEX_STRING *engine_name) +{ + bool err= false; + err|= query->append(gtid_pos_table_definition1); + err|= append_identifier(thd, query, table_name->str, table_name->length); + err|= query->append(gtid_pos_table_definition2); + err|= append_identifier(thd, query, engine_name->str, engine_name->length); + return err; +} + + +static int +gtid_pos_table_creation(THD *thd, plugin_ref engine, LEX_STRING *table_name) +{ + int err; + StringBuffer query; + + if (build_gtid_pos_create_query(thd, &query, table_name, plugin_name(engine))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return 1; + } + + thd->set_db("mysql", 5); + thd->clear_error(); + ulonglong thd_saved_option= thd->variables.option_bits; + /* This query shuold not be binlogged. */ + thd->variables.option_bits&= ~(ulonglong)OPTION_BIN_LOG; + thd->set_query_and_id(query.c_ptr(), query.length(), thd->charset(), + next_query_id()); + Parser_state parser_state; + err= parser_state.init(thd, thd->query(), thd->query_length()); + if (err) + goto end; + mysql_parse(thd, thd->query(), thd->query_length(), &parser_state, + FALSE, FALSE); + if (thd->is_error()) + err= 1; +end: + thd->variables.option_bits= thd_saved_option; + thd->reset_query(); + return err; +} + + +static void +handle_gtid_pos_auto_create_request(THD *thd, void *hton) +{ + int err; + plugin_ref engine= NULL, *auto_engines; + rpl_slave_state::gtid_pos_table *entry; + StringBuffer loc_table_name; + LEX_STRING table_name; + + /* + Check that the plugin is still in @@gtid_pos_auto_engines, and lock + it. + */ + mysql_mutex_lock(&LOCK_global_system_variables); + engine= NULL; + for (auto_engines= opt_gtid_pos_auto_plugins; + auto_engines && *auto_engines; + ++auto_engines) + { + if (plugin_hton(*auto_engines) == hton) + { + engine= my_plugin_lock(NULL, *auto_engines); + break; + } + } + mysql_mutex_unlock(&LOCK_global_system_variables); + if (!engine) + { + /* The engine is gone from @@gtid_pos_auto_engines, so no action. */ + goto end; + } + + /* Find the entry for the table to auto-create. */ + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); + entry= rpl_global_gtid_slave_state->gtid_pos_tables; + while (entry) + { + if (entry->table_hton == hton && + entry->state == rpl_slave_state::GTID_POS_CREATE_REQUESTED) + break; + entry= entry->next; + } + if (entry) + { + entry->state = rpl_slave_state::GTID_POS_CREATE_IN_PROGRESS; + err= loc_table_name.append(entry->table_name.str, entry->table_name.length); + } + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + if (!entry) + goto end; + if (err) + { + sql_print_error("Out of memory while trying to auto-create GTID position table"); + goto end; + } + table_name.str= loc_table_name.c_ptr_safe(); + table_name.length= loc_table_name.length(); + + err= gtid_pos_table_creation(thd, engine, &table_name); + if (err) + { + sql_print_error("Error auto-creating GTID position table `mysql.%s`: %s Error_code: %d", + table_name.str, thd->get_stmt_da()->message(), + thd->get_stmt_da()->sql_errno()); + thd->clear_error(); + goto end; + } + + /* Now enable the entry for the auto-created table. */ + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); + entry= rpl_global_gtid_slave_state->gtid_pos_tables; + while (entry) + { + if (entry->table_hton == hton && + entry->state == rpl_slave_state::GTID_POS_CREATE_IN_PROGRESS) + { + entry->state= rpl_slave_state::GTID_POS_AVAILABLE; + break; + } + entry= entry->next; + } + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + +end: + if (engine) + plugin_unlock(NULL, engine); +} + + static bool slave_background_thread_running; static bool slave_background_thread_stop; static bool slave_background_thread_gtid_loaded; -struct slave_background_kill_t { +static struct slave_background_kill_t { slave_background_kill_t *next; THD *to_kill; } *slave_background_kill_list; +static struct slave_background_gtid_pos_create_t { + slave_background_gtid_pos_create_t *next; + void *hton; +} *slave_background_gtid_pos_create_list; + pthread_handler_t handle_slave_background(void *arg __attribute__((unused))) @@ -321,6 +485,7 @@ handle_slave_background(void *arg __attribute__((unused))) do { slave_background_kill_t *kill_list; + slave_background_gtid_pos_create_t *create_list; thd->ENTER_COND(&COND_slave_background, &LOCK_slave_background, &stage_slave_background_wait_request, @@ -329,12 +494,14 @@ handle_slave_background(void *arg __attribute__((unused))) { stop= abort_loop || thd->killed || slave_background_thread_stop; kill_list= slave_background_kill_list; - if (stop || kill_list) + create_list= slave_background_gtid_pos_create_list; + if (stop || kill_list || create_list) break; mysql_cond_wait(&COND_slave_background, &LOCK_slave_background); } slave_background_kill_list= NULL; + slave_background_gtid_pos_create_list= NULL; thd->EXIT_COND(&old_stage); while (kill_list) @@ -353,6 +520,16 @@ handle_slave_background(void *arg __attribute__((unused))) mysql_mutex_unlock(&to_kill->LOCK_wakeup_ready); my_free(p); } + + while (create_list) + { + slave_background_gtid_pos_create_t *next= create_list->next; + void *hton= create_list->hton; + handle_gtid_pos_auto_create_request(thd, hton); + my_free(create_list); + create_list= next; + } + mysql_mutex_lock(&LOCK_slave_background); } while (!stop); @@ -391,6 +568,41 @@ slave_background_kill_request(THD *to_kill) } +/* + This function must only be called from a slave SQL thread (or worker thread), + to ensure that the table_entry will not go away before we can lock the + LOCK_slave_state. +*/ +void +slave_background_gtid_pos_create_request( + rpl_slave_state::gtid_pos_table *table_entry) +{ + slave_background_gtid_pos_create_t *p; + + if (table_entry->state != rpl_slave_state::GTID_POS_AUTO_CREATE) + return; + p= (slave_background_gtid_pos_create_t *)my_malloc(sizeof(*p), MYF(MY_WME)); + if (!p) + return; + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); + if (table_entry->state != rpl_slave_state::GTID_POS_AUTO_CREATE) + { + my_free(p); + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + return; + } + table_entry->state= rpl_slave_state::GTID_POS_CREATE_REQUESTED; + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + + p->hton= table_entry->table_hton; + mysql_mutex_lock(&LOCK_slave_background); + p->next= slave_background_gtid_pos_create_list; + slave_background_gtid_pos_create_list= p; + mysql_cond_signal(&COND_slave_background); + mysql_mutex_unlock(&LOCK_slave_background); +} + + /* Start the slave background thread. diff --git a/sql/slave.h b/sql/slave.h index ded9d76e49d..2afd5277e25 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -48,6 +48,7 @@ #include "my_list.h" #include "rpl_filter.h" #include "rpl_tblmap.h" +#include "rpl_gtid.h" #define SLAVE_NET_TIMEOUT 60 @@ -268,6 +269,8 @@ void slave_output_error_info(rpl_group_info *rgi, THD *thd); pthread_handler_t handle_slave_sql(void *arg); bool net_request_file(NET* net, const char* fname); void slave_background_kill_request(THD *to_kill); +void slave_background_gtid_pos_create_request + (rpl_slave_state::gtid_pos_table *table_entry); extern bool volatile abort_loop; extern Master_info *active_mi; /* active_mi for multi-master */