From 3271588bb7c7e31ae9474c29b6c370fa3876e7b3 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com> Date: Fri, 14 Jun 2024 12:46:02 +0530 Subject: [PATCH 1/4] MDEV-34381 During innodb_undo_truncate=ON recovery, InnoDB may fail to shrink undo* files - During recovery, InnoDB may fail to shrink the undo tablespaces when there are no pages to recover while applying the redo log. This issue exists only when innodb_undo_truncate is enabled. trx_lists_init_at_db_start() could've applied the redo logs for undo tablespace page0. --- storage/innobase/log/log0recv.cc | 47 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 620e9c9209d..f83f5f45af6 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2668,6 +2668,29 @@ void recv_sys_t::apply(bool last_batch) recv_no_ibuf_operations = !last_batch || srv_operation == SRV_OPERATION_RESTORE || srv_operation == SRV_OPERATION_RESTORE_EXPORT; + for (auto id= srv_undo_tablespaces_open; id--;) + { + const trunc& t= truncated_undo_spaces[id]; + if (t.lsn) + { + /* The entire undo tablespace will be reinitialized by + innodb_undo_log_truncate=ON. Discard old log for all pages. + Even though we recv_sys_t::parse() already invoked trim(), + this will be needed in case recovery consists of multiple batches + (there was an invocation with !last_batch). */ + trim({id + srv_undo_space_id_start, 0}, t.lsn); + if (fil_space_t *space = fil_space_get(id + srv_undo_space_id_start)) + { + ut_ad(UT_LIST_GET_LEN(space->chain) == 1); + ut_ad(space->recv_size >= t.pages); + fil_node_t *file= UT_LIST_GET_FIRST(space->chain); + ut_ad(file->is_open()); + os_file_truncate(file->name, file->handle, + os_offset_t{space->recv_size} << + srv_page_size_shift, true); + } + } + } mtr_t mtr; @@ -2683,30 +2706,6 @@ void recv_sys_t::apply(bool last_batch) apply_log_recs= true; apply_batch_on= true; - for (auto id= srv_undo_tablespaces_open; id--;) - { - const trunc& t= truncated_undo_spaces[id]; - if (t.lsn) - { - /* The entire undo tablespace will be reinitialized by - innodb_undo_log_truncate=ON. Discard old log for all pages. - Even though we recv_sys_t::parse() already invoked trim(), - this will be needed in case recovery consists of multiple batches - (there was an invocation with !last_batch). */ - trim({id + srv_undo_space_id_start, 0}, t.lsn); - if (fil_space_t *space = fil_space_get(id + srv_undo_space_id_start)) - { - ut_ad(UT_LIST_GET_LEN(space->chain) == 1); - ut_ad(space->recv_size >= t.pages); - fil_node_t *file= UT_LIST_GET_FIRST(space->chain); - ut_ad(file->is_open()); - os_file_truncate(file->name, file->handle, - os_offset_t{space->recv_size} << - srv_page_size_shift, true); - } - } - } - fil_system.extend_to_recv_size(); /* Release the log_sys mutex in non-last batches of multi-batch From 4b4c371fe79ebf471450fed042bb1bcb94544a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com> Date: Fri, 14 Jun 2024 13:21:19 +0300 Subject: [PATCH 2/4] MDEV-34297 fixup: -Wconversion on 32-bit --- storage/innobase/include/ut0counter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/include/ut0counter.h b/storage/innobase/include/ut0counter.h index 0083627b91f..b30c3df3f0d 100644 --- a/storage/innobase/include/ut0counter.h +++ b/storage/innobase/include/ut0counter.h @@ -56,7 +56,7 @@ struct ib_counter_t { /** Add to the counter. @param[in] n amount to be added */ - void add(Type n) { add(my_pseudo_random(), n); } + void add(Type n) { add(size_t(my_pseudo_random()), n); } /** Add to the counter. @param[in] index a reasonably thread-unique identifier From fef32fd9ad844baa8b38b7f1a7da3b4ed2a772e5 Mon Sep 17 00:00:00 2001 From: Monty <monty@mariadb.org> Date: Sat, 15 Jun 2024 14:26:07 +0300 Subject: [PATCH 3/4] MDEV-34406 Enhance mariadb_upgrade to print failing query in case of error To make this possible, it was also necessary to enhance the mariadb client with the option --print-query-on-error. This option can also be very useful when running a batch of queries through the mariadb client and one wants to find out where things goes wrong. TODO: It would be good to enhance mariadb_upgrade to not call the mariadb client for executing queries but instead do this internally. This would have made this patch much easier! Reviewed by: Sergei Golubchik <serg@mariadb.com> --- client/mysql.cc | 66 +++++++++++++++++++++++++++----- client/mysql_upgrade.c | 62 +++++++++++++++++++----------- mysql-test/main/mysqldump.result | 6 ++- 3 files changed, 101 insertions(+), 33 deletions(-) diff --git a/client/mysql.cc b/client/mysql.cc index 1e235c8a34d..9591ffb096a 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -158,7 +158,8 @@ static my_bool ignore_errors=0,wait_flag=0,quick=0, default_pager_set= 0, opt_sigint_ignore= 0, auto_vertical_output= 0, show_warnings= 0, executing_query= 0, - ignore_spaces= 0, opt_binhex= 0, opt_progress_reports; + ignore_spaces= 0, opt_binhex= 0, opt_progress_reports, + opt_print_query_on_error; static my_bool debug_info_flag, debug_check_flag, batch_abort_on_error; static my_bool column_types_flag; static my_bool preserve_comments= 0; @@ -237,6 +238,7 @@ static int com_quit(String *str,char*), com_prompt(String *str, char*), com_delimiter(String *str, char*), com_warnings(String *str, char*), com_nowarnings(String *str, char*), com_sandbox(String *str, char*); +static void print_query_to_stderr(String *buffer); #ifdef USE_POPEN static int com_nopager(String *str, char*), com_pager(String *str, char*), @@ -1659,6 +1661,10 @@ static struct my_option my_long_options[] = #endif "built-in default (" STRINGIFY_ARG(MYSQL_PORT) ").", &opt_mysql_port, &opt_mysql_port, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"print-query-on-error", 0, + "Print the query if there was an error. Is only enabled in --batch mode if verbose is not set (as then the query would be printed anyway)", + &opt_print_query_on_error, &opt_print_query_on_error, 0, GET_BOOL, NO_ARG, + 1, 0, 0, 0, 0, 0}, {"progress-reports", 0, "Get progress reports for long running commands (like ALTER TABLE)", &opt_progress_reports, &opt_progress_reports, 0, GET_BOOL, NO_ARG, 1, 0, @@ -3086,6 +3092,11 @@ int mysql_real_query_for_lazy(const char *buf, size_t length) int error; if (!mysql_real_query(&mysql,buf,(ulong)length)) return 0; + if (opt_print_query_on_error) + { + String query(buf, length, charset_info); + (void) print_query_to_stderr(&query); + } error= put_error(&mysql); if (mysql_errno(&mysql) != CR_SERVER_GONE_ERROR || retry > 1 || !opt_reconnect) @@ -3291,7 +3302,6 @@ static int com_charset(String *, char *line) 1 if fatal error */ - static int com_go(String *buffer, char *) { char buff[200]; /* about 110 chars used so far */ @@ -3363,6 +3373,8 @@ static int com_go(String *buffer, char *) { if (!(result=mysql_use_result(&mysql)) && mysql_field_count(&mysql)) { + if (opt_print_query_on_error) + print_query_to_stderr(buffer); error= put_error(&mysql); goto end; } @@ -3416,7 +3428,11 @@ static int com_go(String *buffer, char *) (long) mysql_num_rows(result) == 1 ? "row" : "rows"); end_pager(); if (mysql_errno(&mysql)) + { + if (opt_print_query_on_error) + print_query_to_stderr(buffer); error= put_error(&mysql); + } } } else if (mysql_affected_rows(&mysql) == ~(ulonglong) 0) @@ -3443,13 +3459,21 @@ static int com_go(String *buffer, char *) put_info("",INFO_RESULT); // Empty row if (result && !mysql_eof(result)) /* Something wrong when using quick */ + { + if (opt_print_query_on_error) + print_query_to_stderr(buffer); error= put_error(&mysql); + } else if (unbuffered) fflush(stdout); mysql_free_result(result); } while (!(err= mysql_next_result(&mysql))); if (err >= 1) + { + if (opt_print_query_on_error) + print_query_to_stderr(buffer); error= put_error(&mysql); + } end: @@ -4375,14 +4399,35 @@ static int com_shell(String *, char *line) #endif +static void print_query(String *buffer, FILE *file) +{ + tee_puts("--------------", file); + (void) tee_fputs(buffer->c_ptr(), file); + if (!buffer->length() || (*buffer)[buffer->length()-1] != '\n') + tee_putc('\n', file); + tee_puts("--------------\n", file); +} + + +/* + Print query to stderr in batch mode if verbose is not set +*/ + +static void print_query_to_stderr(String *buffer) +{ + if ((status.batch || in_com_source) && !verbose) + { + fflush(stdout); + print_query(buffer, stderr); + fflush(stderr); + } +} + + static int com_print(String *buffer,char *) { - tee_puts("--------------", stdout); - (void) tee_fputs(buffer->c_ptr(), stdout); - if (!buffer->length() || (*buffer)[buffer->length()-1] != '\n') - tee_putc('\n', stdout); - tee_puts("--------------\n", stdout); - return 0; /* If empty buffer */ + print_query(buffer, stdout); + return 0; } @@ -5117,8 +5162,9 @@ put_info(const char *str,INFO_TYPE info_type, uint error, const char *sqlstate) static int put_error(MYSQL *con) { - return put_info(mysql_error(con), INFO_ERROR, mysql_errno(con), - mysql_sqlstate(con)); + DBUG_ENTER("put_error"); + DBUG_RETURN(put_info(mysql_error(con), INFO_ERROR, + mysql_errno(con), mysql_sqlstate(con))); } diff --git a/client/mysql_upgrade.c b/client/mysql_upgrade.c index b303ffd099a..f81cff2797c 100644 --- a/client/mysql_upgrade.c +++ b/client/mysql_upgrade.c @@ -628,7 +628,7 @@ static int run_query(const char *query, DYNAMIC_STRING *ds_res, { my_close(fd, MYF(MY_WME)); my_delete(query_file_path, MYF(0)); - die("Failed to write to '%s'", query_file_path); + die("Failed to write query to '%s'", query_file_path); } } @@ -637,7 +637,7 @@ static int run_query(const char *query, DYNAMIC_STRING *ds_res, { my_close(fd, MYF(MY_WME)); my_delete(query_file_path, MYF(0)); - die("Failed to write to '%s'", query_file_path); + die("Failed to write query to '%s'", query_file_path); } ret= run_tool(mysql_path, @@ -647,6 +647,7 @@ static int run_query(const char *query, DYNAMIC_STRING *ds_res, "--batch", /* Turns off pager etc. */ force ? "--force": "--skip-force", opt_verbose >= 5 ? "--verbose" : "", + "--print-query-on-error", ds_res || opt_silent ? "--silent": "", "<", query_file_path, @@ -1085,18 +1086,6 @@ static char* get_line(char* line) return line; } - -/* Print the current line to stderr */ -static void print_line(char* line) -{ - while (*line && *line != '\n') - { - fputc(*line, stderr); - line++; - } - fputc('\n', stderr); -} - static my_bool from_before_10_1() { my_bool ret= TRUE; @@ -1308,16 +1297,21 @@ static int check_slave_repositories(void) static int run_sql_fix_privilege_tables(void) { - int found_real_errors= 0; + int found_real_errors= 0, query_started= 0; const char **query_ptr; + const char *end; DYNAMIC_STRING ds_script; DYNAMIC_STRING ds_result; + DYNAMIC_STRING ds_query; DBUG_ENTER("run_sql_fix_privilege_tables"); - if (init_dynamic_string(&ds_script, "", 65536, 1024)) + if (init_dynamic_string(&ds_script, "", 96*1024, 8196)) die("Out of memory"); - if (init_dynamic_string(&ds_result, "", 512, 512)) + if (init_dynamic_string(&ds_result, "", 1024, 1024)) + die("Out of memory"); + + if (init_dynamic_string(&ds_query, "", 1024, 1024)) die("Out of memory"); verbose("Phase %d/%d: Running 'mysql_fix_privilege_tables'", @@ -1346,22 +1340,46 @@ static int run_sql_fix_privilege_tables(void) "Unknown column" and "Duplicate key name" since they just indicate the system tables are already up to date */ - char *line= ds_result.str; + const char *line= ds_result.str; do { + size_t length; + end= strchr(line, '\n'); + if (!end) + end= strend(line); + else + end++; /* Include end \n */ + length= (size_t) (end - line); + if (!is_expected_error(line)) { /* Something unexpected failed, dump error line to screen */ found_real_errors++; - print_line(line); + if (ds_query.length) + fwrite(ds_query.str, sizeof(char), ds_query.length, stderr); + fwrite(line, sizeof(char), length, stderr); + query_started= 0; } else if (strncmp(line, "WARNING", 7) == 0) { - print_line(line); + fwrite(line, sizeof(char), length, stderr); + query_started= 0; } - } while ((line= get_line(line)) && *line); + else if (!strncmp(line, "--------------\n", 16)) + { + /* mariadb separates query from the error with a line of '-' */ + if (!query_started++) + ds_query.length= 0; /* Truncate */ + else + query_started= 0; /* End of query */ + } + else if (query_started) + { + dynstr_append_mem(&ds_query, line, length); + } + } while (*(line= end)); } - + dynstr_free(&ds_query); dynstr_free(&ds_result); dynstr_free(&ds_script); DBUG_RETURN(found_real_errors); diff --git a/mysql-test/main/mysqldump.result b/mysql-test/main/mysqldump.result index 1541a4e6f72..1f59f31b7c8 100644 --- a/mysql-test/main/mysqldump.result +++ b/mysql-test/main/mysqldump.result @@ -6413,9 +6413,13 @@ j integer INSERT INTO t VALUES (1,1),(2,2),(3,3),(4,4); # Dump database 1 # Restore from database 1 to database 2 -ERROR 1100 (HY000) at line 46: Table 'seq_t_i' was not locked with LOCK TABLES SETVAL(`seq_t_i`, 1, 0) 1 +-------------- +INSERT INTO `t` VALUES (1,1),(2,2),(3,3),(4,4) +-------------- + +ERROR 1100 (HY000) at line 46: Table 'seq_t_i' was not locked with LOCK TABLES DROP DATABASE IF EXISTS test1; DROP DATABASE IF EXISTS test2; # From 956bcf8f49982ab99884321eb3f58dcfef3630ef Mon Sep 17 00:00:00 2001 From: Monty <monty@mariadb.org> Date: Sat, 15 Jun 2024 16:55:08 +0300 Subject: [PATCH 4/4] Change mysqldump to use DO instead of 'SELECT' for storing sequences. This avoids a lot of SETVAL() results when applying a mysqldump with sequences. --- client/mysqldump.c | 2 +- mysql-test/main/mysql.result | 8 ++++++ mysql-test/main/mysqldump.result | 26 ------------------- .../suite/sql_sequence/mysqldump.result | 16 ++++++------ 4 files changed, 17 insertions(+), 35 deletions(-) diff --git a/client/mysqldump.c b/client/mysqldump.c index 521e048497d..1c9d92d6e09 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -2959,7 +2959,7 @@ static void get_sequence_structure(const char *seq, const char *db) row= mysql_fetch_row(result); if (row[0]) { - fprintf(sql_file, "SELECT SETVAL(%s, %s, 0);\n", result_seq, row[0]); + fprintf(sql_file, "DO SETVAL(%s, %s, 0);\n", result_seq, row[0]); } // Sequences will not use inserts, so no need for REPLACE and LOCKS mysql_free_result(result); diff --git a/mysql-test/main/mysql.result b/mysql-test/main/mysql.result index 445aa602241..0bb447df47a 100644 --- a/mysql-test/main/mysql.result +++ b/mysql-test/main/mysql.result @@ -137,6 +137,10 @@ c int(11) YES NULL drop table t1; 1 1 +-------------- + use +-------------- + ERROR 1064 (42000) at line 3: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '' at line 1 ERROR at line 1: USE must be followed by a database name 1 +1 @@ -166,6 +170,10 @@ count(*) drop table t17583; Test connect without db- or host-name => reconnect Test connect with dbname only => new dbname, old hostname +-------------- +connecttest +-------------- + ERROR 1064 (42000) at line 1: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'connecttest' at line 1 Test connect with _invalid_ dbname only => new invalid dbname, old hostname ERROR 1049 (42000) at line 1: Unknown database 'invalid' diff --git a/mysql-test/main/mysqldump.result b/mysql-test/main/mysqldump.result index 1f59f31b7c8..b69e6168c86 100644 --- a/mysql-test/main/mysqldump.result +++ b/mysql-test/main/mysqldump.result @@ -6326,14 +6326,6 @@ Table Create Table s4 CREATE SEQUENCE `s4` start with 400 minvalue 400 maxvalue 1400 increment by 40 cache 1000 cycle ENGINE=MyISAM # Dump sequence without `--no-data` # Restore from mysqldump -SETVAL(`s1`, 1101, 0) -1101 -SETVAL(`s2`, 1201, 0) -1201 -SETVAL(`s3`, 1301, 0) -1301 -SETVAL(`s4`, 1401, 0) -1401 # Show create after restore show create sequence d.s1; Table Create Table @@ -6352,14 +6344,6 @@ NEXTVAL(d.s1) NEXTVAL(d.s2) NEXTVAL(d.s3) NEXTVAL(d.s4) 100 200 300 400 # Dump sequence with `--no-data` # Restore from mysqldump -SETVAL(`s1`, 1101, 0) -1101 -SETVAL(`s2`, 1201, 0) -1201 -SETVAL(`s3`, 1301, 0) -1301 -SETVAL(`s4`, 1401, 0) -1401 # Show create after restore `--no-data` show create sequence d.s1; Table Create Table @@ -6378,14 +6362,6 @@ NEXTVAL(d.s1) NEXTVAL(d.s2) NEXTVAL(d.s3) NEXTVAL(d.s4) 100 200 300 400 # Restore to different database than original create database d2; -SETVAL(`s1`, 1101, 0) -1101 -SETVAL(`s2`, 1201, 0) -1201 -SETVAL(`s3`, 1301, 0) -1301 -SETVAL(`s4`, 1401, 0) -1401 show create sequence d2.s1; Table Create Table s1 CREATE SEQUENCE `s1` start with 100 minvalue 100 maxvalue 1100 increment by 10 cache 1000 cycle ENGINE=MyISAM @@ -6413,8 +6389,6 @@ j integer INSERT INTO t VALUES (1,1),(2,2),(3,3),(4,4); # Dump database 1 # Restore from database 1 to database 2 -SETVAL(`seq_t_i`, 1, 0) -1 -------------- INSERT INTO `t` VALUES (1,1),(2,2),(3,3),(4,4) -------------- diff --git a/mysql-test/suite/sql_sequence/mysqldump.result b/mysql-test/suite/sql_sequence/mysqldump.result index b07f07a605b..ea100fb0199 100644 --- a/mysql-test/suite/sql_sequence/mysqldump.result +++ b/mysql-test/suite/sql_sequence/mysqldump.result @@ -5,9 +5,9 @@ CREATE SEQUENCE x1 engine=innodb; # dump whole database /*!999999\- enable the sandbox mode */ CREATE SEQUENCE `a1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=Aria; -SELECT SETVAL(`a1`, 1, 0); +DO SETVAL(`a1`, 1, 0); CREATE SEQUENCE `x1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB; -SELECT SETVAL(`x1`, 1, 0); +DO SETVAL(`x1`, 1, 0); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `t1` ( @@ -19,9 +19,9 @@ INSERT INTO `t1` VALUES (1),(2); # dump by tables order 1 /*!999999\- enable the sandbox mode */ CREATE SEQUENCE `a1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=Aria; -SELECT SETVAL(`a1`, 1, 0); +DO SETVAL(`a1`, 1, 0); CREATE SEQUENCE `x1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB; -SELECT SETVAL(`x1`, 1, 0); +DO SETVAL(`x1`, 1, 0); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `t1` ( @@ -33,9 +33,9 @@ INSERT INTO `t1` VALUES (1),(2); # dump by tables order 2 /*!999999\- enable the sandbox mode */ CREATE SEQUENCE `a1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=Aria; -SELECT SETVAL(`a1`, 1, 0); +DO SETVAL(`a1`, 1, 0); CREATE SEQUENCE `x1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB; -SELECT SETVAL(`x1`, 1, 0); +DO SETVAL(`x1`, 1, 0); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `t1` ( @@ -57,9 +57,9 @@ INSERT INTO `t1` VALUES (1),(2); # dump by tables only sequences /*!999999\- enable the sandbox mode */ CREATE SEQUENCE `a1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=Aria; -SELECT SETVAL(`a1`, 1, 0); +DO SETVAL(`a1`, 1, 0); CREATE SEQUENCE `x1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB; -SELECT SETVAL(`x1`, 1, 0); +DO SETVAL(`x1`, 1, 0); # end of dumps DROP TABLE a1,t1,x1; set default_storage_engine=InnoDB;