From 51ca5d517ef67f97e3c38d57174f8cfa251a14d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 10 Jun 2022 08:59:23 +0300 Subject: [PATCH 01/11] Fix ./mtr --embedded This fixes up commit 3d241eb948855dbe0688a04c8111cc78deac3c1c --- mysql-test/suite/maria/encrypt-no-key.result | 6 +++--- mysql-test/suite/maria/encrypt-no-key.test | 2 ++ mysql-test/suite/maria/maria-recover.result | 4 ++-- mysql-test/suite/maria/maria-recover.test | 1 + 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/maria/encrypt-no-key.result b/mysql-test/suite/maria/encrypt-no-key.result index bdc8a79ff9a..5229158d525 100644 --- a/mysql-test/suite/maria/encrypt-no-key.result +++ b/mysql-test/suite/maria/encrypt-no-key.result @@ -6,12 +6,12 @@ create table t1 (pk int primary key, a int, key(a)) engine=aria transactional=1; alter table t1 disable keys; insert into t1 values (1,1); alter table t1 enable keys; -ERROR HY000: Unknown key id 1 for ./test/t1. Can't continue! +ERROR HY000: Unknown key id 1 for test/t1. Can't continue! repair table t1 use_frm; Table Op Msg_type Msg_text test.t1 repair warning Number of rows changed from 0 to 1 -test.t1 repair Error Unknown key id 1 for ./test/t1. Can't continue! -test.t1 repair Error Unknown key id 1 for ./test/t1. Can't continue! +test.t1 repair Error Unknown key id 1 for test/t1. Can't continue! +test.t1 repair Error Unknown key id 1 for test/t1. Can't continue! test.t1 repair status OK drop table t1; set global aria_encrypt_tables= default; diff --git a/mysql-test/suite/maria/encrypt-no-key.test b/mysql-test/suite/maria/encrypt-no-key.test index eebc2a102d3..1157263ca79 100644 --- a/mysql-test/suite/maria/encrypt-no-key.test +++ b/mysql-test/suite/maria/encrypt-no-key.test @@ -10,9 +10,11 @@ create table t1 (pk int primary key, a int, key(a)) engine=aria transactional=1; alter table t1 disable keys; insert into t1 values (1,1); --replace_result \\ / +--replace_regex /for .*test/for test/ --error 192 alter table t1 enable keys; --replace_result \\ / +--replace_regex /for .*test/for test/ repair table t1 use_frm; drop table t1; set global aria_encrypt_tables= default; diff --git a/mysql-test/suite/maria/maria-recover.result b/mysql-test/suite/maria/maria-recover.result index 788cd7eaf5a..7bb90cc0405 100644 --- a/mysql-test/suite/maria/maria-recover.result +++ b/mysql-test/suite/maria/maria-recover.result @@ -26,9 +26,9 @@ a ThursdayMorningsMarket ThursdayMorningsMarketb Warnings: -Error 145 Got error '145 "Table was marked as crashed and should be repaired"' for './mysqltest/t_corrupted2' +Error 145 Got error '145 "Table was marked as crashed and should be repaired"' for 't_corrupted2' Warning 1034 1 client is using or hasn't closed the table properly -Error 176 Got error '176 "Read page with wrong checksum"' for './mysqltest/t_corrupted2.MAI' +Error 176 Got error '176 "Read page with wrong checksum"' for 't_corrupted2.MAI' Error 1034 Can't read indexpage from page: 1, error: 176 select * from t_corrupted2; a diff --git a/mysql-test/suite/maria/maria-recover.test b/mysql-test/suite/maria/maria-recover.test index cea185e7ab5..5ddfd6d202e 100644 --- a/mysql-test/suite/maria/maria-recover.test +++ b/mysql-test/suite/maria/maria-recover.test @@ -79,6 +79,7 @@ perl; close FILE; EOF --replace_result \\ / +--replace_regex /for '.*t_corrupted2/for 't_corrupted2/ --enable_prepare_warnings select * from t_corrupted2; # should show corruption and repair messages --disable_prepare_warnings From 1fd7d3a9adac50de37e40e92188077e3515de505 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 7 Jun 2022 10:07:13 +0530 Subject: [PATCH 02/11] MDEV-25581 Allow user thread to do InnoDB fts cache sync Problem: ======== InnoDB FTS requesting the fts sync of the table once the fts cache size reaches 1/10 of innodb_ft_cache_size. But fts_sync() releases cache lock when writing the word. By doing this, InnoDB insert thread increases the innodb fts cache memory and SYNC operation will take more time to complete. Solution: ========= Remove the fts sync operation(FTS_MSG_SYNC_TABLE) from the fts optimize background thread. Instead of that, allow user thread to sync the InnoDB fts cache when the cache size exceeds 512 kb. User thread holds cache lock while doing cache syncing, it make sure that other threads doesn't add the docs into the cache. Removed FTS_MSG_SYNC_TABLE and its related function because we do remove the FTS_MSG_SYNC_TABLE message itself. Removed fts_sync_index_check() and all related function because other threads doesn't add while cache operation going on. --- .../innodb_fts/r/concurrent_insert.result | 2 +- mysql-test/suite/innodb_fts/r/sync.result | 16 +- .../suite/innodb_fts/r/sync_block.result | 35 +-- .../suite/innodb_fts/t/concurrent_insert.test | 2 +- mysql-test/suite/innodb_fts/t/sync.test | 4 +- mysql-test/suite/innodb_fts/t/sync_block.test | 48 +--- storage/innobase/fts/fts0fts.cc | 265 ++++-------------- storage/innobase/fts/fts0opt.cc | 56 +--- storage/innobase/handler/handler0alter.cc | 8 +- storage/innobase/include/fts0fts.h | 9 +- storage/innobase/include/fts0types.h | 32 +-- 11 files changed, 82 insertions(+), 395 deletions(-) diff --git a/mysql-test/suite/innodb_fts/r/concurrent_insert.result b/mysql-test/suite/innodb_fts/r/concurrent_insert.result index e91ea02b1de..3cb48d22df1 100644 --- a/mysql-test/suite/innodb_fts/r/concurrent_insert.result +++ b/mysql-test/suite/innodb_fts/r/concurrent_insert.result @@ -19,7 +19,7 @@ INSERT INTO t2 VALUES('mariadb'); connection default; SET @saved_dbug = @@GLOBAL.debug_dbug; SET GLOBAL debug_dbug ='+d,fts_instrument_sync_request,ib_optimize_wq_hang'; -SET DEBUG_SYNC= 'fts_instrument_sync_request +SET DEBUG_SYNC= 'fts_sync_end SIGNAL drop_index_start WAIT_FOR sync_op'; INSERT INTO t1 VALUES('Keyword'); connect con1,localhost,root,,,; diff --git a/mysql-test/suite/innodb_fts/r/sync.result b/mysql-test/suite/innodb_fts/r/sync.result index 928efffdb21..74a5d2f13fb 100644 --- a/mysql-test/suite/innodb_fts/r/sync.result +++ b/mysql-test/suite/innodb_fts/r/sync.result @@ -11,19 +11,19 @@ INSERT INTO t1(title) VALUES('database'); connection con1; SET @old_dbug = @@SESSION.debug_dbug; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR selected'; +SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR selected'; INSERT INTO t1(title) VALUES('mysql database'); connection default; SET DEBUG_SYNC= 'now WAIT_FOR written'; SET GLOBAL innodb_ft_aux_table="test/t1"; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION database 2 3 2 2 0 database 2 3 2 3 6 mysql 1 3 2 1 0 mysql 1 3 2 3 0 -SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; -WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION SET GLOBAL innodb_ft_aux_table=default; SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); FTS_DOC_ID title @@ -59,7 +59,7 @@ INSERT INTO t1(title) VALUES('mysql'); INSERT INTO t1(title) VALUES('database'); connection con1; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR inserted'; +SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR inserted'; INSERT INTO t1(title) VALUES('mysql database'); connection default; SET DEBUG_SYNC= 'now WAIT_FOR written'; @@ -70,14 +70,14 @@ SET debug_dbug = @old_dbug; SET GLOBAL innodb_ft_aux_table="test/t1"; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +database 4 4 1 4 6 +mysql 4 4 1 4 0 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION database 2 3 2 2 0 database 2 3 2 3 6 -database 4 4 1 4 6 -mysql 1 4 3 1 0 -mysql 1 4 3 3 0 -mysql 1 4 3 4 0 +mysql 1 3 2 1 0 +mysql 1 3 2 3 0 SET GLOBAL innodb_ft_aux_table=default; SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); FTS_DOC_ID title diff --git a/mysql-test/suite/innodb_fts/r/sync_block.result b/mysql-test/suite/innodb_fts/r/sync_block.result index 65bee127e80..c105a3d3b12 100644 --- a/mysql-test/suite/innodb_fts/r/sync_block.result +++ b/mysql-test/suite/innodb_fts/r/sync_block.result @@ -30,6 +30,7 @@ connection con1; connection con2; /* conneciton con2 */ SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); FTS_DOC_ID title +1 mysql database connection default; # make con1 & con2 show up in mysql.slow_log SELECT SLEEP(2); @@ -39,41 +40,11 @@ SLEEP(2) SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; sql_text INSERT INTO t1(title) VALUES('mysql database') +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') SET GLOBAL debug_dbug = @old_debug; TRUNCATE TABLE mysql.slow_log; DROP TABLE t1; -# Case 2: Sync blocks DML(insert) on other tables. -CREATE TABLE t1 ( -FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, -title VARCHAR(200), -FULLTEXT(title) -) ENGINE = InnoDB; -CREATE TABLE t2(id INT); -connection con1; -SET GLOBAL debug_dbug='+d,fts_instrument_sync_request,fts_instrument_sync_sleep'; -SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL begin WAIT_FOR continue'; -INSERT INTO t1(title) VALUES('mysql database'); -connection con2; -SET DEBUG_SYNC= 'now WAIT_FOR begin'; -INSERT INTO t2 VALUES(1); -connection default; -SET DEBUG_SYNC= 'now SIGNAL continue'; -connection con1; -/* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); -connection con2; -/* conneciton con2 */ INSERT INTO t2 VALUES(1); -connection default; -SET DEBUG_SYNC = 'RESET'; -# make con1 & con2 show up in mysql.slow_log -SELECT SLEEP(2); -SLEEP(2) -0 -# slow log results should be empty here. -SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; -sql_text -SET GLOBAL debug_dbug = @old_debug; -TRUNCATE TABLE mysql.slow_log; -DROP TABLE t1,t2; +SET DEBUG_SYNC=RESET; disconnect con1; disconnect con2; # Restore slow log settings. diff --git a/mysql-test/suite/innodb_fts/t/concurrent_insert.test b/mysql-test/suite/innodb_fts/t/concurrent_insert.test index 35debd87cea..d70fc0f63c4 100644 --- a/mysql-test/suite/innodb_fts/t/concurrent_insert.test +++ b/mysql-test/suite/innodb_fts/t/concurrent_insert.test @@ -31,7 +31,7 @@ INSERT INTO t2 VALUES('mariadb'); connection default; SET @saved_dbug = @@GLOBAL.debug_dbug; SET GLOBAL debug_dbug ='+d,fts_instrument_sync_request,ib_optimize_wq_hang'; -SET DEBUG_SYNC= 'fts_instrument_sync_request +SET DEBUG_SYNC= 'fts_sync_end SIGNAL drop_index_start WAIT_FOR sync_op'; send INSERT INTO t1 VALUES('Keyword'); diff --git a/mysql-test/suite/innodb_fts/t/sync.test b/mysql-test/suite/innodb_fts/t/sync.test index f16953ba09f..92c21c9ac83 100644 --- a/mysql-test/suite/innodb_fts/t/sync.test +++ b/mysql-test/suite/innodb_fts/t/sync.test @@ -26,7 +26,7 @@ connection con1; SET @old_dbug = @@SESSION.debug_dbug; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR selected'; +SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR selected'; send INSERT INTO t1(title) VALUES('mysql database'); @@ -73,7 +73,7 @@ connection con1; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR inserted'; +SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR inserted'; send INSERT INTO t1(title) VALUES('mysql database'); diff --git a/mysql-test/suite/innodb_fts/t/sync_block.test b/mysql-test/suite/innodb_fts/t/sync_block.test index 895d2ba8a59..705567296f7 100644 --- a/mysql-test/suite/innodb_fts/t/sync_block.test +++ b/mysql-test/suite/innodb_fts/t/sync_block.test @@ -65,53 +65,7 @@ SET GLOBAL debug_dbug = @old_debug; TRUNCATE TABLE mysql.slow_log; DROP TABLE t1; - ---echo # Case 2: Sync blocks DML(insert) on other tables. -CREATE TABLE t1 ( - FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, - title VARCHAR(200), - FULLTEXT(title) -) ENGINE = InnoDB; - -CREATE TABLE t2(id INT); - -connection con1; - -SET GLOBAL debug_dbug='+d,fts_instrument_sync_request,fts_instrument_sync_sleep'; - -SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL begin WAIT_FOR continue'; - -send INSERT INTO t1(title) VALUES('mysql database'); - -connection con2; - -SET DEBUG_SYNC= 'now WAIT_FOR begin'; - -send INSERT INTO t2 VALUES(1); - -connection default; -SET DEBUG_SYNC= 'now SIGNAL continue'; - -connection con1; ---echo /* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); ---reap - -connection con2; ---echo /* conneciton con2 */ INSERT INTO t2 VALUES(1); ---reap - -connection default; -SET DEBUG_SYNC = 'RESET'; --- echo # make con1 & con2 show up in mysql.slow_log -SELECT SLEEP(2); --- echo # slow log results should be empty here. -SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; - -SET GLOBAL debug_dbug = @old_debug; -TRUNCATE TABLE mysql.slow_log; - -DROP TABLE t1,t2; - +SET DEBUG_SYNC=RESET; disconnect con1; disconnect con2; diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 0a82dc39303..afbb904d5c5 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -38,6 +38,22 @@ Full Text Search interface #include "dict0stats.h" #include "btr0pcur.h" +/** The SYNC state of the cache. There is one instance of this struct +associated with each ADD thread. */ +struct fts_sync_t { + /** Transaction used for SYNCing the cache to disk */ + trx_t *trx; + /** Table with FTS index(es) */ + dict_table_t *table; + /** Max size in bytes of the cache */ + ulint max_cache_size; + /** The doc id at which the cache was noted as being + full, we use this to set the upper_limit field */ + doc_id_t max_doc_id; + /** SYNC start time; only used if fts_enable_diag_print */ + time_t start_time; +}; + static const ulint FTS_MAX_ID_LEN = 32; /** Column name from the FTS config table */ @@ -185,15 +201,8 @@ struct fts_tokenize_param_t { /** Run SYNC on the table, i.e., write out data from the cache to the FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] sync sync state -@param[in] unlock_cache whether unlock cache lock when write node -@param[in] wait whether wait when a sync is in progress @return DB_SUCCESS if all OK */ -static -dberr_t -fts_sync( - fts_sync_t* sync, - bool unlock_cache, - bool wait); +static dberr_t fts_sync(fts_sync_t *sync); /****************************************************************//** Release all resources help by the words rb tree e.g., the node ilist. */ @@ -266,7 +275,6 @@ fts_cache_destroy(fts_cache_t* cache) mysql_mutex_destroy(&cache->init_lock); mysql_mutex_destroy(&cache->deleted_lock); mysql_mutex_destroy(&cache->doc_id_lock); - pthread_cond_destroy(&cache->sync->cond); if (cache->stopword_info.cached_stopword) { rbt_free(cache->stopword_info.cached_stopword); @@ -540,7 +548,6 @@ fts_index_cache_init( for (i = 0; i < FTS_NUM_AUX_INDEX; ++i) { ut_a(index_cache->ins_graph[i] == NULL); - ut_a(index_cache->sel_graph[i] == NULL); } } @@ -610,7 +617,6 @@ fts_cache_create( mem_heap_zalloc(heap, sizeof(fts_sync_t))); cache->sync->table = table; - pthread_cond_init(&cache->sync->cond, nullptr); /* Create the index cache vector that will hold the inverted indexes. */ cache->indexes = ib_vector_create( @@ -935,10 +941,6 @@ fts_cache_index_cache_create( mem_heap_zalloc(static_cast( cache->self_heap->arg), n_bytes)); - index_cache->sel_graph = static_cast( - mem_heap_zalloc(static_cast( - cache->self_heap->arg), n_bytes)); - fts_index_cache_init(cache->sync_heap, index_cache); if (cache->get_docs) { @@ -1012,13 +1014,6 @@ fts_cache_clear( index_cache->ins_graph[j] = NULL; } - - if (index_cache->sel_graph[j] != NULL) { - - que_graph_free(index_cache->sel_graph[j]); - - index_cache->sel_graph[j] = NULL; - } } index_cache->doc_stats = NULL; @@ -1311,8 +1306,7 @@ fts_cache_add_doc( ib_vector_last(word->nodes)); } - if (fts_node == NULL || fts_node->synced - || fts_node->ilist_size > FTS_ILIST_MAX_SIZE + if (!fts_node || fts_node->ilist_size > FTS_ILIST_MAX_SIZE || doc_id < fts_node->last_doc_id) { fts_node = static_cast( @@ -3284,7 +3278,7 @@ fts_add_doc_from_tuple( if (cache->total_size > fts_max_cache_size / 5 || fts_need_sync) { - fts_sync(cache->sync, true, false); + fts_sync(cache->sync); } mtr_start(&mtr); @@ -3444,42 +3438,34 @@ fts_add_doc_by_id( get_doc->index_cache, doc_id, doc.tokens); - bool need_sync = !cache->sync->in_progress - && (fts_need_sync - || (cache->total_size - - cache->total_size_at_sync) - > fts_max_cache_size / 10); - if (need_sync) { - cache->total_size_at_sync = - cache->total_size; - } + /** FTS cache sync should happen + frequently. Because user thread + shouldn't hold the cache lock for + longer time. So cache should sync + whenever cache size exceeds 512 KB */ + bool need_sync = + cache->total_size > 512*1024; mysql_mutex_unlock(&table->fts->cache->lock); DBUG_EXECUTE_IF( "fts_instrument_sync", - fts_optimize_request_sync_table(table); - mysql_mutex_lock(&cache->lock); - if (cache->sync->in_progress) - my_cond_wait( - &cache->sync->cond, - &cache->lock.m_mutex); - mysql_mutex_unlock(&cache->lock); + fts_sync_table(table); ); DBUG_EXECUTE_IF( "fts_instrument_sync_debug", - fts_sync(cache->sync, true, true); + fts_sync(cache->sync); ); DEBUG_SYNC_C("fts_instrument_sync_request"); DBUG_EXECUTE_IF( "fts_instrument_sync_request", - fts_optimize_request_sync_table(table); + need_sync= true; ); if (need_sync) { - fts_optimize_request_sync_table(table); + fts_sync_table(table); } mtr_start(&mtr); @@ -3846,15 +3832,13 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_sync_write_words( trx_t* trx, - fts_index_cache_t* index_cache, - bool unlock_cache) + fts_index_cache_t* index_cache) { fts_table_t fts_table; ulint n_nodes = 0; ulint n_words = 0; const ib_rbt_node_t* rbt_node; dberr_t error = DB_SUCCESS; - ibool print_error = FALSE; dict_table_t* table = index_cache->index->table; FTS_INIT_INDEX_TABLE( @@ -3885,53 +3869,35 @@ fts_sync_write_words( fts_table.suffix = fts_get_suffix(selected); - /* We iterate over all the nodes even if there was an error */ for (i = 0; i < ib_vector_size(word->nodes); ++i) { fts_node_t* fts_node = static_cast( ib_vector_get(word->nodes, i)); - if (fts_node->synced) { - continue; - } else { - fts_node->synced = true; - } + error = fts_write_node( + trx, &index_cache->ins_graph[selected], + &fts_table, &word->text, fts_node); - /*FIXME: we need to handle the error properly. */ - if (error == DB_SUCCESS) { - if (unlock_cache) { - mysql_mutex_unlock( - &table->fts->cache->lock); - } - - error = fts_write_node( - trx, - &index_cache->ins_graph[selected], - &fts_table, &word->text, fts_node); - - DEBUG_SYNC_C("fts_write_node"); - DBUG_EXECUTE_IF("fts_write_node_crash", + DEBUG_SYNC_C("fts_write_node"); + DBUG_EXECUTE_IF("fts_write_node_crash", DBUG_SUICIDE();); - DBUG_EXECUTE_IF( - "fts_instrument_sync_sleep", + DBUG_EXECUTE_IF("fts_instrument_sync_sleep", std::this_thread::sleep_for( std::chrono::seconds(1));); - if (unlock_cache) { - mysql_mutex_lock( - &table->fts->cache->lock); - } + if (error != DB_SUCCESS) { + goto err_exit; } } n_nodes += ib_vector_size(word->nodes); - if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) { + if (UNIV_UNLIKELY(error != DB_SUCCESS)) { +err_exit: ib::error() << "(" << error << ") writing" " word node to FTS auxiliary index table " << table->name; - print_error = TRUE; } } @@ -3990,58 +3956,7 @@ fts_sync_index( ut_ad(rbt_validate(index_cache->words)); - return(fts_sync_write_words(trx, index_cache, sync->unlock_cache)); -} - -/** Check if index cache has been synced completely -@param[in,out] index_cache index cache -@return true if index is synced, otherwise false. */ -static -bool -fts_sync_index_check( - fts_index_cache_t* index_cache) -{ - const ib_rbt_node_t* rbt_node; - - for (rbt_node = rbt_first(index_cache->words); - rbt_node != NULL; - rbt_node = rbt_next(index_cache->words, rbt_node)) { - - fts_tokenizer_word_t* word; - word = rbt_value(fts_tokenizer_word_t, rbt_node); - - fts_node_t* fts_node; - fts_node = static_cast(ib_vector_last(word->nodes)); - - if (!fts_node->synced) { - return(false); - } - } - - return(true); -} - -/** Reset synced flag in index cache when rollback -@param[in,out] index_cache index cache */ -static -void -fts_sync_index_reset( - fts_index_cache_t* index_cache) -{ - const ib_rbt_node_t* rbt_node; - - for (rbt_node = rbt_first(index_cache->words); - rbt_node != NULL; - rbt_node = rbt_next(index_cache->words, rbt_node)) { - - fts_tokenizer_word_t* word; - word = rbt_value(fts_tokenizer_word_t, rbt_node); - - fts_node_t* fts_node; - fts_node = static_cast(ib_vector_last(word->nodes)); - - fts_node->synced = false; - } + return(fts_sync_write_words(trx, index_cache)); } /** Commit the SYNC, change state of processed doc ids etc. @@ -4074,14 +3989,14 @@ fts_sync_commit( sync, cache->deleted_doc_ids); } - /* We need to do this within the deleted lock since fts_delete() can - attempt to add a deleted doc id to the cache deleted id array. */ - fts_cache_clear(cache); - DEBUG_SYNC_C("fts_deleted_doc_ids_clear"); - fts_cache_init(cache); - mysql_mutex_unlock(&cache->lock); - if (UNIV_LIKELY(error == DB_SUCCESS)) { + /* We need to do this within the deleted lock + since fts_delete() can attempt to add a deleted + doc id to the cache deleted id array. */ + fts_cache_clear(cache); + DEBUG_SYNC_C("fts_deleted_doc_ids_clear"); + fts_cache_init(cache); + mysql_mutex_unlock(&cache->lock); fts_sql_commit(trx); } else { fts_sql_rollback(trx); @@ -4123,10 +4038,6 @@ fts_sync_rollback( index_cache = static_cast( ib_vector_get(cache->indexes, i)); - /* Reset synced flag so nodes will not be skipped - in the next sync, see fts_sync_write_words(). */ - fts_sync_index_reset(index_cache); - for (j = 0; fts_index_selector[j].value; ++j) { if (index_cache->ins_graph[j] != NULL) { @@ -4135,13 +4046,6 @@ fts_sync_rollback( index_cache->ins_graph[j] = NULL; } - - if (index_cache->sel_graph[j] != NULL) { - - que_graph_free(index_cache->sel_graph[j]); - - index_cache->sel_graph[j] = NULL; - } } } @@ -4160,12 +4064,7 @@ FTS auxiliary INDEX table and clear the cache at the end. @param[in] unlock_cache whether unlock cache lock when write node @param[in] wait whether wait when a sync is in progress @return DB_SUCCESS if all OK */ -static -dberr_t -fts_sync( - fts_sync_t* sync, - bool unlock_cache, - bool wait) +static dberr_t fts_sync(fts_sync_t *sync) { if (srv_read_only_mode) { return DB_READ_ONLY; @@ -4176,33 +4075,13 @@ fts_sync( fts_cache_t* cache = sync->table->fts->cache; mysql_mutex_lock(&cache->lock); - - /* Check if cache is being synced. - Note: we release cache lock in fts_sync_write_words() to - avoid long wait for the lock by other threads. */ - if (sync->in_progress) { - if (!wait) { - mysql_mutex_unlock(&cache->lock); - return(DB_SUCCESS); - } - do { - my_cond_wait(&sync->cond, &cache->lock.m_mutex); - } while (sync->in_progress); - } - - sync->unlock_cache = unlock_cache; - sync->in_progress = true; - DEBUG_SYNC_C("fts_sync_begin"); fts_sync_begin(sync); -begin_sync: const size_t fts_cache_size= fts_max_cache_size; if (cache->total_size > fts_cache_size) { /* Avoid the case: sync never finish when insert/update keeps comming. */ - ut_ad(sync->unlock_cache); - sync->unlock_cache = false; ib::warn() << "Total InnoDB FTS size " << cache->total_size << " for the table " << cache->sync->table->name @@ -4226,52 +4105,23 @@ begin_sync: error = fts_sync_index(sync, index_cache); if (error != DB_SUCCESS) { - goto end_sync; - } - - if (!sync->unlock_cache - && cache->total_size < fts_max_cache_size) { - /* Reset the unlock cache if the value - is less than innodb_ft_cache_size */ - sync->unlock_cache = true; + goto err_exit; } } DBUG_EXECUTE_IF("fts_instrument_sync_interrupted", - sync->interrupted = true; error = DB_INTERRUPTED; - goto end_sync; + goto err_exit; ); - /* Make sure all the caches are synced. */ - for (i = 0; i < ib_vector_size(cache->indexes); ++i) { - fts_index_cache_t* index_cache; - - index_cache = static_cast( - ib_vector_get(cache->indexes, i)); - - if (index_cache->index->to_be_dropped - || fts_sync_index_check(index_cache)) { - continue; - } - - goto begin_sync; - } - -end_sync: - if (error == DB_SUCCESS && !sync->interrupted) { + if (error == DB_SUCCESS) { error = fts_sync_commit(sync); } else { +err_exit: fts_sync_rollback(sync); + return error; } - mysql_mutex_lock(&cache->lock); - ut_ad(sync->in_progress); - sync->interrupted = false; - sync->in_progress = false; - pthread_cond_broadcast(&sync->cond); - mysql_mutex_unlock(&cache->lock); - /* We need to check whether an optimize is required, for that we make copies of the two variables that control the trigger. These variables can change behind our back and we don't want to hold the @@ -4283,6 +4133,7 @@ end_sync: mysql_mutex_unlock(&cache->deleted_lock); + DEBUG_SYNC_C("fts_sync_end"); return(error); } @@ -4291,12 +4142,12 @@ FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] table fts table @param[in] wait whether wait for existing sync to finish @return DB_SUCCESS on success, error code on failure. */ -dberr_t fts_sync_table(dict_table_t* table, bool wait) +dberr_t fts_sync_table(dict_table_t* table) { ut_ad(table->fts); return table->space && !table->corrupted && table->fts->cache - ? fts_sync(table->fts->cache->sync, !wait, wait) + ? fts_sync(table->fts->cache->sync) : DB_SUCCESS; } diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 030dc438193..9cc56f184fd 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -83,9 +83,8 @@ enum fts_msg_type_t { FTS_MSG_ADD_TABLE, /*!< Add table to the optimize thread's work queue */ - FTS_MSG_DEL_TABLE, /*!< Remove a table from the optimize + FTS_MSG_DEL_TABLE /*!< Remove a table from the optimize threads work queue */ - FTS_MSG_SYNC_TABLE /*!< Sync fts cache of a table */ }; /** Compressed list of words that have been read from FTS INDEX @@ -2625,36 +2624,6 @@ fts_optimize_remove_table( mysql_mutex_unlock(&fts_optimize_wq->mutex); } -/** Send sync fts cache for the table. -@param[in] table table to sync */ -void -fts_optimize_request_sync_table( - dict_table_t* table) -{ - /* if the optimize system not yet initialized, return */ - if (!fts_optimize_wq) { - return; - } - - mysql_mutex_lock(&fts_optimize_wq->mutex); - - /* FTS optimizer thread is already exited */ - if (fts_opt_start_shutdown) { - ib::info() << "Try to sync table " << table->name - << " after FTS optimize thread exiting."; - } else if (table->fts->sync_message) { - /* If the table already has SYNC message in - fts_optimize_wq queue then ignore it */ - } else { - add_msg(fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, table)); - table->fts->sync_message = true; - DBUG_EXECUTE_IF("fts_optimize_wq_count_check", - DBUG_ASSERT(fts_optimize_wq->length <= 1000);); - } - - mysql_mutex_unlock(&fts_optimize_wq->mutex); -} - /** Add a table to fts_slots if it doesn't already exist. */ static bool fts_optimize_new_table(dict_table_t* table) { @@ -2796,7 +2765,8 @@ static void fts_optimize_sync_table(dict_table_t *table, if (sync_table->fts && sync_table->fts->cache && sync_table->is_accessible()) { - fts_sync_table(sync_table, false); + fts_sync_table(sync_table); + if (process_message) { mysql_mutex_lock(&fts_optimize_wq->mutex); @@ -2896,24 +2866,6 @@ retry_later: --n_tables; } break; - - case FTS_MSG_SYNC_TABLE: - if (UNIV_UNLIKELY(wsrep_sst_disable_writes)) { - add_msg(msg); - goto retry_later; - } - - DBUG_EXECUTE_IF( - "fts_instrument_msg_sync_sleep", - std::this_thread::sleep_for( - std::chrono::milliseconds( - 300));); - - fts_optimize_sync_table( - static_cast(msg->ptr), - true); - break; - default: ut_error; } @@ -3046,7 +2998,7 @@ void fts_sync_during_ddl(dict_table_t* table) if (!sync_message) return; - fts_sync_table(table, false); + fts_sync_table(table); mysql_mutex_lock(&fts_optimize_wq->mutex); table->fts->sync_message = false; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 57eecd47ebb..19b6407f525 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -11415,12 +11415,8 @@ foreign_fail: ut_d(dict_table_check_for_dup_indexes( ctx->new_table, CHECK_ABORTED_OK)); -#ifdef UNIV_DEBUG - if (!(ctx->new_table->fts != NULL - && ctx->new_table->fts->cache->sync->in_progress)) { - ut_a(fts_check_cached_index(ctx->new_table)); - } -#endif + ut_ad(!ctx->new_table->fts + || fts_check_cached_index(ctx->new_table)); } unlock_and_close_files(deleted, trx); diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 774221dd208..043a464dd30 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -656,12 +656,6 @@ fts_optimize_remove_table( void fts_optimize_shutdown(); -/** Send sync fts cache for the table. -@param[in] table table to sync */ -void -fts_optimize_request_sync_table( - dict_table_t* table); - /**********************************************************************//** Take a FTS savepoint. */ void @@ -716,9 +710,8 @@ fts_savepoint_rollback_last_stmt( /** Run SYNC on the table, i.e., write out data from the cache to the FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] table fts table -@param[in] wait whether to wait for existing sync to finish @return DB_SUCCESS on success, error code on failure. */ -dberr_t fts_sync_table(dict_table_t* table, bool wait = true); +dberr_t fts_sync_table(dict_table_t* table); /****************************************************************//** Create an FTS index cache. */ diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index fb278d543c4..04e99d595c5 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -75,7 +75,6 @@ struct fts_index_cache_t { que_t** ins_graph; /*!< Insert query graphs */ - que_t** sel_graph; /*!< Select query graphs */ CHARSET_INFO* charset; /*!< charset */ }; @@ -87,35 +86,7 @@ struct fts_stopword_t { CHARSET_INFO* charset; /*!< charset for stopword */ }; -/** The SYNC state of the cache. There is one instance of this struct -associated with each ADD thread. */ -struct fts_sync_t { - trx_t* trx; /*!< The transaction used for SYNCing - the cache to disk */ - dict_table_t* table; /*!< Table with FTS index(es) */ - ulint max_cache_size; /*!< Max size in bytes of the cache */ - ibool cache_full; /*!< flag, when true it indicates that - we need to sync the cache to disk */ - ulint lower_index; /*!< the start index of the doc id - vector from where to start adding - documents to the FTS cache */ - ulint upper_index; /*!< max index of the doc id vector to - add to the FTS cache */ - ibool interrupted; /*!< TRUE if SYNC was interrupted */ - doc_id_t min_doc_id; /*!< The smallest doc id added to the - cache. It should equal to - doc_ids[lower_index] */ - doc_id_t max_doc_id; /*!< The doc id at which the cache was - noted as being full, we use this to - set the upper_limit field */ - time_t start_time; /*!< SYNC start time; only used if - fts_enable_diag_print */ - bool in_progress; /*!< flag whether sync is in progress.*/ - bool unlock_cache; /*!< flag whether unlock cache when - write fts node */ - /** condition variable for in_progress; used with table->fts->cache->lock */ - pthread_cond_t cond; -}; +struct fts_sync_t; /** The cache for the FTS system. It is a memory-based inverted index that new entries are added to, until it grows over the configured maximum @@ -204,7 +175,6 @@ struct fts_node_t { ulint ilist_size_alloc; /*!< Allocated size of ilist in bytes */ - bool synced; /*!< flag whether the node is synced */ }; /** A tokenizer word. Contains information about one word. */ From d4760d8c016c6b65a473fbd6f66ca4db02749106 Mon Sep 17 00:00:00 2001 From: Tuukka Pasanen Date: Mon, 23 May 2022 10:00:13 +0300 Subject: [PATCH 03/11] MDEV-28642: Suspend obvious false-positive Lintian warnings MariaDB codebase is huge and Lintian has lots of test than can fire false-positive warnings which leads to situation where real problems can't be spotted. Suspend obvious false-positive Lintian warnings and let Lintian problems that needs some love shine out. Suspends in package mariadb-test-data Supporting BSD family needs to use '/usr/bin/env perl' and not '/usr/bin/perl' Perl script are for testing and not for production in mariadb-test-data package: * incorrect-path-for-interpreter There is several files with national-encoding which are test file so they can't be in unicode charset * national-encoding Serveral test paths are intentionally repeated: * repeated-path-segment Suspends in package mariadb-test Supporting BSD family needs to use '/usr/bin/env perl' and not '/usr/bin/perl' Perl script are for testing and not for production in mariadb-test-data package: * incorrect-path-for-interpreter Suspends in package source package Remade some 'version-substvar-for-external-package' to use regex. MGroonga is missing source file 'jquery-ui-1.8.18.custom.js' correct lintian suspend with regex: * source-is-missing There is several files with very long line lenghts. Add suspends for those that can't be corrected in several places. Most of them are test result files, SQL test files or intentional long lines that can't be splitted. * very-long-line-length-in-source-file There is several autogenerated C++ files which probably should not be there but they should not do any harm: * source-contains-autogenerated-visual-c++-file --- debian/mariadb-test-data.lintian-overrides | 30 +++++++ debian/mariadb-test.lintian-overrides | 5 ++ debian/source/lintian-overrides | 93 +++++++++++++++++----- 3 files changed, 108 insertions(+), 20 deletions(-) diff --git a/debian/mariadb-test-data.lintian-overrides b/debian/mariadb-test-data.lintian-overrides index 4ff05a3be2a..2fb5bc6e604 100644 --- a/debian/mariadb-test-data.lintian-overrides +++ b/debian/mariadb-test-data.lintian-overrides @@ -1,3 +1,33 @@ # These should be moved, see https://jira.mariadb.org/browse/MDEV-21654 arch-dependent-file-in-usr-share usr/share/mysql/mysql-test/suite/plugins/pam/pam_mariadb_mtr.so arch-independent-package-contains-binary-or-object usr/share/mysql/mysql-test/suite/plugins/pam/pam_mariadb_mtr.so +# Mainly for support for *BSD family. Not right way to do but this is test package and not for production +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/std_data/checkDBI_DBD-MariaDB.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/engines/rr_trx/run_stress_tx_rr.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/funcs_1/lib/DataGen_local.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/funcs_1/lib/DataGen_modify.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/funcs_2/lib/gen_charset_utf8.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/suite/rpl/extension/checksum.pl] +# Intentional for test files +national-encoding usr/share/mysql/mysql-test/* +# Extra test documentation files that really need to be kept in context in test directory +package-contains-documentation-outside-usr-share-doc usr/share/mysql/mysql-test/* +# Intentional directory structure +repeated-path-segment auth_gssapi usr/share/mysql/mysql-test/plugin/auth_gssapi/auth_gssapi/ +repeated-path-segment connect usr/share/mysql/mysql-test/plugin/connect/connect/ +repeated-path-segment disks usr/share/mysql/mysql-test/plugin/disks/disks/ +repeated-path-segment func_test usr/share/mysql/mysql-test/plugin/func_test/func_test/ +repeated-path-segment metadata_lock_info usr/share/mysql/mysql-test/plugin/metadata_lock_info/metadata_lock_info/ +repeated-path-segment mroonga usr/share/mysql/mysql-test/plugin/mroonga/mroonga/ +repeated-path-segment mroonga usr/share/mysql/mysql-test/plugin/mroonga/mroonga/include/mroonga/ +repeated-path-segment oqgraph usr/share/mysql/mysql-test/plugin/oqgraph/oqgraph/ +repeated-path-segment query_response_time usr/share/mysql/mysql-test/plugin/query_response_time/query_response_time/ +repeated-path-segment rocksdb usr/share/mysql/mysql-test/plugin/rocksdb/rocksdb/ +repeated-path-segment sequence usr/share/mysql/mysql-test/plugin/sequence/sequence/ +repeated-path-segment sphinx usr/share/mysql/mysql-test/plugin/sphinx/sphinx/ +repeated-path-segment spider usr/share/mysql/mysql-test/plugin/spider/spider/ +repeated-path-segment tokudb usr/share/mysql/mysql-test/plugin/tokudb/tokudb/ +repeated-path-segment type_inet usr/share/mysql/mysql-test/plugin/type_inet/type_inet/ +repeated-path-segment type_test usr/share/mysql/mysql-test/plugin/type_test/type_test/ +repeated-path-segment user_variables usr/share/mysql/mysql-test/plugin/user_variables/user_variables/ +repeated-path-segment wsrep_info usr/share/mysql/mysql-test/plugin/wsrep_info/wsrep_info/ diff --git a/debian/mariadb-test.lintian-overrides b/debian/mariadb-test.lintian-overrides index 9de1b3784ff..b9f45b862b5 100644 --- a/debian/mariadb-test.lintian-overrides +++ b/debian/mariadb-test.lintian-overrides @@ -1,3 +1,8 @@ # These should be moved, see https://jira.mariadb.org/browse/MDEV-21653 arch-dependent-file-in-usr-share usr/share/mysql/mysql-test/lib/My/SafeProcess/my_safe_process arch-dependent-file-in-usr-share usr/share/mysql/mysql-test/lib/My/SafeProcess/wsrep_check_version +# Mainly for support for *BSD family. Not right way to do but this is test package and not for production +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/lib/process-purecov-annotations.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/lib/v1/mysql-test-run.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/mysql-stress-test.pl] +incorrect-path-for-interpreter /usr/bin/env perl != /usr/bin/perl [usr/share/mysql/mysql-test/mysql-test-run.pl] diff --git a/debian/source/lintian-overrides b/debian/source/lintian-overrides index a9500a44598..2fab3096024 100644 --- a/debian/source/lintian-overrides +++ b/debian/source/lintian-overrides @@ -1,32 +1,85 @@ # Necessary for drop-in-place-replacement upgrades on mysql-server/-client # since package breaks/replaces these but at the same time also provides them -version-substvar-for-external-package mariadb-client-core-10.5 -> mysql-client-5.5 -version-substvar-for-external-package mariadb-server-10.5 -> mysql-server +version-substvar-for-external-package mariadb-client-core-10.5 -> mysql-client-5.* version-substvar-for-external-package mariadb-server-10.5 -> mysql-server version-substvar-for-external-package libmariadb-dev -> libmysqlclient-dev version-substvar-for-external-package libmariadb-dev -> libmysqld-dev -version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-5.5 -version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-5.6 -version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-5.7 -version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-8.0 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.1 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.1 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.5 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.5 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.6 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.6 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.7 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.7 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-8.0 -version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-8.0 +version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-5.* +version-substvar-for-external-package mariadb-server-10.5 -> mysql-client-8.* +version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-5.* +version-substvar-for-external-package mariadb-client-10.5 -> mysql-client-core-8.* version-substvar-for-external-package libmariadbd-dev -> libmariadbclient-dev # ColumnStore not used in Debian, safe to ignore. Reported upstream in https://jira.mariadb.org/browse/MDEV-24124 source-is-missing storage/columnstore/columnstore/utils/jemalloc/libjemalloc.so.2 # Must be fixed upstream -source-is-missing storage/mroonga/vendor/groonga/examples/dictionary/html/js/jquery-ui-1.8.18.custom.js * +source-is-missing storage/mroonga/vendor/groonga/examples/dictionary/html/js/jquery-ui-*.custom.js # Intentional control relationships -version-substvar-for-external-package Replaces (line 216) ${source:Version} libmariadbd-dev -> libmariadbclient-dev -version-substvar-for-external-package Replaces (line 66) ${source:Version} libmariadb-dev -> libmysqlclient-dev -version-substvar-for-external-package Replaces (line 66) ${source:Version} libmariadb-dev -> libmysqld-dev +version-substvar-for-external-package Replaces * libmariadbd-dev -> libmariadbclient-dev +version-substvar-for-external-package Replaces * libmariadb-dev -> libmysqlclient-dev +version-substvar-for-external-package Replaces * libmariadb-dev -> libmysqld-dev # We can't change build dependencies on a stable branch (10.5..10.8) so just override this missing-build-dependency-for-dh-addon systemd * +# Data or test files where long lines are justified +very-long-line-length-in-source-file *.test * +very-long-line-length-in-source-file *.result * +very-long-line-length-in-source-file BUILD/compile-* +very-long-line-length-in-source-file *COPYING.rtf * +# These are mainly found under extra/wolfssl +very-long-line-length-in-source-file *.cproject * +very-long-line-length-in-source-file *.md * +very-long-line-length-in-source-file *.scfg * +very-long-line-length-in-source-file *.launch * +very-long-line-length-in-source-file extra/wolfssl/wolfssl/IDE/Espressif/ESP-IDF/test/test_wolfssl.c * +very-long-line-length-in-source-file extra/wolfssl/wolfssl/configure.ac * +very-long-line-length-in-source-file extra/wolfssl/wolfssl/doc/formats/html/html_changes/tabs.css * +# Preprocessed C files which have long lines +very-long-line-length-in-source-file extra/wolfssl/wolfssl/wolfcrypt/src/*.i * +# These are all results for test cases and similar so they can be +# especially formatted to be too long +very-long-line-length-in-source-file mysql-test/*.dump * +very-long-line-length-in-source-file mysql-test/*.inc * +very-long-line-length-in-source-file mysql-test/*.rdiff * +very-long-line-length-in-source-file mysql-test/*.txt * +very-long-line-length-in-source-file mysql-test/*.weekly * +# Test file +very-long-line-length-in-source-file plugin/handler_socket/regtest/test_01_lib/test19.expected * +# SQL source file that has very long inserts/selects +very-long-line-length-in-source-file mysql-test/std_data/init_file_longline_3816.sql * +very-long-line-length-in-source-file scripts/fill_help_tables.sql * +very-long-line-length-in-source-file scripts/mysql_system_tables.sql * +very-long-line-length-in-source-file scripts/mysql_test_data_timezone.sql * +# Machine formated HTML +very-long-line-length-in-source-file sql/share/charsets/languages.html * +very-long-line-length-in-source-file sql/share/errmsg-utf8.txt * +# Very long test string +very-long-line-length-in-source-file storage/archive/archive_test.c line 30 is 1051 characters long (>512) +# autogenerated thrift file +very-long-line-length-in-source-file storage/cassandra/gen-cpp/cassandra_types.h * +# ColumnStore ignores +# In Directory mysql-test are some long test includes +very-long-line-length-in-source-file storage/columnstore/columnstore/.drone.jsonnet * +very-long-line-length-in-source-file storage/columnstore/columnstore/CMakeLists.txt * +very-long-line-length-in-source-file storage/columnstore/columnstore/mysql-test/columnstore/csinternal/include/autopilot_create_datatypetestm_tables.inc * +very-long-line-length-in-source-file storage/columnstore/columnstore/mysql-test/columnstore/csinternal/include/autopilot_create_datatypeupdate_table.inc * +very-long-line-length-in-source-file storage/columnstore/columnstore/*.xmi * +very-long-line-length-in-source-file storage/columnstore/columnstore/dbcon/doc/q19_plan.txt * +very-long-line-length-in-source-file storage/columnstore/columnstore/utils/udfsdk/docs/source/reference/mcsv1Context.rst * +very-long-line-length-in-source-file storage/columnstore/columnstore/utils/winport/win_setup_mysql_part1.sql * +# Minified CSS files. These appear in several places +very-long-line-length-in-source-file *badge_only.css * +very-long-line-length-in-source-file *theme.css line * +# General storage ignores +very-long-line-length-in-source-file storage/mroonga/vendor/groonga/examples/dictionary/html/css/smoothness/jquery-ui-1.8.12.custom.css * +very-long-line-length-in-source-file storage/rocksdb/mysql-test/rocksdb/t/bypass_select_basic_bloom-master.opt * +very-long-line-length-in-source-file storage/rocksdb/mysql-test/rocksdb/t/type_enum.inc * +very-long-line-length-in-source-file storage/rocksdb/mysql-test/rocksdb/t/type_set.inc * +very-long-line-length-in-source-file storage/rocksdb/rocksdb/docs/_includes/footer.html * +very-long-line-length-in-source-file storage/rocksdb/rocksdb/docs/_posts/*.markdown line * +very-long-line-length-in-source-file storage/spider/mysql-test/spider/bugfix/include/sql_mode_init.inc * +very-long-line-length-in-source-file storage/tokudb/PerconaFT/cmake_modules/TokuBuildTagDatabases.cmake * +very-long-line-length-in-source-file storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/m4/po.m4 * +# These are generated files which should not make any harm +source-contains-autogenerated-visual-c++-file storage/columnstore/columnstore/*.rc +source-contains-autogenerated-visual-c++-file storage/columnstore/columnstore/*.h +source-contains-autogenerated-visual-c++-file win/upgrade_wizard/resource.h +source-contains-autogenerated-visual-c++-file win/upgrade_wizard/upgrade.rc From 1bcd87cdcc8d6db7f17e06f7114ab9ee657a61b6 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 13 Jun 2022 16:10:41 +0300 Subject: [PATCH 04/11] Fixed maria.maria-recover and maria.encrypt-no-key test files These had been damaged during some merge which caused --embedded test to fail --- mysql-test/suite/maria/encrypt-no-key.result | 6 +++--- mysql-test/suite/maria/encrypt-no-key.test | 2 ++ mysql-test/suite/maria/maria-recover.result | 4 ++-- mysql-test/suite/maria/maria-recover.test | 1 + 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/maria/encrypt-no-key.result b/mysql-test/suite/maria/encrypt-no-key.result index bdc8a79ff9a..5229158d525 100644 --- a/mysql-test/suite/maria/encrypt-no-key.result +++ b/mysql-test/suite/maria/encrypt-no-key.result @@ -6,12 +6,12 @@ create table t1 (pk int primary key, a int, key(a)) engine=aria transactional=1; alter table t1 disable keys; insert into t1 values (1,1); alter table t1 enable keys; -ERROR HY000: Unknown key id 1 for ./test/t1. Can't continue! +ERROR HY000: Unknown key id 1 for test/t1. Can't continue! repair table t1 use_frm; Table Op Msg_type Msg_text test.t1 repair warning Number of rows changed from 0 to 1 -test.t1 repair Error Unknown key id 1 for ./test/t1. Can't continue! -test.t1 repair Error Unknown key id 1 for ./test/t1. Can't continue! +test.t1 repair Error Unknown key id 1 for test/t1. Can't continue! +test.t1 repair Error Unknown key id 1 for test/t1. Can't continue! test.t1 repair status OK drop table t1; set global aria_encrypt_tables= default; diff --git a/mysql-test/suite/maria/encrypt-no-key.test b/mysql-test/suite/maria/encrypt-no-key.test index eebc2a102d3..1157263ca79 100644 --- a/mysql-test/suite/maria/encrypt-no-key.test +++ b/mysql-test/suite/maria/encrypt-no-key.test @@ -10,9 +10,11 @@ create table t1 (pk int primary key, a int, key(a)) engine=aria transactional=1; alter table t1 disable keys; insert into t1 values (1,1); --replace_result \\ / +--replace_regex /for .*test/for test/ --error 192 alter table t1 enable keys; --replace_result \\ / +--replace_regex /for .*test/for test/ repair table t1 use_frm; drop table t1; set global aria_encrypt_tables= default; diff --git a/mysql-test/suite/maria/maria-recover.result b/mysql-test/suite/maria/maria-recover.result index 788cd7eaf5a..7bb90cc0405 100644 --- a/mysql-test/suite/maria/maria-recover.result +++ b/mysql-test/suite/maria/maria-recover.result @@ -26,9 +26,9 @@ a ThursdayMorningsMarket ThursdayMorningsMarketb Warnings: -Error 145 Got error '145 "Table was marked as crashed and should be repaired"' for './mysqltest/t_corrupted2' +Error 145 Got error '145 "Table was marked as crashed and should be repaired"' for 't_corrupted2' Warning 1034 1 client is using or hasn't closed the table properly -Error 176 Got error '176 "Read page with wrong checksum"' for './mysqltest/t_corrupted2.MAI' +Error 176 Got error '176 "Read page with wrong checksum"' for 't_corrupted2.MAI' Error 1034 Can't read indexpage from page: 1, error: 176 select * from t_corrupted2; a diff --git a/mysql-test/suite/maria/maria-recover.test b/mysql-test/suite/maria/maria-recover.test index cea185e7ab5..5ddfd6d202e 100644 --- a/mysql-test/suite/maria/maria-recover.test +++ b/mysql-test/suite/maria/maria-recover.test @@ -79,6 +79,7 @@ perl; close FILE; EOF --replace_result \\ / +--replace_regex /for '.*t_corrupted2/for 't_corrupted2/ --enable_prepare_warnings select * from t_corrupted2; # should show corruption and repair messages --disable_prepare_warnings From 65dd31088a62b87d38899453720e667e4f1bf982 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 13 Jun 2022 15:40:51 +0300 Subject: [PATCH 05/11] Update magic file with Aria table files and ddl log --- support-files/magic | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/support-files/magic b/support-files/magic index b3f3b3ea29d..accc3999ddb 100644 --- a/support-files/magic +++ b/support-files/magic @@ -1,26 +1,26 @@ # # Add the following to the end of your /etc/magic file to get the 'file' -# command to recognize some MySQL files. +# command to recognize some MariaDB / MySQL files. # -0 beshort 0xfe01 MySQL table definition file +0 beshort 0xfe01 MariaDB/MySQL table definition file >2 byte x Version %d -0 belong&0xffffff00 0xfefe0700 MySQL MyISAM index file +0 belong&0xffffff00 0xfefe0700 MariaDB/MySQL MyISAM index file >3 byte x Version %d -0 belong&0xffffff00 0xfefe0800 MySQL MyISAM compressed data file +0 belong&0xffffff00 0xfefe0800 MariaDB/MySQL MyISAM compressed data file >3 byte x Version %d -0 belong&0xffffff00 0xfefe0900 MySQL Maria index file +0 belong&0xffffff00 0xfefe0900 MariaDB Aaria index file >3 byte x Version %d -0 belong&0xffffff00 0xfefe0A00 MySQL Maria compressed data file +0 belong&0xffffff00 0xfefe0A00 MariaDB Aaria compressed data file >3 byte x Version %d -0 belong&0xffffff00 0xfefe0500 MySQL ISAM index file +0 belong&0xffffff00 0xfefe0500 MariaDB/MySQL ISAM index file >3 byte x Version %d -0 belong&0xffffff00 0xfefe0600 MySQL ISAM compressed data file +0 belong&0xffffff00 0xfefe0600 MariaDB/MySQL ISAM compressed data file >3 byte x Version %d -0 string \376bin MySQL replication log +0 string \376bin MariaDB/MySQL replication log 0 belong&0xffffff00 0xfefe0b00 ->4 string MARIALOG MySQL Maria transaction log file +>4 string MARIALOG MariaDB Aria transaction log file >>3 byte x Version %d -0 belong&0xffffff00 0xfefe0c00 ->4 string MACF MySQL Maria control file ->>3 byte x Version %d - +0 belong&0xffffff00 0xfefe0c00 MariaDB Aria control file +>3 byte x Version %d +0 belong&0xffffff00 0xfefe0b00 MariaDB DDL recovery log +>3 byte x Version %d From 1f3f4571934140fd81f54aa366cc4656544c89f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 13 Jun 2022 17:05:31 +0300 Subject: [PATCH 06/11] MDEV-28802 DROP DATABASE in InnoDB still is case-insensitive innodb_drop_database(): Use explicit TO_BINARY casts on SYS_TABLES.NAME, which for historical reasons uses the wrong collation latin1_swedish_ci instead of BINARY. --- mysql-test/suite/innodb/r/dropdb_cs.result | 16 ++++++++++++++++ mysql-test/suite/innodb/t/dropdb_cs.test | 17 +++++++++++++++++ storage/innobase/handler/ha_innodb.cc | 3 ++- 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/innodb/r/dropdb_cs.result create mode 100644 mysql-test/suite/innodb/t/dropdb_cs.test diff --git a/mysql-test/suite/innodb/r/dropdb_cs.result b/mysql-test/suite/innodb/r/dropdb_cs.result new file mode 100644 index 00000000000..59f02c74a04 --- /dev/null +++ b/mysql-test/suite/innodb/r/dropdb_cs.result @@ -0,0 +1,16 @@ +# +# MDEV-28802 DROP DATABASE in InnoDB still is case-insensitive +# +SET @save_fpt=@@GLOBAL.innodb_file_per_table; +SET GLOBAL innodb_file_per_table=0; +CREATE DATABASE Db; +CREATE TABLE Db.t1 (c1 INT KEY) ENGINE=InnoDB; +CREATE DATABASE DB; +DROP DATABASE DB; +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'D%'; +NAME +Db/t1 +DROP DATABASE Db; +SELECT * FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'D%'; +TABLE_ID NAME FLAG N_COLS SPACE ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE +SET GLOBAL innodb_file_per_table=@save_fpt; diff --git a/mysql-test/suite/innodb/t/dropdb_cs.test b/mysql-test/suite/innodb/t/dropdb_cs.test new file mode 100644 index 00000000000..0053ca732be --- /dev/null +++ b/mysql-test/suite/innodb/t/dropdb_cs.test @@ -0,0 +1,17 @@ +--source include/have_innodb.inc +--source include/have_case_sensitive_file_system.inc + +--echo # +--echo # MDEV-28802 DROP DATABASE in InnoDB still is case-insensitive +--echo # + +SET @save_fpt=@@GLOBAL.innodb_file_per_table; +SET GLOBAL innodb_file_per_table=0; +CREATE DATABASE Db; +CREATE TABLE Db.t1 (c1 INT KEY) ENGINE=InnoDB; +CREATE DATABASE DB; +DROP DATABASE DB; +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'D%'; +DROP DATABASE Db; +SELECT * FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'D%'; +SET GLOBAL innodb_file_per_table=@save_fpt; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 300a1cd83ae..b676f83b8a3 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1476,7 +1476,8 @@ static void innodb_drop_database(handlerton*, char *path) "WHILE 1 = 1 LOOP\n" " FETCH tab INTO tid,name;\n" " IF (SQL % NOTFOUND) THEN EXIT; END IF;\n" - " IF SUBSTR(name, 0, LENGTH(:db)) <> :db THEN EXIT; END IF;\n" + " IF TO_BINARY(SUBSTR(name, 0, LENGTH(:db))) <> TO_BINARY(:db)" + " THEN EXIT; END IF;\n" " DELETE FROM SYS_COLUMNS WHERE TABLE_ID=tid;\n" " DELETE FROM SYS_TABLES WHERE ID=tid;\n" " OPEN idx;\n" From 06e9ce798c582ffb5a2f3cfbb0f8e33fe221feae Mon Sep 17 00:00:00 2001 From: Nayuta Yanagisawa Date: Fri, 29 Oct 2021 19:04:53 +0900 Subject: [PATCH 07/11] MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION During rebuild of partition, the partitioning engine calls alter_close_table(), which does not unlock and close some table instances of the target table. Then, the engine fails to rename partitions because there are table instances that are still locked. Closing all the table instance of the target table fixes the bug. --- .../suite/parts/inc/part_alter_values.inc | 8 +++++ .../parts/r/partition_alter_innodb.result | 7 ++++ .../parts/r/partition_alter_maria.result | 7 ++++ .../parts/r/partition_alter_myisam.result | 7 ++++ sql/sql_partition.cc | 35 ++++++++++++++----- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/mysql-test/suite/parts/inc/part_alter_values.inc b/mysql-test/suite/parts/inc/part_alter_values.inc index ca18faa5758..d3b63a4610f 100644 --- a/mysql-test/suite/parts/inc/part_alter_values.inc +++ b/mysql-test/suite/parts/inc/part_alter_values.inc @@ -78,3 +78,11 @@ if (`SELECT IF('$engine' != 'InnoDB', 1, 0)`) --remove_files_wildcard $MYSQLTEST_VARDIR/tmp/mdev_27065 * --rmdir $MYSQLTEST_VARDIR/tmp/mdev_27065 + +--echo # +--echo # MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION +--echo # +--eval CREATE TABLE t1 (c INT) ENGINE=$engine PARTITION BY KEY(c) PARTITIONS 4; +LOCK TABLES t1 WRITE, t1 AS a READ; +ALTER TABLE t1 REBUILD PARTITION p0; +DROP TABLE t1; diff --git a/mysql-test/suite/parts/r/partition_alter_innodb.result b/mysql-test/suite/parts/r/partition_alter_innodb.result index ae3caaa4981..6afa133f989 100644 --- a/mysql-test/suite/parts/r/partition_alter_innodb.result +++ b/mysql-test/suite/parts/r/partition_alter_innodb.result @@ -61,3 +61,10 @@ PARTITION p1 VALUES LESS THAN MAXVALUE Warnings: Warning 1618 table option of old schema is ignored DROP TABLE t1; +# +# MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION +# +CREATE TABLE t1 (c INT) ENGINE=InnoDB PARTITION BY KEY(c) PARTITIONS 4;; +LOCK TABLES t1 WRITE, t1 AS a READ; +ALTER TABLE t1 REBUILD PARTITION p0; +DROP TABLE t1; diff --git a/mysql-test/suite/parts/r/partition_alter_maria.result b/mysql-test/suite/parts/r/partition_alter_maria.result index 358ffbdfbe7..eca8378430f 100644 --- a/mysql-test/suite/parts/r/partition_alter_maria.result +++ b/mysql-test/suite/parts/r/partition_alter_maria.result @@ -95,3 +95,10 @@ PARTITION p1 VALUES LESS THAN MAXVALUE Warnings: Warning 1618 table option of old schema is ignored DROP TABLE t2; +# +# MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION +# +CREATE TABLE t1 (c INT) ENGINE=Aria PARTITION BY KEY(c) PARTITIONS 4;; +LOCK TABLES t1 WRITE, t1 AS a READ; +ALTER TABLE t1 REBUILD PARTITION p0; +DROP TABLE t1; diff --git a/mysql-test/suite/parts/r/partition_alter_myisam.result b/mysql-test/suite/parts/r/partition_alter_myisam.result index 9d76881fdfa..ba1a0fe05c4 100644 --- a/mysql-test/suite/parts/r/partition_alter_myisam.result +++ b/mysql-test/suite/parts/r/partition_alter_myisam.result @@ -68,6 +68,13 @@ PARTITION p1 VALUES LESS THAN MAXVALUE Warnings: Warning 1618 table option of old schema is ignored DROP TABLE t2; +# +# MDEV-26127 Assertion `err != DB_DUPLICATE_KEY' failed or InnoDB: Failing assertion: id != 0 on ALTER ... REBUILD PARTITION +# +CREATE TABLE t1 (c INT) ENGINE=MyISAM PARTITION BY KEY(c) PARTITIONS 4;; +LOCK TABLES t1 WRITE, t1 AS a READ; +ALTER TABLE t1 REBUILD PARTITION p0; +DROP TABLE t1; create table t1 ( c1 int, c2 int, c3 varchar(100)) delay_key_write=1 partition by key(c1) ( partition p01 data directory = 'MYSQL_TMP_DIR' diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index 1cf2d009d37..78f56992f46 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2005, 2017, Oracle and/or its affiliates. - Copyright (c) 2009, 2020, MariaDB + Copyright (c) 2009, 2022, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -6817,16 +6817,33 @@ static bool alter_partition_lock_handling(ALTER_PARTITION_PARAM_TYPE *lpt) static int alter_close_table(ALTER_PARTITION_PARAM_TYPE *lpt) { - int error= 0; + THD *thd= lpt->thd; + TABLE_SHARE *share= lpt->table->s; DBUG_ENTER("alter_close_table"); - if (lpt->table->db_stat) - { - error= mysql_lock_remove(lpt->thd, lpt->thd->lock, lpt->table); - error= lpt->table->file->ha_close(); - lpt->table->db_stat= 0; // Mark file closed - } - DBUG_RETURN(error); + TABLE *table= thd->open_tables; + do { + table= find_locked_table(table, share->db.str, share->table_name.str); + if (!table) + { + DBUG_RETURN(0); + } + + if (table->db_stat) + { + if (int error= mysql_lock_remove(thd, thd->lock, table)) + { + DBUG_RETURN(error); + } + if (int error= table->file->ha_close()) + { + DBUG_RETURN(error); + } + table->db_stat= 0; // Mark file closed + } + } while ((table= table->next)); + + DBUG_RETURN(0); } From 4849d94fe6bd918fd3dc6dc21424681a0ab5fa02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 14 Jun 2022 09:14:24 +0300 Subject: [PATCH 08/11] MDEV-28828 SIGSEGV in buf_flush_LRU_list_batch In commit 73fee39ea62037780c59161507e89dd76c10b7a3 (MDEV-27985) a regression was introduced that would cause bpage=nullptr to be referenced. buf_flush_LRU_list_batch(): Always terminate the loop upon encountering a null pointer. --- storage/innobase/buf/buf0flu.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index def598934c9..3d2669e2cc5 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1294,7 +1294,6 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n) n->flushed + n->evicted < max) || recv_recovery_is_on()); ++scanned) { - retry: buf_page_t *prev= UT_LIST_GET_PREV(LRU, bpage); const lsn_t oldest_modification= bpage->oldest_modification(); buf_pool.lru_hp.set(prev); @@ -1326,7 +1325,6 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n) mysql_mutex_lock(&buf_pool.mutex); if (p.second) buf_pool.stat.n_pages_written+= p.second; - bpage= buf_pool.lru_hp.get(); goto retry; } else @@ -1357,6 +1355,7 @@ reacquire_mutex: else /* Can't evict or dispatch this block. Go to previous. */ ut_ad(buf_pool.lru_hp.is_hp(prev)); + retry: bpage= buf_pool.lru_hp.get(); } From 6c669b9586f72d6d760cc3956c1a0cb09ace2367 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 14 Jun 2022 16:52:47 +0530 Subject: [PATCH 09/11] MDEV-25581 Allow user thread to do InnoDB fts cache sync - innodb_fts.sync_block doesn't make sense after MDEV-25581's patch because fts cache syncing is done as a part of insert operation and it leads to completion of select over insert sometimes. This test case is not relevant any more --- .../suite/innodb_fts/r/sync_block.result | 54 ------------- mysql-test/suite/innodb_fts/t/sync_block.test | 78 ------------------- 2 files changed, 132 deletions(-) delete mode 100644 mysql-test/suite/innodb_fts/r/sync_block.result delete mode 100644 mysql-test/suite/innodb_fts/t/sync_block.test diff --git a/mysql-test/suite/innodb_fts/r/sync_block.result b/mysql-test/suite/innodb_fts/r/sync_block.result deleted file mode 100644 index c105a3d3b12..00000000000 --- a/mysql-test/suite/innodb_fts/r/sync_block.result +++ /dev/null @@ -1,54 +0,0 @@ -SET @old_log_output = @@global.log_output; -SET @old_slow_query_log = @@global.slow_query_log; -SET @old_general_log = @@global.general_log; -SET @old_long_query_time = @@global.long_query_time; -SET @old_debug = @@global.debug_dbug; -SET GLOBAL log_output = 'TABLE'; -SET GLOBAL general_log = 1; -SET GLOBAL slow_query_log = 1; -SET GLOBAL long_query_time = 1; -connect con1,localhost,root,,; -connect con2,localhost,root,,; -connection default; -# Case 1: Sync blocks DML(insert) on the same table. -CREATE TABLE t1 ( -FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, -title VARCHAR(200), -FULLTEXT(title) -) ENGINE = InnoDB; -connection con1; -SET GLOBAL debug_dbug='+d,fts_instrument_sync_debug,fts_instrument_sync_sleep'; -SET DEBUG_SYNC= 'fts_sync_begin SIGNAL begin WAIT_FOR continue'; -INSERT INTO t1(title) VALUES('mysql database'); -connection con2; -SET DEBUG_SYNC= 'now WAIT_FOR begin'; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); -connection default; -SET DEBUG_SYNC= 'now SIGNAL continue'; -connection con1; -/* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); -connection con2; -/* conneciton con2 */ SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); -FTS_DOC_ID title -1 mysql database -connection default; -# make con1 & con2 show up in mysql.slow_log -SELECT SLEEP(2); -SLEEP(2) -0 -# slow log results should only contain INSERT INTO t1. -SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; -sql_text -INSERT INTO t1(title) VALUES('mysql database') -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') -SET GLOBAL debug_dbug = @old_debug; -TRUNCATE TABLE mysql.slow_log; -DROP TABLE t1; -SET DEBUG_SYNC=RESET; -disconnect con1; -disconnect con2; -# Restore slow log settings. -SET GLOBAL log_output = @old_log_output; -SET GLOBAL general_log = @old_general_log; -SET GLOBAL slow_query_log = @old_slow_query_log; -SET GLOBAL long_query_time = @old_long_query_time; diff --git a/mysql-test/suite/innodb_fts/t/sync_block.test b/mysql-test/suite/innodb_fts/t/sync_block.test deleted file mode 100644 index 705567296f7..00000000000 --- a/mysql-test/suite/innodb_fts/t/sync_block.test +++ /dev/null @@ -1,78 +0,0 @@ -# -# BUG#22516559 MYSQL INSTANCE STALLS WHEN SYNCING FTS INDEX -# - ---source include/have_innodb.inc ---source include/have_debug.inc ---source include/have_debug_sync.inc ---source include/have_log_bin.inc ---source include/count_sessions.inc - -SET @old_log_output = @@global.log_output; -SET @old_slow_query_log = @@global.slow_query_log; -SET @old_general_log = @@global.general_log; -SET @old_long_query_time = @@global.long_query_time; -SET @old_debug = @@global.debug_dbug; - -SET GLOBAL log_output = 'TABLE'; -SET GLOBAL general_log = 1; -SET GLOBAL slow_query_log = 1; -SET GLOBAL long_query_time = 1; - -connect (con1,localhost,root,,); -connect (con2,localhost,root,,); -connection default; - ---echo # Case 1: Sync blocks DML(insert) on the same table. -CREATE TABLE t1 ( - FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, - title VARCHAR(200), - FULLTEXT(title) -) ENGINE = InnoDB; - -connection con1; - -SET GLOBAL debug_dbug='+d,fts_instrument_sync_debug,fts_instrument_sync_sleep'; - -SET DEBUG_SYNC= 'fts_sync_begin SIGNAL begin WAIT_FOR continue'; - -send INSERT INTO t1(title) VALUES('mysql database'); - -connection con2; - -SET DEBUG_SYNC= 'now WAIT_FOR begin'; - -send SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); - -connection default; -SET DEBUG_SYNC= 'now SIGNAL continue'; - -connection con1; ---echo /* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); ---reap - -connection con2; ---echo /* conneciton con2 */ SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); ---reap - -connection default; --- echo # make con1 & con2 show up in mysql.slow_log -SELECT SLEEP(2); --- echo # slow log results should only contain INSERT INTO t1. -SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; - -SET GLOBAL debug_dbug = @old_debug; -TRUNCATE TABLE mysql.slow_log; - -DROP TABLE t1; -SET DEBUG_SYNC=RESET; -disconnect con1; -disconnect con2; - ---source include/wait_until_count_sessions.inc - --- echo # Restore slow log settings. -SET GLOBAL log_output = @old_log_output; -SET GLOBAL general_log = @old_general_log; -SET GLOBAL slow_query_log = @old_slow_query_log; -SET GLOBAL long_query_time = @old_long_query_time; From 6c82ab4f726b83f09646c22afddd4c102e60607b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 14 Jun 2022 15:33:11 +0300 Subject: [PATCH 10/11] MDEV-28840 innodb_undo_log_truncate is not crash-safe trx_purge_free_segment(): Do mark that the block will be modified. It seems possible that this regression was introduced by the changes to the page-freeing logic in commit 4179f93d28035ea2798cb1c16feeaaef87ab4775 (MDEV-18976). Tested by: Matthias Leich --- storage/innobase/include/mtr0mtr.h | 2 +- storage/innobase/include/mtr0mtr.inl | 2 +- storage/innobase/trx/trx0purge.cc | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 02f469e3a53..8a2f862f47f 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -290,9 +290,9 @@ struct mtr_t { @param[in] type object type: MTR_MEMO_PAGE_X_FIX, ... */ void release_page(const void *ptr, mtr_memo_type_t type); -private: /** Note that the mini-transaction will modify data. */ void flag_modified() { m_modifications = true; } +private: /** Mark the given latched page as modified. @param block page that will be modified */ void modify(const buf_block_t& block); diff --git a/storage/innobase/include/mtr0mtr.inl b/storage/innobase/include/mtr0mtr.inl index 21ff912e9c7..71b476a2f5d 100644 --- a/storage/innobase/include/mtr0mtr.inl +++ b/storage/innobase/include/mtr0mtr.inl @@ -46,7 +46,7 @@ mtr_t::memo_push(void* object, mtr_memo_type_t type) ut_ad(object != NULL); ut_ad(type >= MTR_MEMO_PAGE_S_FIX); ut_ad(type <= MTR_MEMO_SPACE_S_LOCK); - ut_ad(ut_is_2pow(type)); + ut_ad(type == MTR_MEMO_PAGE_X_MODIFY || ut_is_2pow(type)); /* If this mtr has x-fixed a clean page then we set the made_dirty flag. This tells us if we need to diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 84d7d785490..e37efa733e2 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -402,8 +402,9 @@ static dberr_t trx_purge_free_segment(trx_rseg_t *rseg, fil_addr_t hdr_addr) block->fix(); mtr.commit(); mtr.start(); + mtr.flag_modified(); mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_FIX); - mtr.memo_push(block, MTR_MEMO_PAGE_X_FIX); + mtr.memo_push(block, MTR_MEMO_PAGE_X_MODIFY); rseg->latch.wr_lock(SRW_LOCK_CALL); rseg_hdr->page.lock.x_lock(); block->page.lock.x_lock(); From 4c0cd953abffea841271f3a5cce1712d5e6c5633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 14 Jun 2022 17:46:47 +0300 Subject: [PATCH 11/11] MDEV-28766: SET GLOBAL innodb_log_file_buffering In commit c4c88307091cb16886562e9e7b77f5fd077d34b5 (MDEV-28111) we disabled the file system cache on the InnoDB write-ahead log file (ib_logfile0) by default on Linux. It turns out that especially with innodb_flush_trx_log_at_commit=2, writing to the log via the file system cache typically improves throughput, especially on slow storage or at a small number of concurrent transactions. For other values of innodb_flush_log_at_trx_commit, direct writes were observed to be mostly but not always faster. Whether it pays off to disable the file system cache on the log may depend on the type of storage, the workload, and the operating system kernel version. On Linux and Microsoft Windows, we will introduce the settable Boolean global variable innodb_log_file_buffering that indicates whether the file system cache on the redo log file is enabled. The default value is innodb_log_file_buffering=OFF. If the server is started up with innodb_flush_log_at_trx_commit=2, the value will be changed to innodb_log_file_buffering=ON. When a persistent memory interface is being used for the log, the value cannot be changed from innodb_log_file_buffering=OFF. On Linux, when the physical block size cannot be determined to be a power of 2 between 64 and 4096 bytes, the file system cache cannot be disabled, and innodb_log_file_buffering=ON cannot be changed. Server log messages will indicate whether the file system cache is enabled for the redo log: [Note] InnoDB: Buffered log writes (block size=512 bytes) [Note] InnoDB: File system buffers for log disabled (block size=512 bytes) After this change, the startup parameter innodb_flush_method will no longer control whether O_DIRECT will be set on the redo log on Linux. On other operating systems that support O_DIRECT, no interface has been implemented for controlling the file system cache for the redo log. The innodb_flush_method values O_DIRECT, O_DIRECT_NO_FSYNC, O_DSYNC will enable O_DIRECT for data files, not the log. Tested by: Matthias Leich, Axel Schwenke --- .../suite/sys_vars/r/sysvars_innodb.result | 3 +- .../suite/sys_vars/t/sysvars_innodb.test | 1 + storage/innobase/handler/ha_innodb.cc | 30 +++++++- storage/innobase/include/log0log.h | 17 +++++ storage/innobase/log/log0log.cc | 75 ++++++++++++++++--- storage/innobase/os/os0file.cc | 51 +++++++------ 6 files changed, 139 insertions(+), 38 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 11b90f41d5f..c3dd970f6e3 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -5,6 +5,7 @@ variable_name not in ( 'innodb_numa_interleave', # only available WITH_NUMA 'innodb_evict_tables_on_commit_debug', # one may want to override this 'innodb_use_native_aio', # default value depends on OS +'innodb_log_file_buffering', # only available on Linux and Windows 'innodb_buffer_pool_load_pages_abort') # debug build only, and is only for testing order by variable_name; VARIABLE_NAME INNODB_ADAPTIVE_FLUSHING @@ -1020,7 +1021,7 @@ SESSION_VALUE NULL DEFAULT_VALUE VARIABLE_SCOPE GLOBAL VARIABLE_TYPE VARCHAR -VARIABLE_COMMENT Path to InnoDB log files. +VARIABLE_COMMENT Path to ib_logfile0 NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL diff --git a/mysql-test/suite/sys_vars/t/sysvars_innodb.test b/mysql-test/suite/sys_vars/t/sysvars_innodb.test index 15fd99e9984..6d46c22683f 100644 --- a/mysql-test/suite/sys_vars/t/sysvars_innodb.test +++ b/mysql-test/suite/sys_vars/t/sysvars_innodb.test @@ -12,5 +12,6 @@ select VARIABLE_NAME, SESSION_VALUE, DEFAULT_VALUE, VARIABLE_SCOPE, VARIABLE_TYP 'innodb_numa_interleave', # only available WITH_NUMA 'innodb_evict_tables_on_commit_debug', # one may want to override this 'innodb_use_native_aio', # default value depends on OS + 'innodb_log_file_buffering', # only available on Linux and Windows 'innodb_buffer_pool_load_pages_abort') # debug build only, and is only for testing order by variable_name; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3ad8ae1c070..d286a034dc1 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4066,6 +4066,14 @@ static int innodb_init_params() } #endif +#if defined __linux__ || defined _WIN32 + if (srv_flush_log_at_trx_commit == 2) { + /* Do not disable the file system cache if + innodb_flush_log_at_trx_commit=2. */ + log_sys.log_buffered = true; + } +#endif + if (srv_read_only_mode) { ib::info() << "Started in read only mode"; srv_use_doublewrite_buf = FALSE; @@ -18442,6 +18450,16 @@ buffer_pool_load_abort( } } +#if defined __linux__ || defined _WIN32 +static void innodb_log_file_buffering_update(THD *thd, st_mysql_sys_var*, + void *, const void *save) +{ + mysql_mutex_unlock(&LOCK_global_system_variables); + log_sys.set_buffered(*static_cast(save)); + mysql_mutex_lock(&LOCK_global_system_variables); +} +#endif + /** Update innodb_status_output or innodb_status_output_locks, which control InnoDB "status monitor" output to the error log. @param[out] var current value @@ -18858,7 +18876,7 @@ static MYSQL_SYSVAR_ENUM(flush_method, srv_file_flush_method, static MYSQL_SYSVAR_STR(log_group_home_dir, srv_log_group_home_dir, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Path to InnoDB log files.", NULL, NULL, NULL); + "Path to ib_logfile0", NULL, NULL, NULL); static MYSQL_SYSVAR_DOUBLE(max_dirty_pages_pct, srv_max_buf_pool_modified_pct, PLUGIN_VAR_RQCMDARG, @@ -19250,6 +19268,13 @@ static MYSQL_SYSVAR_SIZE_T(log_buffer_size, log_sys.buf_size, "Redo log buffer size in bytes.", NULL, NULL, 16U << 20, 2U << 20, SIZE_T_MAX, 4096); +#if defined __linux__ || defined _WIN32 +static MYSQL_SYSVAR_BOOL(log_file_buffering, log_sys.log_buffered, + PLUGIN_VAR_OPCMDARG, + "Whether the file system cache for ib_logfile0 is enabled", + nullptr, innodb_log_file_buffering_update, FALSE); +#endif + static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Redo log size in bytes.", @@ -19692,6 +19717,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(deadlock_report), MYSQL_SYSVAR(page_size), MYSQL_SYSVAR(log_buffer_size), +#if defined __linux__ || defined _WIN32 + MYSQL_SYSVAR(log_file_buffering), +#endif MYSQL_SYSVAR(log_file_size), MYSQL_SYSVAR(log_group_home_dir), MYSQL_SYSVAR(max_dirty_pages_pct), diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 39c2fb8b01e..d1c6e40d946 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -249,6 +249,16 @@ public: uint32_t format; /** Log file */ log_file_t log; +#if defined __linux__ || defined _WIN32 + /** whether file system caching is enabled for the log */ + my_bool log_buffered; +# ifdef _WIN32 + static constexpr bool log_maybe_unbuffered= true; +# else + /** whether file system caching may be disabled */ + bool log_maybe_unbuffered; +# endif +#endif /** Fields involved in checkpoints @{ */ lsn_t log_capacity; /*!< capacity of the log; if @@ -289,10 +299,17 @@ public: bool is_opened() const noexcept { return log.is_opened(); } + static constexpr bool resize_in_progress() { return false; } + /** Rename a log file after resizing. @return whether an error occurred */ static bool rename_resized() noexcept; +#if defined __linux__ || defined _WIN32 + /** Try to enable or disable file system caching (update log_buffered) */ + void set_buffered(bool buffered); +#endif + void attach(log_file_t file, os_offset_t size); void close_file(); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 8a0ef712ab1..efdd527a28f 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -209,6 +209,8 @@ void log_t::attach(log_file_t file, os_offset_t size) #if defined __linux__ || defined _WIN32 set_block_size(CPU_LEVEL1_DCACHE_LINESIZE); #endif + log_maybe_unbuffered= true; + log_buffered= false; return; } } @@ -220,18 +222,11 @@ void log_t::attach(log_file_t file, os_offset_t size) #endif #if defined __linux__ || defined _WIN32 - if (!block_size) - set_block_size(512); -# ifdef __linux__ - else if (srv_file_flush_method != SRV_O_DSYNC && - srv_file_flush_method != SRV_O_DIRECT && - srv_file_flush_method != SRV_O_DIRECT_NO_FSYNC) - sql_print_information("InnoDB: Buffered log writes (block size=%u bytes)", - block_size); -#endif - else - sql_print_information("InnoDB: File system buffers for log" - " disabled (block size=%u bytes)", block_size); + sql_print_information("InnoDB: %s (block size=%u bytes)", + log_buffered + ? "Buffered log writes" + : "File system buffers for log disabled", + block_size); #endif #ifdef HAVE_PMEM @@ -327,6 +322,62 @@ void log_t::close_file() ib::fatal() << "closing ib_logfile0 failed: " << err; } +#if defined __linux__ || defined _WIN32 +/** Acquire all latches that protect the log. */ +static void log_resize_acquire() +{ + if (!log_sys.is_pmem()) + { + while (flush_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + group_commit_lock::ACQUIRED); + while (write_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + group_commit_lock::ACQUIRED); + } + + log_sys.latch.wr_lock(SRW_LOCK_CALL); +} + +/** Release the latches that protect the log. */ +void log_resize_release() +{ + log_sys.latch.wr_unlock(); + + if (!log_sys.is_pmem()) + { + lsn_t lsn1= write_lock.release(write_lock.value()); + lsn_t lsn2= flush_lock.release(flush_lock.value()); + if (lsn1 || lsn2) + log_write_up_to(std::max(lsn1, lsn2), true, nullptr); + } +} + +/** Try to enable or disable file system caching (update log_buffered) */ +void log_t::set_buffered(bool buffered) +{ + if (!log_maybe_unbuffered || is_pmem() || high_level_read_only) + return; + log_resize_acquire(); + if (!resize_in_progress() && is_opened() && bool(log_buffered) != buffered) + { + os_file_close_func(log.m_file); + log.m_file= OS_FILE_CLOSED; + std::string path{get_log_file_path()}; + log_buffered= buffered; + bool success; + log.m_file= os_file_create_func(path.c_str(), + OS_FILE_OPEN, OS_FILE_NORMAL, OS_LOG_FILE, + false, &success); + ut_a(log.m_file != OS_FILE_CLOSED); + sql_print_information("InnoDB: %s (block size=%u bytes)", + log_buffered + ? "Buffered log writes" + : "File system buffers for log disabled", + block_size); + } + log_resize_release(); +} +#endif + /** Write an aligned buffer to ib_logfile0. @param buf buffer to be written @param len length of data to be written diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index e212e9a3ef3..01af15befd3 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -1055,6 +1055,7 @@ os_file_create_simple_func( we open the same file in the same mode, see man page of open(2). */ if (!srv_read_only_mode && *success) { switch (srv_file_flush_method) { + case SRV_O_DSYNC: case SRV_O_DIRECT: case SRV_O_DIRECT_NO_FSYNC: os_file_set_nocache(file, name, mode_str); @@ -1240,13 +1241,13 @@ os_file_create_func( #if (defined(UNIV_SOLARIS) && defined(DIRECTIO_ON)) || defined O_DIRECT if (type == OS_DATA_FILE) { -# ifdef __linux__ -use_o_direct: -# endif switch (srv_file_flush_method) { case SRV_O_DSYNC: case SRV_O_DIRECT: case SRV_O_DIRECT_NO_FSYNC: +# ifdef __linux__ +use_o_direct: +# endif os_file_set_nocache(file, name, mode_str); break; default: @@ -1263,9 +1264,6 @@ use_o_direct: goto skip_o_direct; } MSAN_STAT_WORKAROUND(&st); - if (st.st_size & 4095) { - goto skip_o_direct; - } if (snprintf(b, sizeof b, "/sys/dev/block/%u:%u/queue/physical_block_size", major(st.st_dev), minor(st.st_dev)) @@ -1298,11 +1296,16 @@ use_o_direct: if (s > 4096 || s < 64 || !ut_is_2pow(s)) { goto skip_o_direct; } + log_sys.log_maybe_unbuffered= true; log_sys.set_block_size(uint32_t(s)); - goto use_o_direct; + if (!log_sys.log_buffered && !(st.st_size & (s - 1))) { + goto use_o_direct; + } } else { skip_o_direct: - log_sys.set_block_size(0); + log_sys.log_maybe_unbuffered= false; + log_sys.log_buffered= true; + log_sys.set_block_size(512); } } # endif @@ -2057,7 +2060,7 @@ os_file_create_directory( } /** Get disk sector size for a file. */ -size_t get_sector_size(HANDLE file) +static size_t get_sector_size(HANDLE file) { FILE_STORAGE_INFO fsi; ULONG s= 4096; @@ -2065,9 +2068,7 @@ size_t get_sector_size(HANDLE file) { s= fsi.PhysicalBytesPerSectorForPerformance; if (s > 4096 || s < 64 || !ut_is_2pow(s)) - { return 4096; - } } return s; } @@ -2165,8 +2166,9 @@ os_file_create_func( ? FILE_FLAG_OVERLAPPED : 0; if (type == OS_LOG_FILE) { - if(srv_flush_log_at_trx_commit != 2 && !log_sys.is_opened()) + if (!log_sys.is_opened() && !log_sys.log_buffered) { attributes|= FILE_FLAG_NO_BUFFERING; + } if (srv_file_flush_method == SRV_O_DSYNC) attributes|= FILE_FLAG_WRITE_THROUGH; } @@ -2197,21 +2199,22 @@ os_file_create_func( name, access, share_mode, my_win_file_secattr(), create_flag, attributes, NULL); - if (file != INVALID_HANDLE_VALUE && type == OS_LOG_FILE - && (attributes & FILE_FLAG_NO_BUFFERING)) { - uint32 s= (uint32_t) get_sector_size(file); - log_sys.set_block_size(uint32_t(s)); - /* FIXME! remove it when backup is fixed, so that it - does not produce redo with irregular sizes.*/ - if (os_file_get_size(file) % s) { - attributes &= ~FILE_FLAG_NO_BUFFERING; - create_flag = OPEN_ALWAYS; - CloseHandle(file); - continue; + *success = file != INVALID_HANDLE_VALUE; + + if (*success && type == OS_LOG_FILE) { + uint32_t s = uint32_t(get_sector_size(file)); + log_sys.set_block_size(s); + if (attributes & FILE_FLAG_NO_BUFFERING) { + if (os_file_get_size(file) % s) { + attributes &= ~FILE_FLAG_NO_BUFFERING; + create_flag = OPEN_ALWAYS; + CloseHandle(file); + continue; + } + log_sys.log_buffered = false; } } - *success = (file != INVALID_HANDLE_VALUE); if (*success) { break; }