From ac963142ee351c9359442cbdc1b20ea0283e6a02 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Wed, 24 Nov 2021 23:19:22 -0800 Subject: [PATCH 01/12] MDEV-26553 NOT IN subquery construct crashing 10.1 and up This bug was introduced by commit be00e279c6061134a33a8099fd69d4304735d02e The commit was applied for the task MDEV-6480 that allowed to remove top level disjuncts from WHERE conditions if the range optimizer evaluated them as always equal to FALSE/NULL. If such disjuncts are removed the WHERE condition may become an AND formula and if this formula contains multiple equalities the field JOIN::item_equal must be updated to refer to these equalities. The above mentioned commit forgot to do this and it could cause crashes for some queries. Approved by Oleksandr Byelkin --- mysql-test/r/range.result | 24 ++++++++++++++++++++++++ mysql-test/r/range_mrr_icp.result | 24 ++++++++++++++++++++++++ mysql-test/t/range.test | 25 ++++++++++++++++++++++++- sql/sql_select.cc | 6 ++++++ 4 files changed, 78 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result index 6a3850c0ed9..f1f949acd3b 100644 --- a/mysql-test/r/range.result +++ b/mysql-test/r/range.result @@ -3201,5 +3201,29 @@ pk i v a b 2 2 4 2 4 drop table t1, t2; # +# MDEV-26553: Always FALSE/NULL disjunct on top level of WHERE is removed +# +create table t1 (a int, b int, index idx(a,b)); +insert into t1 values (1,1), (1,2), (2,1), (2,2), (3,3); +create table t2 (c int); +insert into t2 values (5), (2), (3), (4); +select 1 from t1 s1 +where 1 not in (select 1 from t1 +where ((a = 1 or a = 2) and b = 1) or (b > 5 and b < 1)); +1 +select 1 from t1 s1 +where 1 not in (select 1 from t1 +where ((a = 1 or a = 2) and b = 1) or b = NULL); +1 +select c from t2 +where 2 not in (select 1 from t1 +where ((a=1 or a=2) and b = 1) or (b > 5 and b < 1)); +c +5 +2 +3 +4 +drop table t1,t2; +# # End of 10.2 tests # diff --git a/mysql-test/r/range_mrr_icp.result b/mysql-test/r/range_mrr_icp.result index 24f42f34ce5..645576fbe42 100644 --- a/mysql-test/r/range_mrr_icp.result +++ b/mysql-test/r/range_mrr_icp.result @@ -3213,6 +3213,30 @@ pk i v a b 2 2 4 2 4 drop table t1, t2; # +# MDEV-26553: Always FALSE/NULL disjunct on top level of WHERE is removed +# +create table t1 (a int, b int, index idx(a,b)); +insert into t1 values (1,1), (1,2), (2,1), (2,2), (3,3); +create table t2 (c int); +insert into t2 values (5), (2), (3), (4); +select 1 from t1 s1 +where 1 not in (select 1 from t1 +where ((a = 1 or a = 2) and b = 1) or (b > 5 and b < 1)); +1 +select 1 from t1 s1 +where 1 not in (select 1 from t1 +where ((a = 1 or a = 2) and b = 1) or b = NULL); +1 +select c from t2 +where 2 not in (select 1 from t1 +where ((a=1 or a=2) and b = 1) or (b > 5 and b < 1)); +c +5 +2 +3 +4 +drop table t1,t2; +# # End of 10.2 tests # set optimizer_switch=@mrr_icp_extra_tmp; diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test index 890377ed977..7369b9d8a16 100644 --- a/mysql-test/t/range.test +++ b/mysql-test/t/range.test @@ -2232,6 +2232,29 @@ select * from t1 inner join t2 on ( t2.b = t1.v or t2.a = t1.pk); drop table t1, t2; --echo # ---echo # End of 10.2 tests +--echo # MDEV-26553: Always FALSE/NULL disjunct on top level of WHERE is removed --echo # +create table t1 (a int, b int, index idx(a,b)); +insert into t1 values (1,1), (1,2), (2,1), (2,2), (3,3); + +create table t2 (c int); +insert into t2 values (5), (2), (3), (4); + +select 1 from t1 s1 + where 1 not in (select 1 from t1 + where ((a = 1 or a = 2) and b = 1) or (b > 5 and b < 1)); + +select 1 from t1 s1 + where 1 not in (select 1 from t1 + where ((a = 1 or a = 2) and b = 1) or b = NULL); + +select c from t2 + where 2 not in (select 1 from t1 + where ((a=1 or a=2) and b = 1) or (b > 5 and b < 1)); + +drop table t1,t2; + +--echo # +--echo # End of 10.2 tests +--echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 54a2facfe9f..47422116e38 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -4504,7 +4504,13 @@ make_join_statistics(JOIN *join, List &tables_list, if (*s->on_expr_ref) *s->on_expr_ref= select->cond; else + { join->conds= select->cond; + if (join->conds && join->conds->type() == Item::COND_ITEM && + ((Item_cond*) (join->conds))->functype() == + Item_func::COND_AND_FUNC) + join->cond_equal= &((Item_cond_and*) (join->conds))->m_cond_equal; + } s->quick=select->quick; s->needed_reg=select->needed_reg; From 4e0dcf1083ad905b36c69896bdf0731afc9d7fec Mon Sep 17 00:00:00 2001 From: Martin Beck Date: Sat, 20 Nov 2021 14:22:25 +1100 Subject: [PATCH 02/12] MDEV-27088: Server crash on ARM (WMM architecture) due to missing barriers in lf-hash MariaDB server crashes on ARM (weak memory model architecture) while concurrently executing l_find to load node->key and add_to_purgatory to store node->key = NULL. l_find then uses key (which is NULL), to pass it to a comparison function. The specific problem is the out-of-order execution that happens on a weak memory model architecture. Two essential reorderings are possible, which need to be prevented. a) As l_find has no barriers in place between the optimistic read of the key field lf_hash.cc#L117 and the verification of link lf_hash.cc#L124, the processor can reorder the load to happen after the while-loop. In that case, a concurrent thread executing add_to_purgatory on the same node can be scheduled to store NULL at the key field lf_alloc-pin.c#L253 before key is loaded in l_find. b) A node is marked as deleted by a CAS in l_delete lf_hash.cc#L247 and taken off the list with an upfollowing CAS lf_hash.cc#L252. Only if both CAS succeed, the key field is written to by add_to_purgatory. However, due to a missing barrier, the relaxed store of key lf_alloc-pin.c#L253 can be moved ahead of the two CAS operations, which makes the value of the local purgatory list stored by add_to_purgatory visible to all threads operating on the list. As the node is not marked as deleted yet, the same error occurs in l_find. This change three accesses to be atomic. * optimistic read of key in l_find lf_hash.cc#L117 * read of link for verification lf_hash.cc#L124 * write of key in add_to_purgatory lf_alloc-pin.c#L253 Reviewers: Sergei Vojtovich, Sergei Golubchik Fixes: MDEV-23510 / d30c1331a18d875e553f3fcf544997e4f33fb943 --- mysys/lf_alloc-pin.c | 5 +++-- mysys/lf_hash.c | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mysys/lf_alloc-pin.c b/mysys/lf_alloc-pin.c index 0dc524be336..880ef44e9ca 100644 --- a/mysys/lf_alloc-pin.c +++ b/mysys/lf_alloc-pin.c @@ -256,8 +256,9 @@ static int ptr_cmp(void **a, void **b) #define add_to_purgatory(PINS, ADDR) \ do \ { \ - *(void **)((char *)(ADDR)+(PINS)->pinbox->free_ptr_offset)= \ - (PINS)->purgatory; \ + my_atomic_storeptr_explicit( \ + (void **)((char *)(ADDR)+(PINS)->pinbox->free_ptr_offset), \ + (PINS)->purgatory, MY_MEMORY_ORDER_RELEASE); \ (PINS)->purgatory= (ADDR); \ (PINS)->purgatory_count++; \ } while (0) diff --git a/mysys/lf_hash.c b/mysys/lf_hash.c index a7553b349de..ca3a377190f 100644 --- a/mysys/lf_hash.c +++ b/mysys/lf_hash.c @@ -111,13 +111,16 @@ retry: cur_hashnr= cursor->curr->hashnr; cur_keylen= cursor->curr->keylen; - cur_key= cursor->curr->key; + cur_key= my_atomic_loadptr_explicit((void **) &cursor->curr->key, + MY_MEMORY_ORDER_ACQUIRE); do { - link= cursor->curr->link; + link= (intptr) my_atomic_loadptr_explicit((void **) &cursor->curr->link, + MY_MEMORY_ORDER_RELAXED); cursor->next= PTR(link); lf_pin(pins, 0, cursor->next); - } while (link != cursor->curr->link && LF_BACKOFF); + } while (link != (intptr) my_atomic_loadptr((void *) &cursor->curr->link) + && LF_BACKOFF); if (!DELETED(link)) { From 17802165a621e5c739e70d3eb427bd1cbaeaba8a Mon Sep 17 00:00:00 2001 From: Martin Beck Date: Wed, 24 Nov 2021 18:46:46 +1100 Subject: [PATCH 03/12] MDEV-27088: lf unit tests - cycles insufficient Per bug report, cycles was woefully insufficient to detect any implementation error. --- unittest/mysys/thr_template.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittest/mysys/thr_template.c b/unittest/mysys/thr_template.c index 5f01dfbe055..724fa8036cd 100644 --- a/unittest/mysys/thr_template.c +++ b/unittest/mysys/thr_template.c @@ -60,7 +60,7 @@ int main(int argc __attribute__((unused)), char **argv) pthread_mutex_init(&mutex, 0); -#define CYCLES 3000 +#define CYCLES 30000 #define THREADS 30 diag("N CPUs: %d, atomic ops: %s", my_getncpus(), MY_ATOMIC_MODE); From 045f5f7b10d391fc1653583594b4efbb5a495c4d Mon Sep 17 00:00:00 2001 From: Lukas Javorsky Date: Fri, 22 Nov 2019 15:03:20 +0100 Subject: [PATCH 04/12] MDEV-21108 Add option for setting install paths of groonga Include gronnga and groonga-normalizer-mysql install path --- storage/mroonga/vendor/groonga/CMakeLists.txt | 4 +++- .../vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/storage/mroonga/vendor/groonga/CMakeLists.txt b/storage/mroonga/vendor/groonga/CMakeLists.txt index 2929338d7b1..8afa53be1e0 100644 --- a/storage/mroonga/vendor/groonga/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/CMakeLists.txt @@ -86,7 +86,9 @@ set(LIB_DIR "lib") set(INCLUDE_DIR "include") set(GRN_INCLUDE_DIR "include/groonga") set(DATA_DIR "share") -set(GRN_DATA_DIR "${DATA_DIR}/${GRN_PROJECT_NAME}") +if(NOT DEFINED GRN_DATA_DIR) + set(GRN_DATA_DIR "${DATA_DIR}/${GRN_PROJECT_NAME}") +endif() set(CONFIG_DIR "etc") set(GRN_CONFIG_DIR "${CONFIG_DIR}/${GRN_PROJECT_NAME}") set(GRN_CONFIG_PATH "${CMAKE_INSTALL_PREFIX}/${GRN_CONFIG_DIR}/groonga.conf") diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt index ae083028038..4c2aa343089 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt @@ -16,7 +16,9 @@ # MA 02110-1335 USA cmake_minimum_required(VERSION 2.6) -set(GROONGA_NORMALIZER_MYSQL_PROJECT_NAME "groonga-normalizer-mysql") +if(NOT DEFINED GROONGA_NORMALIZER_MYSQL_PROJECT_NAME) + set(GROONGA_NORMALIZER_MYSQL_PROJECT_NAME "groonga-normalizer-mysql") +endif() project("${GROONGA_NORMALIZER_MYSQL_PROJECT_NAME}") if(DEFINED GROONGA_NORMALIZER_MYSQL_EMBED) From 0064316f19c2b3fe07d0722c05b205b4f2906d35 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Fri, 3 Dec 2021 17:21:59 +0600 Subject: [PATCH 05/12] cleanup: reduce code bloat --- storage/innobase/include/log0log.h | 20 ------ storage/innobase/log/log0log.cc | 101 +++++++++++++++++------------ 2 files changed, 60 insertions(+), 61 deletions(-) diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 1dcff513d7c..46cd6bbc4a2 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -346,26 +346,6 @@ or the MySQL version that created the redo log file. */ header */ #define LOG_FILE_HDR_SIZE (4 * OS_FILE_LOG_BLOCK_SIZE) -/** Memory mapped file */ -class mapped_file_t -{ -public: - mapped_file_t()= default; - mapped_file_t(const mapped_file_t &)= delete; - mapped_file_t &operator=(const mapped_file_t &)= delete; - mapped_file_t(mapped_file_t &&)= delete; - mapped_file_t &operator=(mapped_file_t &&)= delete; - ~mapped_file_t() noexcept; - - dberr_t map(const char *path, bool read_only= false, - bool nvme= false) noexcept; - dberr_t unmap() noexcept; - byte *data() noexcept { return m_area.data(); } - -private: - span m_area; -}; - /** Abstraction for reading, writing and flushing file cache to disk */ class file_io { diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 906979ffe20..914337d2a54 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -221,47 +221,6 @@ void log_t::create() (aligned_malloc(OS_FILE_LOG_BLOCK_SIZE, OS_FILE_LOG_BLOCK_SIZE)); } -mapped_file_t::~mapped_file_t() noexcept -{ - if (!m_area.empty()) - unmap(); -} - -dberr_t mapped_file_t::map(const char *path, bool read_only, - bool nvme) noexcept -{ - auto fd= mysql_file_open(innodb_log_file_key, path, - read_only ? O_RDONLY : O_RDWR, MYF(MY_WME)); - if (fd == -1) - return DB_ERROR; - - const auto file_size= os_file_get_size(path).m_total_size; - - const int nvme_flag= nvme ? MAP_SYNC : 0; - void *ptr= my_mmap(0, static_cast(file_size), - read_only ? PROT_READ : PROT_READ | PROT_WRITE, - MAP_SHARED_VALIDATE | nvme_flag, fd, 0); - mysql_file_close(fd, MYF(MY_WME)); - - if (ptr == MAP_FAILED) - return DB_ERROR; - - m_area= {static_cast(ptr), - static_cast::size_type>(file_size)}; - return DB_SUCCESS; -} - -dberr_t mapped_file_t::unmap() noexcept -{ - ut_ad(!m_area.empty()); - - if (my_munmap(m_area.data(), m_area.size())) - return DB_ERROR; - - m_area= {}; - return DB_SUCCESS; -} - file_os_io::file_os_io(file_os_io &&rhs) : m_fd(rhs.m_fd) { rhs.m_fd= OS_FILE_CLOSED; @@ -331,6 +290,66 @@ dberr_t file_os_io::flush() noexcept #include +/** Memory mapped file */ +class mapped_file_t +{ +public: + mapped_file_t()= default; + mapped_file_t(const mapped_file_t &)= delete; + mapped_file_t &operator=(const mapped_file_t &)= delete; + mapped_file_t(mapped_file_t &&)= delete; + mapped_file_t &operator=(mapped_file_t &&)= delete; + ~mapped_file_t() noexcept; + + dberr_t map(const char *path, bool read_only= false, + bool nvme= false) noexcept; + dberr_t unmap() noexcept; + byte *data() noexcept { return m_area.data(); } + +private: + span m_area; +}; + +mapped_file_t::~mapped_file_t() noexcept +{ + if (!m_area.empty()) + unmap(); +} + +dberr_t mapped_file_t::map(const char *path, bool read_only, + bool nvme) noexcept +{ + auto fd= mysql_file_open(innodb_log_file_key, path, + read_only ? O_RDONLY : O_RDWR, MYF(MY_WME)); + if (fd == -1) + return DB_ERROR; + + const auto file_size= size_t{os_file_get_size(path).m_total_size}; + + const int nvme_flag= nvme ? MAP_SYNC : 0; + void *ptr= + my_mmap(0, file_size, read_only ? PROT_READ : PROT_READ | PROT_WRITE, + MAP_SHARED_VALIDATE | nvme_flag, fd, 0); + mysql_file_close(fd, MYF(MY_WME)); + + if (ptr == MAP_FAILED) + return DB_ERROR; + + m_area= {static_cast(ptr), file_size}; + return DB_SUCCESS; +} + +dberr_t mapped_file_t::unmap() noexcept +{ + ut_ad(!m_area.empty()); + + if (my_munmap(m_area.data(), m_area.size())) + return DB_ERROR; + + m_area= {}; + return DB_SUCCESS; +} + static bool is_pmem(const char *path) noexcept { mapped_file_t mf; From 214cad8c3b56810574020a5e995d22b804294cf8 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 29 Nov 2021 17:03:21 +0100 Subject: [PATCH 06/12] fix ./mtr --manual warning after f5441ef4dac9 --- mysql-test/mysql-test-run.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 809f8794e00..4e00a76422b 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -5128,7 +5128,7 @@ sub mysqld_start ($$) { $mysqld->{'started_opts'}= $extra_opts; my $expect_file= "$opt_vardir/tmp/".$mysqld->name().".expect"; - return $oldexe eq $exe || + return $oldexe eq ($exe || '') || sleep_until_file_created($mysqld->value('pid-file'), $expect_file, $opt_start_timeout, $mysqld->{'proc'}, $warn_seconds); } From 71027eceac31c9f35c94c9893abb072e41867541 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 29 Nov 2021 17:02:31 +0100 Subject: [PATCH 07/12] fix srpm builds after fe065f8d90b0 --- cmake/Internal/CPack/CPackRPM.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/Internal/CPack/CPackRPM.cmake b/cmake/Internal/CPack/CPackRPM.cmake index a1040a0161f..d16965be1e9 100644 --- a/cmake/Internal/CPack/CPackRPM.cmake +++ b/cmake/Internal/CPack/CPackRPM.cmake @@ -18,7 +18,8 @@ macro(restore WHAT) endmacro() foreach (WHAT SUMMARY DESCRIPTION) - if(NOT CPACK_RPM_${CPACK_RPM_PACKAGE_COMPONENT}_PACKAGE_${WHAT}) + if(CPACK_RPM_PACKAGE_COMPONENT AND + NOT CPACK_RPM_${CPACK_RPM_PACKAGE_COMPONENT}_PACKAGE_${WHAT}) message(FATAL_ERROR "CPACK_RPM_${CPACK_RPM_PACKAGE_COMPONENT}_PACKAGE_${WHAT} is not defined") endif() endforeach() From 890c55177df3d6b92fa90c7cb49c909fcbe9ef4c Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Tue, 7 Dec 2021 15:22:06 +0600 Subject: [PATCH 08/12] MDEV-27183 optimize std::map lookup in during crash recovery This is a low hanging fruit. Before this patch std::map::emplace() was a ~50% of the whole recv_sys_t::parse() operation in by test. After the fix it's only ~20%. recv_sys_t::parse() recv_sys_t::pages is a collection of all pages to recovery. Often, there are multiple changes for a single page. Often, they go in a row and for such cases let's avoid lookup in a std::map. cached_pages_it serves as a cache of size 1. recv_sys_t::add(): replace page_id argument with a std::map::iterator --- storage/innobase/include/log0recv.h | 4 ++-- storage/innobase/log/log0recv.cc | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index fcc7e989885..9159ba00859 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -332,12 +332,12 @@ public: bool is_initialised() const { return last_stored_lsn != 0; } /** Register a redo log snippet for a page. - @param page_id page identifier + @param it page iterator @param start_lsn start LSN of the mini-transaction @param lsn @see mtr_t::commit_lsn() @param l redo log snippet @see log_t::FORMAT_10_5 @param len length of l, in bytes */ - inline void add(const page_id_t page_id, lsn_t start_lsn, lsn_t lsn, + inline void add(map::iterator it, lsn_t start_lsn, lsn_t lsn, const byte *l, size_t len); /** Parse and register one mini-transaction in log_t::FORMAT_10_5. diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index eb34fd8ede9..3e8986d4651 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1664,20 +1664,17 @@ inline void page_recv_t::will_not_read() /** Register a redo log snippet for a page. -@param page_id page identifier +@param it page iterator @param start_lsn start LSN of the mini-transaction @param lsn @see mtr_t::commit_lsn() @param recs redo log snippet @see log_t::FORMAT_10_5 @param len length of l, in bytes */ -inline void recv_sys_t::add(const page_id_t page_id, - lsn_t start_lsn, lsn_t lsn, const byte *l, - size_t len) +inline void recv_sys_t::add(map::iterator it, lsn_t start_lsn, lsn_t lsn, + const byte *l, size_t len) { ut_ad(mutex_own(&mutex)); - std::pair p= pages.emplace(map::value_type - (page_id, page_recv_t())); - page_recv_t& recs= p.first->second; - ut_ad(p.second == recs.log.empty()); + page_id_t page_id = it->first; + page_recv_t &recs= it->second; switch (*l & 0x70) { case FREE_PAGE: case INIT_PAGE: @@ -1769,6 +1766,7 @@ bool recv_sys_t::parse(lsn_t checkpoint_lsn, store_t *store, bool apply) loop: const byte *const log= buf + recovered_offset; const lsn_t start_lsn= recovered_lsn; + map::iterator cached_pages_it = pages.end(); /* Check that the entire mini-transaction is included within the buffer */ const byte *l; @@ -2092,8 +2090,12 @@ same_page: /* fall through */ case STORE_YES: if (!mlog_init.will_avoid_read(id, start_lsn)) - add(id, start_lsn, end_lsn, recs, + { + if (cached_pages_it == pages.end() || cached_pages_it->first != id) + cached_pages_it= pages.emplace(id, page_recv_t()).first; + add(cached_pages_it, start_lsn, end_lsn, recs, static_cast(l + rlen - recs)); + } continue; case STORE_NO: if (!is_init) From cfcfdc65dfa0ae5e59df1f1c42ee574b8bdc60a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 7 Dec 2021 17:00:46 +0200 Subject: [PATCH 09/12] MDEV-27190 InnoDB upgrade from 10.2, 10.3, 10.4 is not crash-safe During startup, InnoDB must write a FILE_CHECKPOINT record. However, before MDEV-12353 (in MariaDB Server 10.2, 10.3, 10.4) the corresponding record MLOG_CHECKPOINT was encoded in a different way. When we are upgrading from a logically empty 10.2, 10.3, or 10.4 redo log, we must not write anything to the old log file, because if the server were killed during the upgrade, we would end up with a corrupted log file, and both the old and the new server would refuse to start up. On upgrade, we must simply create a new logically empty log file and replace the old ib_logfile0 with that. --- storage/innobase/log/log0recv.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 3e8986d4651..dfadd9df37f 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3661,7 +3661,9 @@ completed: log_sys.last_checkpoint_lsn = checkpoint_lsn; - if (!srv_read_only_mode && srv_operation == SRV_OPERATION_NORMAL) { + if (!srv_read_only_mode && srv_operation == SRV_OPERATION_NORMAL + && (~log_t::FORMAT_ENCRYPTED & log_sys.log.format) + == log_t::FORMAT_10_5) { /* Write a FILE_CHECKPOINT marker as the first thing, before generating any other redo log. This ensures that subsequent crash recovery will be possible even From 88ac91c7cc18425b56e3285c7e533b00af5b8281 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 7 Dec 2021 17:29:05 +0100 Subject: [PATCH 10/12] ColumnStore and S3 SUMMARY/DESCRIPTION for RPM --- storage/columnstore/CMakeLists.txt | 2 ++ storage/maria/CMakeLists.txt | 3 +++ 2 files changed, 5 insertions(+) diff --git a/storage/columnstore/CMakeLists.txt b/storage/columnstore/CMakeLists.txt index 6a6d91b18fe..81690f14b15 100644 --- a/storage/columnstore/CMakeLists.txt +++ b/storage/columnstore/CMakeLists.txt @@ -28,6 +28,8 @@ CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") SET(CPACK_RPM_columnstore-engine_POST_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/columnstore/build/postInstall_storage_engine.sh PARENT_SCOPE) SET(CPACK_RPM_columnstore-engine_PRE_UNINSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/columnstore/build/preUn_storage_engine.sh PARENT_SCOPE) APPEND_FOR_CPACK(CPACK_RPM_columnstore-engine_PACKAGE_VERSION "") + SET(CPACK_RPM_columnstore-engine_PACKAGE_SUMMARY "MariaDB ColumnStore storage engine" PARENT_SCOPE) + SET(CPACK_RPM_columnstore-engine_PACKAGE_DESCRIPTION "The MariaDB ColumnStore storage engine is a high-performance columnar analytical engine, aimed at rapid processing of analytical queries on very large amounts of data." PARENT_SCOPE) ENDIF() INSTALL_MYSQL_TEST("${CMAKE_CURRENT_SOURCE_DIR}/mysql-test/" "plugin/columnstore") ENDIF() diff --git a/storage/maria/CMakeLists.txt b/storage/maria/CMakeLists.txt index 67540a08b09..1091b6d9a2c 100644 --- a/storage/maria/CMakeLists.txt +++ b/storage/maria/CMakeLists.txt @@ -127,6 +127,9 @@ IF (CURL_FOUND) LINK_LIBRARIES curl z STORAGE_ENGINE NOT_EMBEDDED CONFIG s3.cnf) ENDIF() +SET(CPACK_RPM_s3-engine_PACKAGE_SUMMARY "Amazon S3 archival storage engine for MariaDB" PARENT_SCOPE) +SET(CPACK_RPM_s3-engine_PACKAGE_DESCRIPTION "The S3 storage engine allows one to archive MariaDB tables in Amazon S3 (or any third-party public or private cloud that implements S3 API), but still have them accessible in MariaDB in read-only mode." PARENT_SCOPE) + IF(TARGET s3) MYSQL_ADD_EXECUTABLE(aria_s3_copy aria_s3_copy.cc ${S3_SOURCES} COMPONENT s3-engine) TARGET_LINK_LIBRARIES(aria_s3_copy aria myisam mysys mysys_ssl curl z) From e27b1c31378a6d2592b8ffafb0c7fcc5a1c93072 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 5 Nov 2021 15:12:14 +0100 Subject: [PATCH 11/12] require system pcre2 in rpms now when SLES12.3 is gone, we can enforce it --- cmake/build_configurations/mysql_release.cmake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index 97c59eb3ca4..11913da2589 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -106,8 +106,7 @@ ELSEIF(RPM) SET(SEMODULE_PACKAGE /usr/bin/semodule_package CACHE FILEPATH "") SET(PLUGIN_AUTH_SOCKET YES CACHE STRING "") SET(WITH_EMBEDDED_SERVER ON CACHE BOOL "") - # not yet, SLES 12.3 doesn't provide pcre2 - #SET(WITH_PCRE system CACHE STRING "") + SET(WITH_PCRE system CACHE STRING "") IF(RPM MATCHES "fedora|centos|rhel") SET(WITH_INNODB_BZIP2 OFF CACHE STRING "") SET(WITH_INNODB_LZO OFF CACHE STRING "") @@ -137,7 +136,6 @@ ELSE() SET(WITH_JEMALLOC static CACHE STRING "") SET(PLUGIN_AUTH_SOCKET STATIC CACHE STRING "") SET(WITH_STRIPPED_CLIENT ON CACHE BOOL "Strip all client binaries") - SET(WITH_PCRE bundled CACHE STRING "") SET(WITH_INNODB_BZIP2 OFF CACHE STRING "") SET(WITH_INNODB_LZ4 OFF CACHE STRING "") SET(WITH_INNODB_LZO OFF CACHE STRING "") From 62ea1b4407762164982221dc9796502a91c139cc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 28 Oct 2021 09:43:49 +0200 Subject: [PATCH 12/12] BUG#31761802 STATISTICS ANY QUERIES USING VIEWS ARE SUMMARIZED TOGETHER WITH THE VIEW DEFINITION SELECT test case only --- .../suite/perfschema/r/digest_view.result | 144 ++++++++++++++++++ .../suite/perfschema/t/digest_view.test | 76 +++++++++ 2 files changed, 220 insertions(+) create mode 100644 mysql-test/suite/perfschema/r/digest_view.result create mode 100644 mysql-test/suite/perfschema/t/digest_view.test diff --git a/mysql-test/suite/perfschema/r/digest_view.result b/mysql-test/suite/perfschema/r/digest_view.result new file mode 100644 index 00000000000..43a8bdb1577 --- /dev/null +++ b/mysql-test/suite/perfschema/r/digest_view.result @@ -0,0 +1,144 @@ +CREATE TABLE test.v1 (a int, b int); +INSERT INTO test.v1 VALUES (1, 100), (2, 200), (3, 300); +CREATE TABLE test.t1 (a int, b int); +INSERT INTO test.t1 VALUES (1, 100), (2, 200), (3, 300); +TRUNCATE TABLE performance_schema.events_statements_summary_by_digest; +EXPLAIN EXTENDED SELECT * from test.v1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE v1 ALL NULL NULL NULL NULL 3 100.00 +Warnings: +Note 1003 select `test`.`v1`.`a` AS `a`,`test`.`v1`.`b` AS `b` from `test`.`v1` +EXPLAIN EXTENDED SELECT * from test.v1 where a = 1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE v1 ALL NULL NULL NULL NULL 3 100.00 Using where +Warnings: +Note 1003 select `test`.`v1`.`a` AS `a`,`test`.`v1`.`b` AS `b` from `test`.`v1` where `test`.`v1`.`a` = 1 +EXPLAIN EXTENDED SELECT * from test.v1 where b > 100; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE v1 ALL NULL NULL NULL NULL 3 100.00 Using where +Warnings: +Note 1003 select `test`.`v1`.`a` AS `a`,`test`.`v1`.`b` AS `b` from `test`.`v1` where `test`.`v1`.`b` > 100 +EXPLAIN EXTENDED SELECT a, b from test.v1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE v1 ALL NULL NULL NULL NULL 3 100.00 +Warnings: +Note 1003 select `test`.`v1`.`a` AS `a`,`test`.`v1`.`b` AS `b` from `test`.`v1` +EXPLAIN EXTENDED SELECT b, a from test.v1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE v1 ALL NULL NULL NULL NULL 3 100.00 +Warnings: +Note 1003 select `test`.`v1`.`b` AS `b`,`test`.`v1`.`a` AS `a` from `test`.`v1` +SELECT * from test.v1; +a b +1 100 +2 200 +3 300 +SELECT * from test.v1 where a = 1; +a b +1 100 +SELECT * from test.v1 where b > 100; +a b +2 200 +3 300 +SELECT a, b from test.v1; +a b +1 100 +2 200 +3 300 +SELECT b, a from test.v1; +b a +100 1 +200 2 +300 3 +# +# DIGESTS SEEN ON TABLE +# +SELECT SCHEMA_NAME, DIGEST_TEXT, COUNT_STAR +FROM performance_schema.events_statements_summary_by_digest +ORDER BY DIGEST_TEXT; +SCHEMA_NAME DIGEST_TEXT COUNT_STAR +test EXPLAIN EXTENDED SELECT * FROM `test` . `v1` 1 +test EXPLAIN EXTENDED SELECT * FROM `test` . `v1` WHERE `a` = ? 1 +test EXPLAIN EXTENDED SELECT * FROM `test` . `v1` WHERE `b` > ? 1 +test EXPLAIN EXTENDED SELECT `a` , `b` FROM `test` . `v1` 1 +test EXPLAIN EXTENDED SELECT `b` , `a` FROM `test` . `v1` 1 +test SELECT * FROM `test` . `v1` 1 +test SELECT * FROM `test` . `v1` WHERE `a` = ? 1 +test SELECT * FROM `test` . `v1` WHERE `b` > ? 1 +test SELECT `a` , `b` FROM `test` . `v1` 1 +test SELECT `b` , `a` FROM `test` . `v1` 1 +test SHOW WARNINGS 5 +test TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 +DROP TABLE test.v1; +CREATE VIEW test.v1 AS SELECT * FROM test.t1; +EXPLAIN EXTENDED SELECT * from test.v1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 3 100.00 +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` +EXPLAIN EXTENDED SELECT * from test.v1 where a = 1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 3 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` = 1 +EXPLAIN EXTENDED SELECT * from test.v1 where b > 100; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 3 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`b` > 100 +EXPLAIN EXTENDED SELECT a, b from test.v1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 3 100.00 +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` +EXPLAIN EXTENDED SELECT b, a from test.v1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 3 100.00 +Warnings: +Note 1003 select `test`.`t1`.`b` AS `b`,`test`.`t1`.`a` AS `a` from `test`.`t1` +SELECT * from test.v1; +a b +1 100 +2 200 +3 300 +SELECT * from test.v1 where a = 1; +a b +1 100 +SELECT * from test.v1 where b > 100; +a b +2 200 +3 300 +SELECT a, b from test.v1; +a b +1 100 +2 200 +3 300 +SELECT b, a from test.v1; +b a +100 1 +200 2 +300 3 +# +# DIGESTS SEEN ON VIEW +# +SELECT SCHEMA_NAME, DIGEST_TEXT, COUNT_STAR +FROM performance_schema.events_statements_summary_by_digest +ORDER BY DIGEST_TEXT; +SCHEMA_NAME DIGEST_TEXT COUNT_STAR +test CREATE VIEW `test` . `v1` AS SELECT * FROM `test` . `t1` 1 +test DROP TABLE `test` . `v1` 1 +test EXPLAIN EXTENDED SELECT * FROM `test` . `v1` 2 +test EXPLAIN EXTENDED SELECT * FROM `test` . `v1` WHERE `a` = ? 2 +test EXPLAIN EXTENDED SELECT * FROM `test` . `v1` WHERE `b` > ? 2 +test EXPLAIN EXTENDED SELECT `a` , `b` FROM `test` . `v1` 2 +test EXPLAIN EXTENDED SELECT `b` , `a` FROM `test` . `v1` 2 +test SELECT * FROM `test` . `v1` 2 +test SELECT * FROM `test` . `v1` WHERE `a` = ? 2 +test SELECT * FROM `test` . `v1` WHERE `b` > ? 2 +test SELECT SCHEMA_NAME , `DIGEST_TEXT` , `COUNT_STAR` FROM `performance_schema` . `events_statements_summary_by_digest` ORDER BY `DIGEST_TEXT` 1 +test SELECT `a` , `b` FROM `test` . `v1` 2 +test SELECT `b` , `a` FROM `test` . `v1` 2 +test SHOW WARNINGS 10 +test TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 +DROP VIEW test.v1; +DROP TABLE test.t1; diff --git a/mysql-test/suite/perfschema/t/digest_view.test b/mysql-test/suite/perfschema/t/digest_view.test new file mode 100644 index 00000000000..462d60556eb --- /dev/null +++ b/mysql-test/suite/perfschema/t/digest_view.test @@ -0,0 +1,76 @@ +# ---------------------------------------------------- +# Tests for the performance schema statement Digests. +# ---------------------------------------------------- + +# Test case to show behavior of statements digest when +# using a view + +# Test requires: sp-protocol/ps-protocol/view-protocol/cursor-protocol disabled +--source include/no_protocol.inc +--source include/not_embedded.inc + +CREATE TABLE test.v1 (a int, b int); +INSERT INTO test.v1 VALUES (1, 100), (2, 200), (3, 300); + +CREATE TABLE test.t1 (a int, b int); +INSERT INTO test.t1 VALUES (1, 100), (2, 200), (3, 300); + + +TRUNCATE TABLE performance_schema.events_statements_summary_by_digest; + +# +# test.v1 is a table. +# Every query here is different, and should have a different digest. +# + +EXPLAIN EXTENDED SELECT * from test.v1; +EXPLAIN EXTENDED SELECT * from test.v1 where a = 1; +EXPLAIN EXTENDED SELECT * from test.v1 where b > 100; +EXPLAIN EXTENDED SELECT a, b from test.v1; +EXPLAIN EXTENDED SELECT b, a from test.v1; + +SELECT * from test.v1; +SELECT * from test.v1 where a = 1; +SELECT * from test.v1 where b > 100; +SELECT a, b from test.v1; +SELECT b, a from test.v1; + +--echo # +--echo # DIGESTS SEEN ON TABLE +--echo # + +SELECT SCHEMA_NAME, DIGEST_TEXT, COUNT_STAR + FROM performance_schema.events_statements_summary_by_digest + ORDER BY DIGEST_TEXT; + +DROP TABLE test.v1; +CREATE VIEW test.v1 AS SELECT * FROM test.t1; + +# +# test.v1 is now a view. +# the query digests should be unchanged. +# + +EXPLAIN EXTENDED SELECT * from test.v1; +EXPLAIN EXTENDED SELECT * from test.v1 where a = 1; +EXPLAIN EXTENDED SELECT * from test.v1 where b > 100; +EXPLAIN EXTENDED SELECT a, b from test.v1; +EXPLAIN EXTENDED SELECT b, a from test.v1; + +SELECT * from test.v1; +SELECT * from test.v1 where a = 1; +SELECT * from test.v1 where b > 100; +SELECT a, b from test.v1; +SELECT b, a from test.v1; + +--echo # +--echo # DIGESTS SEEN ON VIEW +--echo # + +SELECT SCHEMA_NAME, DIGEST_TEXT, COUNT_STAR + FROM performance_schema.events_statements_summary_by_digest + ORDER BY DIGEST_TEXT; + +DROP VIEW test.v1; +DROP TABLE test.t1; +