mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-01-28 17:54:16 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge 10.6 into 10.11

Browse source
This commit is contained in:
Marko Mäkelä 2024-01-22 08:49:04 +02:00
commit b3ca7fa089
42 changed files with 269 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
client/CMakeLists.txt
View file

@ -16,7 +16,7 @@
INCLUDE_DIRECTORIES(
${CMAKE_SOURCE_DIR}/include
${PCRE_INCLUDES}
${PCRE_INCLUDE_DIRS}
${CMAKE_SOURCE_DIR}/mysys_ssl
${ZLIB_INCLUDE_DIR}
${SSL_INCLUDE_DIRS}

3
cmake/os/AIX.cmake
View file

@ -34,5 +34,8 @@ ELSE()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -maix64 -pthread -mcmodel=large")
ENDIF()
# fcntl(fd, F_SETFL, O_DIRECT) is not supported; O_DIRECT is an open(2) flag
SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "")
# make it WARN by default, not AUTO (that implies -Werror)
SET(MYSQL_MAINTAINER_MODE "WARN" CACHE STRING "Enable MariaDB maintainer-specific warnings. One of: NO (warnings are disabled) WARN (warnings are enabled) ERR (warnings are errors) AUTO (warnings are errors in Debug only)")

4
cmake/os/SunOS.cmake
View file

@ -17,6 +17,10 @@ INCLUDE(CheckSymbolExists)
INCLUDE(CheckCSourceRuns)
INCLUDE(CheckCSourceCompiles)
# fcntl(fd, F_SETFL, O_DIRECT) is not supported,
# and directio(3C) would only work on UFS or NFS, not ZFS.
SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "")
# Enable 64 bit file offsets
SET(_FILE_OFFSET_BITS 64)

1
cmake/os/WindowsCache.cmake
View file

@ -44,6 +44,7 @@ SET(HAVE_EXECINFO_H CACHE INTERNAL "")
SET(HAVE_FCHMOD CACHE INTERNAL "")
SET(HAVE_FCNTL CACHE INTERNAL "")
SET(HAVE_FCNTL_H 1 CACHE INTERNAL "")
SET(HAVE_FCNTL_DIRECT 0 CACHE INTERNAL "")
SET(HAVE_FCNTL_NONBLOCK CACHE INTERNAL "")
SET(HAVE_FDATASYNC CACHE INTERNAL "")
SET(HAVE_DECL_FDATASYNC CACHE INTERNAL "")

34
cmake/pcre.cmake
View file

@ -1,4 +1,3 @@
INCLUDE (CheckCSourceRuns)
INCLUDE (ExternalProject)
SET(WITH_PCRE "auto" CACHE STRING
@ -6,7 +5,8 @@ SET(WITH_PCRE "auto" CACHE STRING
MACRO(BUNDLE_PCRE2)
SET(dir "${CMAKE_BINARY_DIR}/extra/pcre2")
SET(PCRE_INCLUDES ${dir}/src/pcre2-build ${dir}/src/pcre2/src)
SET(PCRE_INCLUDE_DIRS ${dir}/src/pcre2-build ${dir}/src/pcre2/src)
MESSAGE(STATUS "Will download and bundle pcre2")
SET(byproducts)
FOREACH(lib pcre2-posix pcre2-8)
ADD_LIBRARY(${lib} STATIC IMPORTED GLOBAL)
@ -76,18 +76,26 @@ SET_TARGET_PROPERTIES(pcre2 PROPERTIES EXCLUDE_FROM_ALL TRUE)
ENDMACRO()
MACRO (CHECK_PCRE)
IF(WITH_PCRE STREQUAL "system" OR WITH_PCRE STREQUAL "auto")
CHECK_LIBRARY_EXISTS(pcre2-8 pcre2_match_8 "" HAVE_PCRE2)
ENDIF()
IF(NOT HAVE_PCRE2 OR WITH_PCRE STREQUAL "bundled")
IF (WITH_PCRE STREQUAL "system")
MESSAGE(FATAL_ERROR "system pcre2-8 library is not found or unusable")
IF (NOT TARGET pcre2 AND NOT PCRE_FOUND)
IF(WITH_PCRE STREQUAL "system" OR WITH_PCRE STREQUAL "auto")
FIND_PACKAGE(PkgConfig QUIET)
PKG_CHECK_MODULES(PCRE libpcre2-8)
# in case pkg-config or libpcre2-8.pc is not installed:
IF(NOT PCRE_FOUND)
UNSET(PCRE_FOUND CACHE)
CHECK_LIBRARY_EXISTS(pcre2-8 pcre2_match_8 "" PCRE_FOUND)
ENDIF()
ENDIF()
BUNDLE_PCRE2()
ELSE()
CHECK_LIBRARY_EXISTS(pcre2-posix PCRE2regcomp "" NEEDS_PCRE2_DEBIAN_HACK)
IF(NEEDS_PCRE2_DEBIAN_HACK)
SET(PCRE2_DEBIAN_HACK "-Dregcomp=PCRE2regcomp -Dregexec=PCRE2regexec -Dregerror=PCRE2regerror -Dregfree=PCRE2regfree")
IF(NOT PCRE_FOUND OR WITH_PCRE STREQUAL "bundled")
IF (WITH_PCRE STREQUAL "system")
MESSAGE(FATAL_ERROR "system pcre2-8 library is not found or unusable")
ENDIF()
BUNDLE_PCRE2()
ELSE()
CHECK_LIBRARY_EXISTS(pcre2-posix PCRE2regcomp "" NEEDS_PCRE2_DEBIAN_HACK)
IF(NEEDS_PCRE2_DEBIAN_HACK)
SET(PCRE2_DEBIAN_HACK "-Dregcomp=PCRE2regcomp -Dregexec=PCRE2regexec -Dregerror=PCRE2regerror -Dregfree=PCRE2regfree")
ENDIF()
ENDIF()
ENDIF()
ENDMACRO()

2
cmake/plugin.cmake
View file

@ -44,7 +44,7 @@ MACRO(MYSQL_ADD_PLUGIN)
# Add common include directories
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include
${CMAKE_SOURCE_DIR}/sql
${PCRE_INCLUDES}
${PCRE_INCLUDE_DIRS}
${SSL_INCLUDE_DIRS}
${ZLIB_INCLUDE_DIR})

1
config.h.cmake
View file

@ -30,6 +30,7 @@
#cmakedefine HAVE_DLFCN_H 1
#cmakedefine HAVE_EXECINFO_H 1
#cmakedefine HAVE_FCNTL_H 1
#cmakedefine HAVE_FCNTL_DIRECT 1
#cmakedefine HAVE_FENV_H 1
#cmakedefine HAVE_FLOAT_H 1
#cmakedefine HAVE_FNMATCH_H 1

1
configure.cmake
View file

@ -706,6 +706,7 @@ CHECK_SYMBOL_EXISTS(O_NONBLOCK "unistd.h;fcntl.h" HAVE_FCNTL_NONBLOCK)
IF(NOT HAVE_FCNTL_NONBLOCK)
SET(NO_FCNTL_NONBLOCK 1)
ENDIF()
CHECK_SYMBOL_EXISTS(O_DIRECT "fcntl.h" HAVE_FCNTL_DIRECT)
#
# Test for how the C compiler does inline, if at all

2
extra/mariabackup/CMakeLists.txt
View file

@ -36,7 +36,7 @@ INCLUDE_DIRECTORIES(
)
IF(NOT HAVE_SYSTEM_REGEX)
INCLUDE_DIRECTORIES(${PCRE_INCLUDES})
INCLUDE_DIRECTORIES(${PCRE_INCLUDE_DIRS})
ADD_DEFINITIONS(${PCRE2_DEBIAN_HACK})
ENDIF()

2
extra/mariabackup/fil_cur.cc
View file

@ -199,11 +199,13 @@ xb_fil_cur_open(
return(XB_FIL_CUR_SKIP);
}
#ifdef HAVE_FCNTL_DIRECT
if (srv_file_flush_method == SRV_O_DIRECT
|| srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) {
os_file_set_nocache(cursor->file, node->name, "OPEN");
}
#endif
posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);

2
libmariadb

@ -1 +1 @@
Subproject commit 458a4396b443dcefedcf464067560078aa09d8b4
Subproject commit 26cef16b25f01c1b159b0c73e3f6d9bcfcac90d5

2
libmysqld/CMakeLists.txt
View file

@ -23,7 +23,7 @@ ${CMAKE_SOURCE_DIR}/libmysqld
${CMAKE_SOURCE_DIR}/sql
${CMAKE_SOURCE_DIR}/tpool
${CMAKE_BINARY_DIR}/sql
${PCRE_INCLUDES}
${PCRE_INCLUDE_DIRS}
${LIBFMT_INCLUDE_DIR}
${ZLIB_INCLUDE_DIR}
${SSL_INCLUDE_DIRS}

2
libmysqld/examples/CMakeLists.txt
View file

@ -15,7 +15,7 @@
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include
${CMAKE_SOURCE_DIR}/libmysqld/include
${PCRE_INCLUDES}
${PCRE_INCLUDE_DIRS}
${CMAKE_SOURCE_DIR}/sql
${MY_READLINE_INCLUDE_DIR}
)

2
mysql-test/mariadb-test-run.pl
View file

@ -4506,7 +4506,7 @@ sub extract_warning_lines ($$) {
qr|InnoDB: io_setup\(\) failed with EAGAIN|,
qr|io_uring_queue_init\(\) failed with|,
qr|InnoDB: liburing disabled|,
qr/InnoDB: Failed to set (O_DIRECT|DIRECTIO_ON) on file/,
qr/InnoDB: Failed to set O_DIRECT on file/,
qr|setrlimit could not change the size of core files to 'infinity';|,
qr|feedback plugin: failed to retrieve the MAC address|,
qr|Plugin 'FEEDBACK' init function returned error|,

3
mysql-test/suite/innodb/t/doublewrite.test
View file

@ -39,6 +39,9 @@ commit work;
SET GLOBAL innodb_fast_shutdown = 0;
let $shutdown_timeout=;
--source include/restart_mysqld.inc
# Ensure that buf_flush_page_cleaner() has woken up from its
# first my_cond_timedwait() and gone idle.
sleep 1;
--source ../include/no_checkpoint_start.inc
connect (dml,localhost,root,,);
XA START 'x';

22
mysql-test/suite/perfschema/r/sxlock_func,debug.rdiff
View file

@ -1,22 +0,0 @@
@@ -7,7 +7,6 @@
WHERE name LIKE 'wait/synch/rwlock/innodb/%'
AND name!='wait/synch/rwlock/innodb/btr_search_latch' ORDER BY name;
name
-wait/synch/rwlock/innodb/dict_operation_lock
wait/synch/rwlock/innodb/fil_space_latch
wait/synch/rwlock/innodb/lock_latch
wait/synch/rwlock/innodb/trx_i_s_cache_lock
@@ -19,11 +18,13 @@
select name from performance_schema.setup_instruments
where name like "wait/synch/sxlock/%" order by name;
name
+wait/synch/sxlock/innodb/dict_operation_lock
wait/synch/sxlock/innodb/index_tree_rw_lock
SELECT DISTINCT name FROM performance_schema.rwlock_instances
WHERE name LIKE 'wait/synch/sxlock/innodb/%'
ORDER BY name;
name
+wait/synch/sxlock/innodb/dict_operation_lock
wait/synch/sxlock/innodb/index_tree_rw_lock
create table t1(a int) engine=innodb;
begin;

1
mysql-test/suite/perfschema/t/sxlock_func.test
View file

@ -5,7 +5,6 @@
--source include/not_embedded.inc
--source include/have_perfschema.inc
--source include/have_innodb.inc
--source include/maybe_debug.inc
UPDATE performance_schema.setup_instruments SET enabled = 'NO', timed = 'YES';

2
plugin/feedback/CMakeLists.txt
View file

@ -1,5 +1,5 @@
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql
${PCRE_INCLUDES}
${PCRE_INCLUDE_DIRS}
${SSL_INCLUDE_DIRS})
SET(FEEDBACK_SOURCES feedback.cc sender_thread.cc

2
plugin/qc_info/CMakeLists.txt
View file

@ -1,4 +1,4 @@
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql
${PCRE_INCLUDES})
${PCRE_INCLUDE_DIRS})
MYSQL_ADD_PLUGIN(QUERY_CACHE_INFO qc_info.cc RECOMPILE_FOR_EMBEDDED)

2
scripts/mysql_install_db.sh
View file

@ -695,7 +695,7 @@ then
then
echo
echo "You can start the MariaDB daemon with:"
echo "cd '$basedir' ; $bindir/mariadb-safe --datadir='$ldata'"
echo "cd '$basedir' ; $bindir/mariadbd-safe --datadir='$ldata'"
echo
echo "You can test the MariaDB daemon with mysql-test-run.pl"
echo "cd '$basedir/@INSTALL_MYSQLTESTDIR@' ; perl mariadb-test-run.pl"

2
sql/CMakeLists.txt
View file

@ -55,7 +55,7 @@ INCLUDE_DIRECTORIES(
${CMAKE_SOURCE_DIR}/include
${CMAKE_SOURCE_DIR}/sql
${LIBFMT_INCLUDE_DIR}
${PCRE_INCLUDES}
${PCRE_INCLUDE_DIRS}
${ZLIB_INCLUDE_DIR}
${SSL_INCLUDE_DIRS}
${CMAKE_BINARY_DIR}/sql

4
storage/innobase/buf/buf0dump.cc
View file

@ -180,7 +180,7 @@ static void buf_dump_generate_path(char *path, size_t path_size)
char buf[FN_REFLEN];
mysql_mutex_lock(&LOCK_global_system_variables);
snprintf(buf, sizeof buf, "%s/%s", get_buf_dump_dir(),
snprintf(buf, sizeof buf, "%s" FN_ROOTDIR "%s", get_buf_dump_dir(),
srv_buf_dump_filename);
mysql_mutex_unlock(&LOCK_global_system_variables);
@ -214,7 +214,7 @@ static void buf_dump_generate_path(char *path, size_t path_size)
format = "%s%s";
break;
default:
format = "%s/%s";
format = "%s" FN_ROOTDIR "%s";
}
snprintf(path, path_size, format,

79
storage/innobase/buf/buf0flu.cc
View file

@ -1759,6 +1759,28 @@ ulint buf_flush_LRU(ulint max_n, bool evict)
buf_pool.try_LRU_scan= true;
pthread_cond_broadcast(&buf_pool.done_free);
}
else if (!pages && !buf_pool.try_LRU_scan &&
!buf_pool.LRU_warned.test_and_set(std::memory_order_acquire))
{
/* For example, with the minimum innodb_buffer_pool_size=5M and
the default innodb_page_size=16k there are only a little over 316
pages in the buffer pool. The buffer pool can easily be exhausted
by a workload of some dozen concurrent connections. The system could
reach a deadlock like the following:
(1) Many threads are waiting in buf_LRU_get_free_block()
for buf_pool.done_free.
(2) Some threads are waiting for a page latch which is held by
another thread that is waiting in buf_LRU_get_free_block().
(3) This thread is the only one that could make progress, but
we fail to do so because all the pages that we scanned are
buffer-fixed or latched by some thread. */
sql_print_warning("InnoDB: Could not free any blocks in the buffer pool!"
" %zu blocks are in use and %zu free."
" Consider increasing innodb_buffer_pool_size.",
UT_LIST_GET_LEN(buf_pool.LRU),
UT_LIST_GET_LEN(buf_pool.free));
}
return pages;
}
@ -2391,11 +2413,19 @@ func_exit:
goto func_exit;
}
TPOOL_SUPPRESS_TSAN
bool buf_pool_t::need_LRU_eviction() const
{
/* try_LRU_scan==false means that buf_LRU_get_free_block() is waiting
for buf_flush_page_cleaner() to evict some blocks */
return UNIV_UNLIKELY(!try_LRU_scan ||
(UT_LIST_GET_LEN(LRU) > BUF_LRU_MIN_LEN &&
UT_LIST_GET_LEN(free) < srv_LRU_scan_depth / 2));
}
#if defined __aarch64__&&defined __GNUC__&&__GNUC__==4&&!defined __clang__
/* Avoid GCC 4.8.5 internal compiler error "could not split insn".
We would only need this for buf_flush_page_cleaner(),
but GCC 4.8.5 does not support pop_options. */
# pragma GCC optimize ("O0")
/* Avoid GCC 4.8.5 internal compiler error "could not split insn". */
__attribute__((optimize(0)))
#endif
/** page_cleaner thread tasked with flushing dirty pages from the buffer
pools. As of now we'll have only one coordinator. */
@ -2429,21 +2459,24 @@ static void buf_flush_page_cleaner()
}
mysql_mutex_lock(&buf_pool.flush_list_mutex);
if (buf_pool.ran_out())
goto no_wait;
else if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED)
break;
if (!buf_pool.need_LRU_eviction())
{
if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED)
break;
if (buf_pool.page_cleaner_idle() &&
(!UT_LIST_GET_LEN(buf_pool.flush_list) ||
srv_max_dirty_pages_pct_lwm == 0.0))
/* We are idle; wait for buf_pool.page_cleaner_wakeup() */
my_cond_wait(&buf_pool.do_flush_list,
&buf_pool.flush_list_mutex.m_mutex);
else
my_cond_timedwait(&buf_pool.do_flush_list,
&buf_pool.flush_list_mutex.m_mutex, &abstime);
no_wait:
if (buf_pool.page_cleaner_idle() &&
(!UT_LIST_GET_LEN(buf_pool.flush_list) ||
srv_max_dirty_pages_pct_lwm == 0.0))
{
buf_pool.LRU_warned.clear(std::memory_order_release);
/* We are idle; wait for buf_pool.page_cleaner_wakeup() */
my_cond_wait(&buf_pool.do_flush_list,
&buf_pool.flush_list_mutex.m_mutex);
}
else
my_cond_timedwait(&buf_pool.do_flush_list,
&buf_pool.flush_list_mutex.m_mutex, &abstime);
}
set_timespec(abstime, 1);
lsn_limit= buf_flush_sync_lsn;
@ -2475,7 +2508,7 @@ static void buf_flush_page_cleaner()
}
while (false);
if (!buf_pool.ran_out())
if (!buf_pool.need_LRU_eviction())
continue;
mysql_mutex_lock(&buf_pool.flush_list_mutex);
oldest_lsn= buf_pool.get_oldest_modification(0);
@ -2504,7 +2537,7 @@ static void buf_flush_page_cleaner()
if (oldest_lsn >= soft_lsn_limit)
buf_flush_async_lsn= soft_lsn_limit= 0;
}
else if (buf_pool.ran_out())
else if (buf_pool.need_LRU_eviction())
{
buf_pool.page_cleaner_set_idle(false);
buf_pool.n_flush_inc();
@ -2619,9 +2652,13 @@ static void buf_flush_page_cleaner()
MONITOR_FLUSH_ADAPTIVE_PAGES,
n_flushed);
}
else if (buf_flush_async_lsn <= oldest_lsn)
else if (buf_flush_async_lsn <= oldest_lsn &&
!buf_pool.need_LRU_eviction())
goto check_oldest_and_set_idle;
else
mysql_mutex_lock(&buf_pool.mutex);
n= srv_max_io_capacity;
n= n >= n_flushed ? n - n_flushed : 0;
goto LRU_flush;
}

31
storage/innobase/buf/buf0lru.cc
View file

@ -60,10 +60,6 @@ static constexpr ulint BUF_LRU_OLD_TOLERANCE = 20;
frames in the buffer pool, we set this to TRUE */
static bool buf_lru_switched_on_innodb_mon = false;
/** True if diagnostic message about difficult to find free blocks
in the buffer bool has already printed. */
static bool buf_lru_free_blocks_error_printed;
/******************************************************************//**
These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O
and page_zip_decompress() operations. Based on the statistics,
@ -408,6 +404,7 @@ got_mutex:
buf_LRU_check_size_of_non_data_objects();
buf_block_t* block;
IF_DBUG(static bool buf_lru_free_blocks_error_printed,);
DBUG_EXECUTE_IF("ib_lru_force_no_free_page",
if (!buf_lru_free_blocks_error_printed) {
n_iterations = 21;
@ -417,9 +414,25 @@ retry:
/* If there is a block in the free list, take it */
if ((block = buf_LRU_get_free_only()) != nullptr) {
got_block:
const ulint LRU_size = UT_LIST_GET_LEN(buf_pool.LRU);
const ulint available = UT_LIST_GET_LEN(buf_pool.free);
const ulint scan_depth = srv_LRU_scan_depth / 2;
ut_ad(LRU_size <= BUF_LRU_MIN_LEN || available >= scan_depth
|| buf_pool.need_LRU_eviction());
if (!have_mutex) {
mysql_mutex_unlock(&buf_pool.mutex);
}
if (UNIV_UNLIKELY(available < scan_depth)
&& LRU_size > BUF_LRU_MIN_LEN) {
mysql_mutex_lock(&buf_pool.flush_list_mutex);
if (!buf_pool.page_cleaner_active()) {
buf_pool.page_cleaner_wakeup(true);
}
mysql_mutex_unlock(&buf_pool.flush_list_mutex);
}
block->page.zip.clear();
return block;
}
@ -445,10 +458,11 @@ got_block:
if ((block = buf_LRU_get_free_only()) != nullptr) {
goto got_block;
}
const bool wake = buf_pool.need_LRU_eviction();
mysql_mutex_unlock(&buf_pool.mutex);
mysql_mutex_lock(&buf_pool.flush_list_mutex);
const auto n_flush = buf_pool.n_flush();
if (!buf_pool.try_LRU_scan) {
if (wake && !buf_pool.page_cleaner_active()) {
buf_pool.page_cleaner_wakeup(true);
}
mysql_mutex_unlock(&buf_pool.flush_list_mutex);
@ -467,9 +481,10 @@ not_found:
MONITOR_INC( MONITOR_LRU_GET_FREE_WAITS );
}
if (n_iterations == 21 && !buf_lru_free_blocks_error_printed
&& srv_buf_pool_old_size == srv_buf_pool_size) {
buf_lru_free_blocks_error_printed = true;
if (n_iterations == 21
&& srv_buf_pool_old_size == srv_buf_pool_size
&& buf_pool.LRU_warned.test_and_set(std::memory_order_acquire)) {
IF_DBUG(buf_lru_free_blocks_error_printed = true,);
mysql_mutex_unlock(&buf_pool.mutex);
ib::warn() << "Difficult to find free blocks in the buffer pool"
" (" << n_iterations << " search iterations)! "

37
storage/innobase/dict/dict0dict.cc
View file

@ -958,12 +958,11 @@ void dict_sys_t::lock_wait(SRW_LOCK_ARGS(const char *file, unsigned line))
if (latch_ex_wait_start.compare_exchange_strong
(old, now, std::memory_order_relaxed, std::memory_order_relaxed))
{
#ifdef UNIV_DEBUG
latch.x_lock(SRW_LOCK_ARGS(file, line));
#else
latch.wr_lock(SRW_LOCK_ARGS(file, line));
#endif
latch_ex_wait_start.store(0, std::memory_order_relaxed);
ut_ad(!latch_readers);
ut_ad(!latch_ex);
ut_d(latch_ex= pthread_self());
return;
}
@ -978,39 +977,33 @@ void dict_sys_t::lock_wait(SRW_LOCK_ARGS(const char *file, unsigned line))
if (waited > threshold / 4)
ib::warn() << "A long wait (" << waited
<< " seconds) was observed for dict_sys.latch";
#ifdef UNIV_DEBUG
latch.x_lock(SRW_LOCK_ARGS(file, line));
#else
latch.wr_lock(SRW_LOCK_ARGS(file, line));
#endif
ut_ad(!latch_readers);
ut_ad(!latch_ex);
ut_d(latch_ex= pthread_self());
}
#ifdef UNIV_PFS_RWLOCK
ATTRIBUTE_NOINLINE void dict_sys_t::unlock()
{
# ifdef UNIV_DEBUG
latch.x_unlock();
# else
ut_ad(latch_ex == pthread_self());
ut_ad(!latch_readers);
ut_d(latch_ex= 0);
latch.wr_unlock();
# endif
}
ATTRIBUTE_NOINLINE void dict_sys_t::freeze(const char *file, unsigned line)
{
# ifdef UNIV_DEBUG
latch.s_lock(file, line);
# else
latch.rd_lock(file, line);
# endif
ut_ad(!latch_ex);
ut_d(latch_readers++);
}
ATTRIBUTE_NOINLINE void dict_sys_t::unfreeze()
{
# ifdef UNIV_DEBUG
latch.s_unlock();
# else
ut_ad(!latch_ex);
ut_ad(latch_readers--);
latch.rd_unlock();
# endif
}
#endif /* UNIV_PFS_RWLOCK */
@ -4550,11 +4543,7 @@ void dict_sys_t::close()
temp_id_hash.free();
unlock();
#ifdef UNIV_DEBUG
latch.free();
#else
latch.destroy();
#endif
mysql_mutex_destroy(&dict_foreign_err_mutex);

12
storage/innobase/fil/fil0fil.cc
View file

@ -341,8 +341,9 @@ static bool fil_node_open_file_low(fil_node_t *node)
ut_ad(!node->is_open());
ut_ad(node->space->is_closing());
mysql_mutex_assert_owner(&fil_system.mutex);
ulint type;
static_assert(((UNIV_ZIP_SIZE_MIN >> 1) << 3) == 4096, "compatibility");
#if defined _WIN32 || defined HAVE_FCNTL_DIRECT
ulint type;
switch (FSP_FLAGS_GET_ZIP_SSIZE(node->space->flags)) {
case 1:
case 2:
@ -351,6 +352,9 @@ static bool fil_node_open_file_low(fil_node_t *node)
default:
type= OS_DATA_FILE;
}
#else
constexpr auto type= OS_DATA_FILE;
#endif
for (;;)
{
@ -1900,9 +1904,10 @@ fil_ibd_create(
mtr.flag_wr_unlock();
log_write_up_to(lsn, true);
ulint type;
static_assert(((UNIV_ZIP_SIZE_MIN >> 1) << 3) == 4096,
"compatibility");
#if defined _WIN32 || defined HAVE_FCNTL_DIRECT
ulint type;
switch (FSP_FLAGS_GET_ZIP_SSIZE(flags)) {
case 1:
case 2:
@ -1911,6 +1916,9 @@ fil_ibd_create(
default:
type = OS_DATA_FILE;
}
#else
constexpr auto type = OS_DATA_FILE;
#endif
file = os_file_create(
innodb_data_file_key, path,

5
storage/innobase/fil/fil0pagecompress.cc
View file

@ -49,11 +49,6 @@ Updated 14/02/2015
#include "buf0lru.h"
#include "ibuf0ibuf.h"
#include "zlib.h"
#ifdef __linux__
#include <linux/fs.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#endif
#include "row0mysql.h"
#include "lz4.h"
#include "lzo/lzo1x.h"

45
storage/innobase/handler/ha_innodb.cc
View file

@ -600,13 +600,7 @@ static PSI_rwlock_info all_innodb_rwlocks[] =
# ifdef BTR_CUR_HASH_ADAPT
{ &btr_search_latch_key, "btr_search_latch", 0 },
# endif
{ &dict_operation_lock_key, "dict_operation_lock",
# ifdef UNIV_DEBUG
PSI_RWLOCK_FLAG_SX
# else
0
# endif
},
{ &dict_operation_lock_key, "dict_operation_lock", 0 },
{ &fil_space_latch_key, "fil_space_latch", 0 },
{ &trx_i_s_cache_lock_key, "trx_i_s_cache_lock", 0 },
{ &trx_purge_latch_key, "trx_purge_latch", 0 },
@ -4009,7 +4003,7 @@ static int innodb_init_params()
data_mysql_default_charset_coll = (ulint) default_charset_info->number;
#ifndef _WIN32
#ifdef HAVE_FCNTL_DIRECT
if (srv_use_atomic_writes && my_may_have_atomic_write) {
/*
Force O_DIRECT on Unixes (on Windows writes are always
@ -4049,9 +4043,7 @@ static int innodb_init_params()
}
#endif
#ifndef _WIN32
ut_ad(srv_file_flush_method <= SRV_O_DIRECT_NO_FSYNC);
#else
#ifdef _WIN32
switch (srv_file_flush_method) {
case SRV_ALL_O_DIRECT_FSYNC + 1 /* "async_unbuffered"="unbuffered" */:
srv_file_flush_method = SRV_ALL_O_DIRECT_FSYNC;
@ -4062,6 +4054,8 @@ static int innodb_init_params()
default:
ut_ad(srv_file_flush_method <= SRV_ALL_O_DIRECT_FSYNC);
}
#else
ut_ad(srv_file_flush_method <= SRV_O_DIRECT_NO_FSYNC);
#endif
innodb_buffer_pool_size_init();
@ -13473,7 +13467,14 @@ int ha_innobase::delete_table(const char *name)
/* FOREIGN KEY constraints cannot exist on partitioned tables. */;
#endif
else
err= lock_table_children(table, trx);
{
dict_sys.freeze(SRW_LOCK_CALL);
for (const dict_foreign_t* f : table->referenced_set)
if (dict_table_t* child= f->foreign_table)
if ((err= lock_table_for_trx(child, trx, LOCK_X)) != DB_SUCCESS)
break;
dict_sys.unfreeze();
}
}
dict_table_t *table_stats= nullptr, *index_stats= nullptr;
@ -13871,7 +13872,14 @@ int ha_innobase::truncate()
dict_table_t *table_stats = nullptr, *index_stats = nullptr;
MDL_ticket *mdl_table = nullptr, *mdl_index = nullptr;
dberr_t error= lock_table_children(ib_table, trx);
dberr_t error= DB_SUCCESS;
dict_sys.freeze(SRW_LOCK_CALL);
for (const dict_foreign_t *f : ib_table->referenced_set)
if (dict_table_t *child= f->foreign_table)
if ((error= lock_table_for_trx(child, trx, LOCK_X)) != DB_SUCCESS)
break;
dict_sys.unfreeze();
if (error == DB_SUCCESS)
error= lock_table_for_trx(ib_table, trx, LOCK_X);
@ -14062,7 +14070,16 @@ ha_innobase::rename_table(
/* There is no need to lock any FOREIGN KEY child tables. */
} else if (dict_table_t *table = dict_table_open_on_name(
norm_from, false, DICT_ERR_IGNORE_FK_NOKEY)) {
error = lock_table_children(table, trx);
dict_sys.freeze(SRW_LOCK_CALL);
for (const dict_foreign_t* f : table->referenced_set) {
if (dict_table_t* child = f->foreign_table) {
error = lock_table_for_trx(child, trx, LOCK_X);
if (error != DB_SUCCESS) {
break;
}
}
}
dict_sys.unfreeze();
if (error == DB_SUCCESS) {
error = lock_table_for_trx(table, trx, LOCK_X);
}

11
storage/innobase/handler/handler0alter.cc
View file

@ -11224,7 +11224,16 @@ ha_innobase::commit_inplace_alter_table(
fts_optimize_remove_table(ctx->old_table);
}
error = lock_table_children(ctx->old_table, trx);
dict_sys.freeze(SRW_LOCK_CALL);
for (auto f : ctx->old_table->referenced_set) {
if (dict_table_t* child = f->foreign_table) {
error = lock_table_for_trx(child, trx, LOCK_X);
if (error != DB_SUCCESS) {
break;
}
}
}
dict_sys.unfreeze();
if (ctx->new_table->fts) {
ut_ad(!ctx->new_table->fts->add_wq);

9
storage/innobase/include/buf0buf.h
View file

@ -1498,10 +1498,8 @@ public:
n_chunks_new / 4 * chunks->size;
}
/** @return whether the buffer pool has run out */
TPOOL_SUPPRESS_TSAN
bool ran_out() const
{ return UNIV_UNLIKELY(!try_LRU_scan || !UT_LIST_GET_LEN(free)); }
/** @return whether the buffer pool is running low */
bool need_LRU_eviction() const;
/** @return whether the buffer pool is shrinking */
inline bool is_shrinking() const
@ -1827,6 +1825,9 @@ public:
Set whenever the free list grows, along with a broadcast of done_free.
Protected by buf_pool.mutex. */
Atomic_relaxed<bool> try_LRU_scan;
/** Whether we have warned to be running out of buffer pool */
std::atomic_flag LRU_warned;
/* @} */
/** @name LRU replacement algorithm fields */

60
storage/innobase/include/dict0dict.h
View file

@ -1313,14 +1313,14 @@ class dict_sys_t
/** The my_hrtime_coarse().val of the oldest lock_wait() start, or 0 */
std::atomic<ulonglong> latch_ex_wait_start;
#ifdef UNIV_DEBUG
typedef index_lock dict_lock;
#else
typedef srw_lock dict_lock;
#endif
/** the rw-latch protecting the data dictionary cache */
alignas(CPU_LEVEL1_DCACHE_LINESIZE) dict_lock latch;
alignas(CPU_LEVEL1_DCACHE_LINESIZE) srw_lock latch;
#ifdef UNIV_DEBUG
/** whether latch is being held in exclusive mode (by any thread) */
Atomic_relaxed<pthread_t> latch_ex;
/** number of S-latch holders */
Atomic_counter<uint32_t> latch_readers;
#endif
public:
/** Indexes of SYS_TABLE[] */
enum
@ -1488,12 +1488,15 @@ public:
}
#ifdef UNIV_DEBUG
/** @return whether the current thread is holding the latch */
bool frozen() const { return latch.have_any(); }
/** @return whether the current thread is holding a shared latch */
bool frozen_not_locked() const { return latch.have_s(); }
/** @return whether any thread (not necessarily the current thread)
is holding the latch; that is, this check may return false
positives */
bool frozen() const { return latch_readers || latch_ex; }
/** @return whether any thread (not necessarily the current thread)
is holding a shared latch */
bool frozen_not_locked() const { return latch_readers; }
/** @return whether the current thread holds the exclusive latch */
bool locked() const { return latch.have_x(); }
bool locked() const { return latch_ex == pthread_self(); }
#endif
private:
/** Acquire the exclusive latch */
@ -1508,12 +1511,13 @@ public:
/** Exclusively lock the dictionary cache. */
void lock(SRW_LOCK_ARGS(const char *file, unsigned line))
{
#ifdef UNIV_DEBUG
ut_ad(!latch.have_any());
if (!latch.x_lock_try())
#else
if (!latch.wr_lock_try())
#endif
if (latch.wr_lock_try())
{
ut_ad(!latch_readers);
ut_ad(!latch_ex);
ut_d(latch_ex= pthread_self());
}
else
lock_wait(SRW_LOCK_ARGS(file, line));
}
@ -1528,30 +1532,24 @@ public:
/** Unlock the data dictionary cache. */
void unlock()
{
# ifdef UNIV_DEBUG
latch.x_unlock();
# else
ut_ad(latch_ex == pthread_self());
ut_ad(!latch_readers);
ut_d(latch_ex= 0);
latch.wr_unlock();
# endif
}
/** Acquire a shared lock on the dictionary cache. */
void freeze()
{
# ifdef UNIV_DEBUG
ut_ad(!latch.have_any());
latch.s_lock();
# else
latch.rd_lock();
# endif
ut_ad(!latch_ex);
ut_d(latch_readers++);
}
/** Release a shared lock on the dictionary cache. */
void unfreeze()
{
# ifdef UNIV_DEBUG
latch.s_unlock();
# else
ut_ad(!latch_ex);
ut_ad(latch_readers--);
latch.rd_unlock();
# endif
}
#endif

7
storage/innobase/include/lock0lock.h
View file

@ -438,13 +438,6 @@ dberr_t lock_table_for_trx(dict_table_t *table, trx_t *trx, lock_mode mode,
bool no_wait= false)
MY_ATTRIBUTE((nonnull, warn_unused_result));
/** Lock the child tables of a table.
@param table parent table
@param trx transaction
@return error code */
dberr_t lock_table_children(dict_table_t *table, trx_t *trx)
MY_ATTRIBUTE((nonnull, warn_unused_result));
/** Exclusively lock the data dictionary tables.
@param trx dictionary transaction
@return error code

10
storage/innobase/include/os0file.h
View file

@ -142,9 +142,11 @@ static const ulint OS_FILE_NORMAL = 62;
/* @} */
/** Types for file create @{ */
static const ulint OS_DATA_FILE = 100;
static const ulint OS_LOG_FILE = 101;
static const ulint OS_DATA_FILE_NO_O_DIRECT = 103;
static constexpr ulint OS_DATA_FILE = 100;
static constexpr ulint OS_LOG_FILE = 101;
#if defined _WIN32 || defined HAVE_FCNTL_DIRECT
static constexpr ulint OS_DATA_FILE_NO_O_DIRECT = 103;
#endif
/* @} */
/** Error codes from os_file_get_last_error @{ */
@ -373,7 +375,7 @@ os_file_create_simple_no_error_handling_func(
bool* success)
MY_ATTRIBUTE((warn_unused_result));
#ifdef _WIN32
#ifndef HAVE_FCNTL_DIRECT
#define os_file_set_nocache(fd, file_name, operation_name) do{}while(0)
#else
/** Tries to disable OS caching on an opened file descriptor.

32
storage/innobase/lock/lock0lock.cc
View file

@ -3940,8 +3940,6 @@ static void lock_table_dequeue(lock_t *in_lock, bool owns_wait_mutex)
dberr_t lock_table_for_trx(dict_table_t *table, trx_t *trx, lock_mode mode,
bool no_wait)
{
ut_ad(!dict_sys.frozen());
mem_heap_t *heap= mem_heap_create(512);
sel_node_t *node= sel_node_create(heap);
que_thr_t *thr= pars_complete_graph_for_exec(node, trx, heap, nullptr);
@ -3978,36 +3976,6 @@ run_again:
return err;
}
/** Lock the child tables of a table.
@param table parent table
@param trx transaction
@return error code */
dberr_t lock_table_children(dict_table_t *table, trx_t *trx)
{
dict_sys.freeze(SRW_LOCK_CALL);
std::vector<dict_table_t*> children;
for (auto f : table->referenced_set)
if (dict_table_t *child= f->foreign_table)
{
child->acquire();
children.emplace_back(child);
}
dict_sys.unfreeze();
dberr_t err= DB_SUCCESS;
for (auto child : children)
if ((err= lock_table_for_trx(child, trx, LOCK_X)) != DB_SUCCESS)
break;
for (auto child : children)
child->release();
return err;
}
/** Exclusively lock the data dictionary tables.
@param trx dictionary transaction
@return error code

15
storage/innobase/log/log0recv.cc
View file

@ -4910,16 +4910,20 @@ uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file)
for (const page_t *page : pages)
{
uint32_t space_id= page_get_space_id(page);
byte *read_page= nullptr;
if (page_get_page_no(page) > 0 || space_id == 0)
{
next_page:
aligned_free(read_page);
continue;
}
uint32_t flags= mach_read_from_4(
FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page);
page_id_t page_id(space_id, 0);
size_t page_size= fil_space_t::physical_size(flags);
if (file_size < 4 * page_size)
goto next_page;
byte *read_page=
read_page=
static_cast<byte*>(aligned_malloc(3 * page_size, page_size));
/* Read 3 pages from the file and match the space id
with the space id which is stored in
@ -4931,7 +4935,10 @@ next_page:
{
byte *cur_page= read_page + j * page_size;
if (buf_is_zeroes(span<const byte>(cur_page, page_size)))
return 0;
{
space_id= 0;
goto early_exit;
}
if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 ||
memcmp(cur_page + FIL_PAGE_SPACE_ID,
page + FIL_PAGE_SPACE_ID, 4) ||
@ -4939,7 +4946,11 @@ next_page:
goto next_page;
}
if (!restore_first_page(space_id, name, file))
{
early_exit:
aligned_free(read_page);
return space_id;
}
break;
}
}

44
storage/innobase/os/os0file.cc
View file

@ -975,7 +975,7 @@ os_file_create_simple_func(
*success = false;
int create_flag;
const char* mode_str = NULL;
const char* mode_str __attribute__((unused));
ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT));
ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT));
@ -1051,6 +1051,7 @@ os_file_create_simple_func(
} while (retry);
#ifdef HAVE_FCNTL_DIRECT
/* This function is always called for data files, we should disable
OS caching (O_DIRECT) here as we do in os_file_create_func(), so
we open the same file in the same mode, see man page of open(2). */
@ -1065,6 +1066,7 @@ os_file_create_simple_func(
break;
}
}
#endif
#ifndef _WIN32
if (!read_only
@ -1150,7 +1152,7 @@ os_file_create_func(
);
int create_flag;
const char* mode_str = NULL;
const char* mode_str __attribute__((unused));
on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT
? true : false;
@ -1192,10 +1194,13 @@ os_file_create_func(
return(OS_FILE_CLOSED);
}
#ifdef HAVE_FCNTL_DIRECT
ut_a(type == OS_LOG_FILE
|| type == OS_DATA_FILE
|| type == OS_DATA_FILE_NO_O_DIRECT);
#else
ut_a(type == OS_LOG_FILE || type == OS_DATA_FILE);
#endif
ut_a(purpose == OS_FILE_AIO || purpose == OS_FILE_NORMAL);
/* We let O_DSYNC only affect log files */
@ -1241,7 +1246,7 @@ os_file_create_func(
return file;
}
#if (defined __sun__ && defined DIRECTIO_ON) || defined O_DIRECT
#ifdef HAVE_FCNTL_DIRECT
if (type == OS_DATA_FILE) {
switch (srv_file_flush_method) {
case SRV_O_DSYNC:
@ -2175,10 +2180,8 @@ os_file_create_func(
if (srv_file_flush_method == SRV_O_DSYNC)
attributes|= FILE_FLAG_WRITE_THROUGH;
}
else if (type == OS_DATA_FILE)
{
switch (srv_file_flush_method)
{
else if (type == OS_DATA_FILE) {
switch (srv_file_flush_method) {
case SRV_FSYNC:
case SRV_LITTLESYNC:
case SRV_NOSYNC:
@ -3042,30 +3045,15 @@ os_file_handle_error_cond_exit(
return(false);
}
#ifndef _WIN32
#ifdef HAVE_FCNTL_DIRECT
/** Tries to disable OS caching on an opened file descriptor.
@param[in] fd file descriptor to alter
@param[in] file_name file name, used in the diagnostic message
@param[in] name "open" or "create"; used in the diagnostic
message */
void
os_file_set_nocache(
int fd MY_ATTRIBUTE((unused)),
const char* file_name MY_ATTRIBUTE((unused)),
const char* operation_name MY_ATTRIBUTE((unused)))
os_file_set_nocache(int fd, const char *file_name, const char *operation_name)
{
/* some versions of Solaris may not have DIRECTIO_ON */
#if defined(__sun__) && defined(DIRECTIO_ON)
if (directio(fd, DIRECTIO_ON) == -1) {
int errno_save = errno;
ib::error()
<< "Failed to set DIRECTIO_ON on file "
<< file_name << "; " << operation_name << ": "
<< strerror(errno_save) << ","
" continuing anyway.";
}
#elif defined(O_DIRECT)
if (fcntl(fd, F_SETFL, O_DIRECT) == -1) {
int errno_save = errno;
static bool warning_message_printed = false;
@ -3084,10 +3072,8 @@ os_file_set_nocache(
<< ", continuing anyway.";
}
}
#endif /* defined(__sun__) && defined(DIRECTIO_ON) */
}
#endif /* _WIN32 */
#endif /* HAVE_FCNTL_DIRECT */
/** Check if the file system supports sparse files.
@param fh file handle
@ -3457,7 +3443,7 @@ static void write_io_callback(void *c)
if (UNIV_UNLIKELY(cb->m_err != 0))
ib::info () << "IO Error: " << cb->m_err
<< "during write of "
<< " during write of "
<< cb->m_len << " bytes, for file "
<< request.node->name << "(" << cb->m_fh << "), returned "
<< cb->m_ret_len;

3
storage/innobase/row/row0merge.cc
View file

@ -4411,13 +4411,14 @@ row_merge_file_create(
merge_file->fd = row_merge_file_create_low(path);
merge_file->offset = 0;
merge_file->n_rec = 0;
#ifdef HAVE_FCNTL_DIRECT
if (merge_file->fd != OS_FILE_CLOSED) {
if (srv_disable_sort_file_cache) {
os_file_set_nocache(merge_file->fd,
"row0merge.cc", "sort");
}
}
#endif
return(merge_file->fd);
}

2
storage/perfschema/CMakeLists.txt
View file

@ -24,7 +24,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include
${CMAKE_SOURCE_DIR}/sql
${CMAKE_BINARY_DIR}/sql
${CMAKE_CURRENT_BINARY_DIR}
${PCRE_INCLUDES}
${PCRE_INCLUDE_DIRS}
${SSL_INCLUDE_DIRS})
ADD_DEFINITIONS(-DMYSQL_SERVER)

2
storage/perfschema/unittest/CMakeLists.txt
View file

@ -22,7 +22,7 @@
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include
${CMAKE_SOURCE_DIR}/include/mysql
${PCRE_INCLUDES}
${PCRE_INCLUDE_DIRS}
${CMAKE_SOURCE_DIR}/sql
${SSL_INCLUDE_DIRS}
${CMAKE_SOURCE_DIR}/unittest/mytap

2
storage/spider/mysql-test/spider/bugfix/t/perfschema.opt
View file

@ -1 +1 @@
--performance-schema
--loose-performance-schema

1
storage/spider/mysql-test/spider/bugfix/t/perfschema.test
View file

@ -1,3 +1,4 @@
source include/have_perfschema.inc;
disable_query_log;
source ../../include/init_spider.inc;
enable_query_log;

2
unittest/embedded/CMakeLists.txt
View file

@ -1,7 +1,7 @@
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include
${CMAKE_SOURCE_DIR}/libmysqld/include
${PCRE_INCLUDES}
${PCRE_INCLUDE_DIRS}
${CMAKE_SOURCE_DIR}/sql
${MY_READLINE_INCLUDE_DIR}
)