mariadb/storage/innobase/sync/srw_lock.cc

/*****************************************************************************

Copyright (c) 2020, 2022, MariaDB Corporation.

This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.

This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA

*****************************************************************************/

#include "srw_lock.h"
#include "srv0srv.h"
#include "my_cpu.h"
#include "transactional_lock_guard.h"

#ifdef NO_ELISION
#elif defined _MSC_VER && (defined _M_IX86 || defined _M_X64)
# include <intrin.h>
bool have_transactional_memory;
bool transactional_lock_enabled() noexcept
{
  int regs[4];
  __cpuid(regs, 0);
  if (regs[0] < 7)
    return false;
  __cpuidex(regs, 7, 0);
  /* Restricted Transactional Memory (RTM) */
  have_transactional_memory= regs[1] & 1U << 11;
  return have_transactional_memory;
}
#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__)
# include <cpuid.h>
bool have_transactional_memory;
bool transactional_lock_enabled() noexcept
{
  if (__get_cpuid_max(0, nullptr) < 7)
    return false;
  unsigned eax, ebx, ecx, edx;
  __cpuid_count(7, 0, eax, ebx, ecx, edx);
  /* Restricted Transactional Memory (RTM) */
  have_transactional_memory= ebx & 1U << 11;
  return have_transactional_memory;
}

# ifdef UNIV_DEBUG
TRANSACTIONAL_TARGET
bool xtest() noexcept { return have_transactional_memory && _xtest(); }
# endif
#elif defined __powerpc64__ || defined __s390__
# include <htmxlintrin.h>
# include <setjmp.h>
# include <signal.h>

__attribute__((target("htm"),hot))
bool xbegin() noexcept
{
  return have_transactional_memory &&
    __TM_simple_begin() == _HTM_TBEGIN_STARTED;
}

__attribute__((target("htm"),hot))
void xabort() noexcept { __TM_abort(); }

__attribute__((target("htm"),hot))
void xend() noexcept { __TM_end(); }

bool have_transactional_memory;
static sigjmp_buf ill_jmp;
static void ill_handler(int sig) noexcept
{
  siglongjmp(ill_jmp, sig);
}
/**
  Here we are testing we can do a transaction without SIGILL
  and a 1 instruction store can succeed.
*/
__attribute__((noinline))
static void test_tm(bool *r) noexcept
{
  if (__TM_simple_begin() == _HTM_TBEGIN_STARTED)
  {
    *r= true;
    __TM_end();
  }
}
bool transactional_lock_enabled() noexcept
{
  bool r= false;
  sigset_t oset;
  struct sigaction ill_act, oact_ill;

  memset(&ill_act, 0, sizeof(ill_act));
  ill_act.sa_handler = ill_handler;
  sigfillset(&ill_act.sa_mask);
  sigdelset(&ill_act.sa_mask, SIGILL);

  sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset);
  sigaction(SIGILL, &ill_act, &oact_ill);
  if (sigsetjmp(ill_jmp, 1) == 0)
  {
    test_tm(&r);
  }
  sigaction(SIGILL, &oact_ill, NULL);
  sigprocmask(SIG_SETMASK, &oset, NULL);
  return r;
}

# ifdef UNIV_DEBUG
__attribute__((target("htm"),hot))
bool xtest() noexcept
{
# ifdef __s390x__
  return have_transactional_memory &&
    __builtin_tx_nesting_depth() > 0;
# else
  return have_transactional_memory &&
    _HTM_STATE (__builtin_ttest ()) == _HTM_TRANSACTIONAL;
# endif
}
# endif
#endif

/** @return the parameter for srw_pause() */
static inline unsigned srw_pause_delay() noexcept
{
  return my_cpu_relax_multiplier / 4 * srv_spin_wait_delay;
}

/** Pause the CPU for some time, with no memory accesses. */
static inline void srw_pause(unsigned delay) noexcept
{
  HMT_low();
  while (delay--)
    MY_RELAX_CPU();
  HMT_medium();
}

#ifndef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP
template<> void pthread_mutex_wrapper<true>::wr_wait() noexcept
{
  const unsigned delay= srw_pause_delay();

  for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
  {
    srw_pause(delay);
    if (wr_lock_try())
      return;
  }

  pthread_mutex_lock(&lock);
}
#endif

#ifdef SUX_LOCK_GENERIC
template void ssux_lock_impl<false>::init() noexcept;
template void ssux_lock_impl<true>::init() noexcept;
template void ssux_lock_impl<false>::destroy() noexcept;
template void ssux_lock_impl<true>::destroy() noexcept;

template<bool spinloop>
inline void srw_mutex_impl<spinloop>::wait(uint32_t lk) noexcept
{
  pthread_mutex_lock(&mutex);
  while (lock.load(std::memory_order_relaxed) == lk)
    pthread_cond_wait(&cond, &mutex);
  pthread_mutex_unlock(&mutex);
}

template<bool spinloop>
inline void ssux_lock_impl<spinloop>::wait(uint32_t lk) noexcept
{
  pthread_mutex_lock(&writer.mutex);
  while (readers.load(std::memory_order_relaxed) == lk)
    pthread_cond_wait(&readers_cond, &writer.mutex);
  pthread_mutex_unlock(&writer.mutex);
}

template<bool spinloop>
void srw_mutex_impl<spinloop>::wake() noexcept
{
  pthread_mutex_lock(&mutex);
  pthread_cond_signal(&cond);
  pthread_mutex_unlock(&mutex);
}
template<bool spinloop>
inline void srw_mutex_impl<spinloop>::wake_all() noexcept
{
  pthread_mutex_lock(&mutex);
  pthread_cond_broadcast(&cond);
  pthread_mutex_unlock(&mutex);
}
template<bool spinloop>
void ssux_lock_impl<spinloop>::wake() noexcept
{
  pthread_mutex_lock(&writer.mutex);
  pthread_cond_signal(&readers_cond);
  pthread_mutex_unlock(&writer.mutex);
}
#else
static_assert(4 == sizeof(rw_lock), "ABI");
# ifdef _WIN32
#  include <synchapi.h>

template<bool spinloop>
inline void srw_mutex_impl<spinloop>::wait(uint32_t lk) noexcept
{ WaitOnAddress(&lock, &lk, 4, INFINITE); }
template<bool spinloop>
void srw_mutex_impl<spinloop>::wake() noexcept { WakeByAddressSingle(&lock); }
template<bool spinloop>
inline void srw_mutex_impl<spinloop>::wake_all() noexcept { WakeByAddressAll(&lock); }

template<bool spinloop>
inline void ssux_lock_impl<spinloop>::wait(uint32_t lk) noexcept
{ WaitOnAddress(&readers, &lk, 4, INFINITE); }
template<bool spinloop>
void ssux_lock_impl<spinloop>::wake() noexcept { WakeByAddressSingle(&readers); }
# else
#  ifdef __linux__
#   include <linux/futex.h>
#   include <sys/syscall.h>
#   define SRW_FUTEX(a,op,n) \
    syscall(SYS_futex, a, FUTEX_ ## op ## _PRIVATE, n, nullptr, nullptr, 0)
#  elif defined __OpenBSD__
#   include <sys/time.h>
#   include <sys/futex.h>
#   define SRW_FUTEX(a,op,n) \
    futex((volatile uint32_t*) a, FUTEX_ ## op, n, nullptr, nullptr)
#  elif defined __FreeBSD__
#   include <sys/types.h>
#   include <sys/umtx.h>
#   define FUTEX_WAKE UMTX_OP_WAKE_PRIVATE
#   define FUTEX_WAIT UMTX_OP_WAIT_UINT_PRIVATE
#   define SRW_FUTEX(a,op,n) _umtx_op(a, FUTEX_ ## op, n, nullptr, nullptr)
#  elif defined __DragonFly__
#   include <unistd.h>
#   define FUTEX_WAKE(a,n) umtx_wakeup(a,n)
#   define FUTEX_WAIT(a,n) umtx_sleep(a,n,0)
#   define SRW_FUTEX(a,op,n) FUTEX_ ## op((volatile int*) a, int(n))
#  else
#   error "no futex support"
#  endif

template<bool spinloop>
inline void srw_mutex_impl<spinloop>::wait(uint32_t lk) noexcept
{ SRW_FUTEX(&lock, WAIT, lk); }
template<bool spinloop>
void srw_mutex_impl<spinloop>::wake() noexcept { SRW_FUTEX(&lock, WAKE, 1); }
template<bool spinloop>
void srw_mutex_impl<spinloop>::wake_all() noexcept { SRW_FUTEX(&lock, WAKE, INT_MAX); }

template<bool spinloop>
inline void ssux_lock_impl<spinloop>::wait(uint32_t lk) noexcept
{ SRW_FUTEX(&readers, WAIT, lk); }
template<bool spinloop>
void ssux_lock_impl<spinloop>::wake() noexcept { SRW_FUTEX(&readers, WAKE, 1); }
# endif
#endif

template void srw_mutex_impl<false>::wake() noexcept;
template void ssux_lock_impl<false>::wake() noexcept;
template void srw_mutex_impl<true>::wake() noexcept;
template void ssux_lock_impl<true>::wake() noexcept;

template<bool spinloop>
void srw_mutex_impl<spinloop>::wait_and_lock() noexcept
{
  uint32_t lk= WAITER + lock.fetch_add(WAITER, std::memory_order_relaxed);

  if (spinloop)
  {
    const unsigned delay= srw_pause_delay();

    for (auto spin= srv_n_spin_wait_rounds;;)
    {
      DBUG_ASSERT(~HOLDER & lk);
      lk= lock.load(std::memory_order_relaxed);
      if (!(lk & HOLDER))
      {
#if defined __i386__||defined __x86_64__||defined _M_IX86||defined _M_X64
        lk |= HOLDER;
# ifdef _MSC_VER
        static_assert(HOLDER == (1U << 0), "compatibility");
        if (!_interlockedbittestandset
            (reinterpret_cast<volatile long*>(&lock), 0))
# else
        if (!(lock.fetch_or(HOLDER, std::memory_order_relaxed) & HOLDER))
# endif
          goto acquired;
#else
        if (!((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) & HOLDER))
          goto acquired;
#endif
      }
      if (!--spin)
        break;
      srw_pause(delay);
    }
  }

  for (;;)
  {
    DBUG_ASSERT(~HOLDER & lk);
    if (lk & HOLDER)
    {
      wait(lk);
#if defined __i386__||defined __x86_64__||defined _M_IX86||defined _M_X64
reload:
#endif
      lk= lock.load(std::memory_order_relaxed);
    }
    else
    {
#if defined __i386__||defined __x86_64__||defined _M_IX86||defined _M_X64
# ifdef _MSC_VER
      static_assert(HOLDER == (1U << 0), "compatibility");
      if (_interlockedbittestandset
          (reinterpret_cast<volatile long*>(&lock), 0))
# else
      if (lock.fetch_or(HOLDER, std::memory_order_relaxed) & HOLDER)
# endif
        goto reload;
#else
      if ((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) & HOLDER)
        continue;
      DBUG_ASSERT(lk);
#endif
acquired:
      std::atomic_thread_fence(std::memory_order_acquire);
      return;
    }
  }
}

template void srw_mutex_impl<false>::wait_and_lock() noexcept;
template void srw_mutex_impl<true>::wait_and_lock() noexcept;

template<bool spinloop>
void ssux_lock_impl<spinloop>::wr_wait(uint32_t lk) noexcept
{
  DBUG_ASSERT(writer.is_locked());
  DBUG_ASSERT(lk);
  DBUG_ASSERT(lk < WRITER);

  if (spinloop)
  {
    const unsigned delay= srw_pause_delay();

    for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
    {
      srw_pause(delay);
      lk= readers.load(std::memory_order_acquire);
      if (lk == WRITER)
        return;
      DBUG_ASSERT(lk > WRITER);
    }
  }

  lk|= WRITER;

  do
  {
    DBUG_ASSERT(lk > WRITER);
    wait(lk);
    lk= readers.load(std::memory_order_acquire);
  }
  while (lk != WRITER);
}

template void ssux_lock_impl<true>::wr_wait(uint32_t) noexcept;
template void ssux_lock_impl<false>::wr_wait(uint32_t) noexcept;

template<bool spinloop>
void ssux_lock_impl<spinloop>::rd_wait() noexcept
{
  const unsigned delay= srw_pause_delay();

  if (spinloop)
  {
    for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
    {
      srw_pause(delay);
      if (rd_lock_try())
        return;
    }
  }

  /* Subscribe to writer.wake() or write.wake_all() calls by
  concurrently executing rd_wait() or writer.wr_unlock(). */
  uint32_t wl= writer.WAITER +
    writer.lock.fetch_add(writer.WAITER, std::memory_order_acquire);

  for (;;)
  {
    if (UNIV_LIKELY(writer.HOLDER & wl))
      writer.wait(wl);
    uint32_t lk= rd_lock_try_low();
    if (!lk)
      break;
    if (UNIV_UNLIKELY(lk == WRITER)) /* A wr_lock() just succeeded. */
      /* Immediately wake up (also) wr_lock(). We may also unnecessarily
      wake up other concurrent threads that are executing rd_wait().
      If we invoked writer.wake() here to wake up just one thread,
      we could wake up a rd_wait(), which then would invoke writer.wake(),
      waking up possibly another rd_wait(), and we could end up doing
      lots of non-productive context switching until the wr_lock()
      is finally woken up. */
      writer.wake_all();
    srw_pause(delay);
    wl= writer.lock.load(std::memory_order_acquire);
    ut_ad(wl);
  }

  /* Unsubscribe writer.wake() and writer.wake_all(). */
  wl= writer.lock.fetch_sub(writer.WAITER, std::memory_order_release);
  ut_ad(wl);

  /* Wake any other threads that may be blocked in writer.wait().
  All other waiters than this rd_wait() would end up acquiring writer.lock
  and waking up other threads on unlock(). */
  if (wl > writer.WAITER)
    writer.wake_all();
}

template void ssux_lock_impl<true>::rd_wait() noexcept;
template void ssux_lock_impl<false>::rd_wait() noexcept;

#if defined _WIN32 || defined SUX_LOCK_GENERIC
template<> void srw_lock_<true>::rd_wait() noexcept
{
  const unsigned delay= srw_pause_delay();

  for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
  {
    srw_pause(delay);
    if (rd_lock_try())
      return;
  }

  IF_WIN(AcquireSRWLockShared(&lk), rw_rdlock(&lk));
}

template<> void srw_lock_<true>::wr_wait() noexcept
{
  const unsigned delay= srw_pause_delay();

  for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
  {
    srw_pause(delay);
    if (wr_lock_try())
      return;
  }

  IF_WIN(AcquireSRWLockExclusive(&lk), rw_wrlock(&lk));
}
#endif

#ifdef UNIV_PFS_RWLOCK
template void srw_lock_impl<false>::psi_rd_lock(const char*, unsigned) noexcept;
template void srw_lock_impl<false>::psi_wr_lock(const char*, unsigned) noexcept;
template void srw_lock_impl<true>::psi_rd_lock(const char*, unsigned) noexcept;
template void srw_lock_impl<true>::psi_wr_lock(const char*, unsigned) noexcept;

template<bool spinloop>
void srw_lock_impl<spinloop>::psi_rd_lock(const char *file, unsigned line) noexcept
{
  PSI_rwlock_locker_state state;
  const bool nowait= lock.rd_lock_try();
  if (PSI_rwlock_locker *locker= PSI_RWLOCK_CALL(start_rwlock_rdwait)
      (&state, pfs_psi,
       nowait ? PSI_RWLOCK_TRYREADLOCK : PSI_RWLOCK_READLOCK, file, line))
  {
    if (!nowait)
      lock.rd_lock();
    PSI_RWLOCK_CALL(end_rwlock_rdwait)(locker, 0);
  }
  else if (!nowait)
    lock.rd_lock();
}

template<bool spinloop>
void srw_lock_impl<spinloop>::psi_wr_lock(const char *file, unsigned line) noexcept
{
  PSI_rwlock_locker_state state;
# if defined _WIN32 || defined SUX_LOCK_GENERIC
  const bool nowait2= lock.wr_lock_try();
# else
  const bool nowait1= lock.writer.wr_lock_try();
  uint32_t lk= 0;
  const bool nowait2= nowait1 &&
    lock.readers.compare_exchange_strong(lk, lock.WRITER,
                                         std::memory_order_acquire,
                                         std::memory_order_relaxed);
# endif
  if (PSI_rwlock_locker *locker= PSI_RWLOCK_CALL(start_rwlock_wrwait)
      (&state, pfs_psi,
       nowait2 ? PSI_RWLOCK_TRYWRITELOCK : PSI_RWLOCK_WRITELOCK, file, line))
  {
# if defined _WIN32 || defined SUX_LOCK_GENERIC
    if (!nowait2)
      lock.wr_lock();
# else
    if (!nowait1)
      lock.wr_lock();
    else if (!nowait2)
      lock.u_wr_upgrade();
# endif
    PSI_RWLOCK_CALL(end_rwlock_rdwait)(locker, 0);
  }
# if defined _WIN32 || defined SUX_LOCK_GENERIC
  else if (!nowait2)
    lock.wr_lock();
# else
  else if (!nowait1)
    lock.wr_lock();
  else if (!nowait2)
    lock.u_wr_upgrade();
# endif
}

void ssux_lock::psi_rd_lock(const char *file, unsigned line) noexcept
{
  PSI_rwlock_locker_state state;
  const bool nowait= lock.rd_lock_try();
  if (PSI_rwlock_locker *locker= PSI_RWLOCK_CALL(start_rwlock_rdwait)
      (&state, pfs_psi,
       nowait ? PSI_RWLOCK_TRYSHAREDLOCK : PSI_RWLOCK_SHAREDLOCK, file, line))
  {
    if (!nowait)
      lock.rd_lock();
    PSI_RWLOCK_CALL(end_rwlock_rdwait)(locker, 0);
  }
  else if (!nowait)
    lock.rd_lock();
}

void ssux_lock::psi_u_lock(const char *file, unsigned line) noexcept
{
  PSI_rwlock_locker_state state;
  if (PSI_rwlock_locker *locker= PSI_RWLOCK_CALL(start_rwlock_wrwait)
      (&state, pfs_psi, PSI_RWLOCK_SHAREDEXCLUSIVELOCK, file, line))
  {
    lock.u_lock();
    PSI_RWLOCK_CALL(end_rwlock_rdwait)(locker, 0);
  }
  else
    lock.u_lock();
}

void ssux_lock::psi_wr_lock(const char *file, unsigned line) noexcept
{
  PSI_rwlock_locker_state state;
# if defined _WIN32 || defined SUX_LOCK_GENERIC
  const bool nowait2= lock.wr_lock_try();
# else
  const bool nowait1= lock.writer.wr_lock_try();
  uint32_t lk= 0;
  const bool nowait2= nowait1 &&
    lock.readers.compare_exchange_strong(lk, lock.WRITER,
                                         std::memory_order_acquire,
                                         std::memory_order_relaxed);
# endif
  if (PSI_rwlock_locker *locker= PSI_RWLOCK_CALL(start_rwlock_wrwait)
      (&state, pfs_psi,
       nowait2 ? PSI_RWLOCK_TRYEXCLUSIVELOCK : PSI_RWLOCK_EXCLUSIVELOCK,
       file, line))
  {
# if defined _WIN32 || defined SUX_LOCK_GENERIC
    if (!nowait2)
      lock.wr_lock();
# else
    if (!nowait1)
      lock.wr_lock();
    else if (!nowait2)
      lock.u_wr_upgrade();
# endif
    PSI_RWLOCK_CALL(end_rwlock_rdwait)(locker, 0);
  }
# if defined _WIN32 || defined SUX_LOCK_GENERIC
  else if (!nowait2)
    lock.wr_lock();
# else
  else if (!nowait1)
    lock.wr_lock();
  else if (!nowait2)
    lock.u_wr_upgrade();
# endif
}

void ssux_lock::psi_u_wr_upgrade(const char *file, unsigned line) noexcept
{
  PSI_rwlock_locker_state state;
  DBUG_ASSERT(lock.writer.is_locked());
  uint32_t lk= 0;
  const bool nowait=
    lock.readers.compare_exchange_strong(lk, ssux_lock_impl<false>::WRITER,
                                         std::memory_order_acquire,
                                         std::memory_order_relaxed);
  if (PSI_rwlock_locker *locker= PSI_RWLOCK_CALL(start_rwlock_wrwait)
      (&state, pfs_psi,
       nowait ? PSI_RWLOCK_TRYEXCLUSIVELOCK : PSI_RWLOCK_EXCLUSIVELOCK,
       file, line))
  {
    if (!nowait)
      lock.u_wr_upgrade();
    PSI_RWLOCK_CALL(end_rwlock_rdwait)(locker, 0);
  }
  else if (!nowait)
    lock.u_wr_upgrade();
}
#else /* UNIV_PFS_RWLOCK */
template void ssux_lock_impl<false>::rd_lock() noexcept;
template void ssux_lock_impl<false>::rd_unlock() noexcept;
template void ssux_lock_impl<false>::u_unlock() noexcept;
template void ssux_lock_impl<false>::wr_unlock() noexcept;
#endif /* UNIV_PFS_RWLOCK */

#ifdef UNIV_DEBUG
void srw_lock_debug::SRW_LOCK_INIT(mysql_pfs_key_t key) noexcept
{
  srw_lock::SRW_LOCK_INIT(key);
  readers_lock.init();
  ut_ad(!readers.load(std::memory_order_relaxed));
  ut_ad(!have_any());
}

void srw_lock_debug::destroy() noexcept
{
  ut_ad(!writer);
  if (auto r= readers.load(std::memory_order_relaxed))
  {
    readers.store(0, std::memory_order_relaxed);
    ut_ad(r->empty());
    delete r;
  }
  readers_lock.destroy();
  srw_lock::destroy();
}

bool srw_lock_debug::wr_lock_try() noexcept
{
  ut_ad(!have_any());
  if (!srw_lock::wr_lock_try())
    return false;
  ut_ad(!writer);
  writer.store(pthread_self(), std::memory_order_relaxed);
  return true;
}

void srw_lock_debug::wr_lock(SRW_LOCK_ARGS(const char *file, unsigned line)) noexcept
{
  ut_ad(!have_any());
  srw_lock::wr_lock(SRW_LOCK_ARGS(file, line));
  ut_ad(!writer);
  writer.store(pthread_self(), std::memory_order_relaxed);
}

void srw_lock_debug::wr_unlock() noexcept
{
  ut_ad(have_wr());
  writer.store(0, std::memory_order_relaxed);
  srw_lock::wr_unlock();
}

void srw_lock_debug::readers_register() noexcept
{
  readers_lock.wr_lock();
  auto r= readers.load(std::memory_order_relaxed);
  if (!r)
  {
    r= new std::unordered_multiset<pthread_t>();
    readers.store(r, std::memory_order_relaxed);
  }
  r->emplace(pthread_self());
  readers_lock.wr_unlock();
}

bool srw_lock_debug::rd_lock_try() noexcept
{
  ut_ad(!have_any());
  if (!srw_lock::rd_lock_try())
    return false;
  readers_register();
  return true;
}

void srw_lock_debug::rd_lock(SRW_LOCK_ARGS(const char *file, unsigned line)) noexcept
{
  ut_ad(!have_any());
  srw_lock::rd_lock(SRW_LOCK_ARGS(file, line));
  readers_register();
}

void srw_lock_debug::rd_unlock() noexcept
{
  const pthread_t self= pthread_self();
  ut_ad(writer != self);
  readers_lock.wr_lock();
  auto r= readers.load(std::memory_order_relaxed);
  ut_ad(r);
  auto i= r->find(self);
  ut_ad(i != r->end());
  r->erase(i);
  readers_lock.wr_unlock();

  srw_lock::rd_unlock();
}

bool srw_lock_debug::have_rd() const noexcept
{
  if (auto r= readers.load(std::memory_order_relaxed))
  {
    readers_lock.wr_lock();
    bool found= r->find(pthread_self()) != r->end();
    readers_lock.wr_unlock();
# ifndef SUX_LOCK_GENERIC
    ut_ad(!found || is_locked());
# endif
    return found;
  }
  return false;
}

bool srw_lock_debug::have_wr() const noexcept
{
  if (writer != pthread_self())
    return false;
# ifndef SUX_LOCK_GENERIC
  ut_ad(is_write_locked());
# endif
  return true;
}

bool srw_lock_debug::have_any() const noexcept
{
  return have_wr() || have_rd();
}
#endif