Merge 10.6 into 10.11

mysql-test/include/rowid_filter_debug_kill.inc

@@ -55,5 +55,33 @@ disconnect con1;
 reap;
 set debug_sync='RESET';
 
+--echo #
+--echo # MDEV-30651: SIGSEGV in st_join_table::save_explain_data and
+--echo #             Assertion `sel->quick' failed in make_range_rowid_filters
+--echo #
+
+--echo # Reusing table t2 and t3 from previous test
+let $target_id= `select connection_id()`;
+
+set debug_sync='in_forced_range_optimize SIGNAL ready1 WAIT_FOR go1';
+send
+explain
+select * from t2, t3
+where
+  t3.key1=t2.a and t3.key2 in (2,3);
+
+connect (con1, localhost, root,,);
+set debug_sync='now WAIT_FOR ready1';
+evalp kill query $target_id;
+set debug_sync='now SIGNAL go1';
+
+connection default;
+disconnect con1;
+
+--error ER_QUERY_INTERRUPTED
+reap;
+set debug_sync='RESET';
+
+
 drop table t2,t3;
 --source include/wait_until_count_sessions.inc

mysql-test/main/mysql_upgrade-34014.opt

+--character-set-server=utf8mb3
+--collation-server=utf8mb3_unicode_ci

mysql-test/main/mysql_upgrade-34014.result

+#
+# Stat of 10.6 tests
+#
+#
+# MDEV-34014 mysql_upgrade failed
+#
+SHOW CREATE DATABASE sys;
+Database	Create Database
+sys	CREATE DATABASE `sys` /*!40100 DEFAULT CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci */
+# Emulate db.opt file was removed in a mistake
+FLUSH TABLES;
+SHOW CREATE DATABASE sys;
+Database	Create Database
+sys	CREATE DATABASE `sys` /*!40100 DEFAULT CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci */
+Phase 1/8: Checking and upgrading mysql database
+Processing databases
+mysql
+mysql.column_stats                                 OK
+mysql.columns_priv                                 OK
+mysql.db                                           OK
+mysql.event                                        OK
+mysql.func                                         OK
+mysql.global_priv                                  OK
+mysql.gtid_slave_pos                               OK
+mysql.help_category                                OK
+mysql.help_keyword                                 OK
+mysql.help_relation                                OK
+mysql.help_topic                                   OK
+mysql.index_stats                                  OK
+mysql.innodb_index_stats
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+mysql.innodb_table_stats
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+mysql.plugin                                       OK
+mysql.proc                                         OK
+mysql.procs_priv                                   OK
+mysql.proxies_priv                                 OK
+mysql.roles_mapping                                OK
+mysql.servers                                      OK
+mysql.table_stats                                  OK
+mysql.tables_priv                                  OK
+mysql.time_zone                                    OK
+mysql.time_zone_leap_second                        OK
+mysql.time_zone_name                               OK
+mysql.time_zone_transition                         OK
+mysql.time_zone_transition_type                    OK
+mysql.transaction_registry
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+
+Repairing tables
+mysql.innodb_index_stats
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+mysql.innodb_table_stats
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+mysql.transaction_registry
+Error    : Unknown storage engine 'InnoDB'
+error    : Corrupt
+Phase 2/8: Installing used storage engines... Skipped
+Phase 3/8: Running 'mysql_fix_privilege_tables'
+Phase 4/8: Fixing views
+mysql.user                                         OK
+sys.host_summary                                   OK
+sys.host_summary_by_file_io                        OK
+sys.host_summary_by_file_io_type                   OK
+sys.host_summary_by_stages                         OK
+sys.host_summary_by_statement_latency              OK
+sys.host_summary_by_statement_type                 OK
+sys.innodb_buffer_stats_by_schema                  OK
+sys.innodb_buffer_stats_by_table                   OK
+sys.innodb_lock_waits                              OK
+sys.io_by_thread_by_latency                        OK
+sys.io_global_by_file_by_bytes                     OK
+sys.io_global_by_file_by_latency                   OK
+sys.io_global_by_wait_by_bytes                     OK
+sys.io_global_by_wait_by_latency                   OK
+sys.latest_file_io                                 OK
+sys.memory_by_host_by_current_bytes                OK
+sys.memory_by_thread_by_current_bytes              OK
+sys.memory_by_user_by_current_bytes                OK
+sys.memory_global_by_current_bytes                 OK
+sys.memory_global_total                            OK
+sys.metrics                                        OK
+sys.processlist                                    OK
+sys.ps_check_lost_instrumentation                  OK
+sys.schema_auto_increment_columns                  OK
+sys.schema_index_statistics                        OK
+sys.schema_object_overview                         OK
+sys.schema_redundant_indexes                       OK
+sys.schema_table_lock_waits                        OK
+sys.schema_table_statistics                        OK
+sys.schema_table_statistics_with_buffer            OK
+sys.schema_tables_with_full_table_scans            OK
+sys.schema_unused_indexes                          OK
+sys.session                                        OK
+sys.session_ssl_status                             OK
+sys.statement_analysis                             OK
+sys.statements_with_errors_or_warnings             OK
+sys.statements_with_full_table_scans               OK
+sys.statements_with_runtimes_in_95th_percentile    OK
+sys.statements_with_sorting                        OK
+sys.statements_with_temp_tables                    OK
+sys.user_summary                                   OK
+sys.user_summary_by_file_io                        OK
+sys.user_summary_by_file_io_type                   OK
+sys.user_summary_by_stages                         OK
+sys.user_summary_by_statement_latency              OK
+sys.user_summary_by_statement_type                 OK
+sys.version                                        OK
+sys.wait_classes_global_by_avg_latency             OK
+sys.wait_classes_global_by_latency                 OK
+sys.waits_by_host_by_latency                       OK
+sys.waits_by_user_by_latency                       OK
+sys.waits_global_by_latency                        OK
+sys.x$host_summary                                 OK
+sys.x$host_summary_by_file_io                      OK
+sys.x$host_summary_by_file_io_type                 OK
+sys.x$host_summary_by_stages                       OK
+sys.x$host_summary_by_statement_latency            OK
+sys.x$host_summary_by_statement_type               OK
+sys.x$innodb_buffer_stats_by_schema                OK
+sys.x$innodb_buffer_stats_by_table                 OK
+sys.x$innodb_lock_waits                            OK
+sys.x$io_by_thread_by_latency                      OK
+sys.x$io_global_by_file_by_bytes                   OK
+sys.x$io_global_by_file_by_latency                 OK
+sys.x$io_global_by_wait_by_bytes                   OK
+sys.x$io_global_by_wait_by_latency                 OK
+sys.x$latest_file_io                               OK
+sys.x$memory_by_host_by_current_bytes              OK
+sys.x$memory_by_thread_by_current_bytes            OK
+sys.x$memory_by_user_by_current_bytes              OK
+sys.x$memory_global_by_current_bytes               OK
+sys.x$memory_global_total                          OK
+sys.x$processlist                                  OK
+sys.x$ps_digest_95th_percentile_by_avg_us          OK
+sys.x$ps_digest_avg_latency_distribution           OK
+sys.x$ps_schema_table_statistics_io                OK
+sys.x$schema_flattened_keys                        OK
+sys.x$schema_index_statistics                      OK
+sys.x$schema_table_lock_waits                      OK
+sys.x$schema_table_statistics                      OK
+sys.x$schema_table_statistics_with_buffer          OK
+sys.x$schema_tables_with_full_table_scans          OK
+sys.x$session                                      OK
+sys.x$statement_analysis                           OK
+sys.x$statements_with_errors_or_warnings           OK
+sys.x$statements_with_full_table_scans             OK
+sys.x$statements_with_runtimes_in_95th_percentile  OK
+sys.x$statements_with_sorting                      OK
+sys.x$statements_with_temp_tables                  OK
+sys.x$user_summary                                 OK
+sys.x$user_summary_by_file_io                      OK
+sys.x$user_summary_by_file_io_type                 OK
+sys.x$user_summary_by_stages                       OK
+sys.x$user_summary_by_statement_latency            OK
+sys.x$user_summary_by_statement_type               OK
+sys.x$wait_classes_global_by_avg_latency           OK
+sys.x$wait_classes_global_by_latency               OK
+sys.x$waits_by_host_by_latency                     OK
+sys.x$waits_by_user_by_latency                     OK
+sys.x$waits_global_by_latency                      OK
+Phase 5/8: Fixing table and database names
+Phase 6/8: Checking and upgrading tables
+Processing databases
+information_schema
+mtr
+mtr.global_suppressions                            OK
+mtr.test_suppressions                              OK
+performance_schema
+sys
+sys.sys_config                                     OK
+test
+Phase 7/8: uninstalling plugins
+Phase 8/8: Running 'FLUSH PRIVILEGES'
+OK
+SHOW CREATE DATABASE sys;
+Database	Create Database
+sys	CREATE DATABASE `sys` /*!40100 DEFAULT CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci */
+#
+# End of 10.6 tests
+#

mysql-test/main/mysql_upgrade-34014.test

+--source include/mysql_upgrade_preparation.inc
+
+let $MYSQLD_DATADIR= `select @@datadir`;
+
+--echo #
+--echo # Stat of 10.6 tests
+--echo #
+
+--echo #
+--echo # MDEV-34014 mysql_upgrade failed
+--echo #
+
+SHOW CREATE DATABASE sys;
+
+--echo # Emulate db.opt file was removed in a mistake
+--remove_file $MYSQLD_DATADIR/sys/db.opt
+FLUSH TABLES;
+SHOW CREATE DATABASE sys;
+
+--exec $MYSQL_UPGRADE --force 2>&1
+--remove_file $MYSQLD_DATADIR/mysql_upgrade_info
+SHOW CREATE DATABASE sys;
+
+--echo #
+--echo # End of 10.6 tests
+--echo #

mysql-test/main/rowid_filter_innodb_debug.result

@@ -46,5 +46,23 @@ connection default;
 disconnect con1;
 ERROR 70100: Query execution was interrupted
 set debug_sync='RESET';
+#
+# MDEV-30651: SIGSEGV in st_join_table::save_explain_data and
+#             Assertion `sel->quick' failed in make_range_rowid_filters
+#
+# Reusing table t2 and t3 from previous test
+set debug_sync='in_forced_range_optimize SIGNAL ready1 WAIT_FOR go1';
+explain
+select * from t2, t3
+where
+t3.key1=t2.a and t3.key2 in (2,3);
+connect  con1, localhost, root,,;
+set debug_sync='now WAIT_FOR ready1';
+kill query $target_id;
+set debug_sync='now SIGNAL go1';
+connection default;
+disconnect con1;
+ERROR 70100: Query execution was interrupted
+set debug_sync='RESET';
 drop table t2,t3;
 set default_storage_engine=default;

mysql-test/main/rowid_filter_myisam_debug.result

@@ -45,4 +45,22 @@ connection default;
 disconnect con1;
 ERROR 70100: Query execution was interrupted
 set debug_sync='RESET';
+#
+# MDEV-30651: SIGSEGV in st_join_table::save_explain_data and
+#             Assertion `sel->quick' failed in make_range_rowid_filters
+#
+# Reusing table t2 and t3 from previous test
+set debug_sync='in_forced_range_optimize SIGNAL ready1 WAIT_FOR go1';
+explain
+select * from t2, t3
+where
+t3.key1=t2.a and t3.key2 in (2,3);
+connect  con1, localhost, root,,;
+set debug_sync='now WAIT_FOR ready1';
+kill query $target_id;
+set debug_sync='now SIGNAL go1';
+connection default;
+disconnect con1;
+ERROR 70100: Query execution was interrupted
+set debug_sync='RESET';
 drop table t2,t3;

scripts/sys_schema/before_setup.sql

@@ -17,6 +17,11 @@ SET NAMES utf8;
 SET @sql_log_bin = @@sql_log_bin;
 SET sql_log_bin = 0;
 
-CREATE DATABASE IF NOT EXISTS sys DEFAULT CHARACTER SET utf8;
+CREATE DATABASE IF NOT EXISTS sys DEFAULT CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci;
+
+-- If the database had existed, let's recreate its db.opt:
+-- * to fix it if it contained unexpected charset/collation values
+-- * to create it if it was removed in a mistake
+ALTER DATABASE sys CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci;
 
 USE sys;

sql/opt_range.cc

@@ -2722,7 +2722,10 @@ SQL_SELECT::test_quick_select(THD *thd,
     only_single_index_range_scan= 1;
 
   if (head->force_index || force_quick_range)
+  {
+    DEBUG_SYNC(thd, "in_forced_range_optimize");
     scan_time= read_time= DBL_MAX;
+  }
   else
   {
     scan_time= rows2double(records) / TIME_FOR_COMPARE;
@@ -3119,6 +3122,12 @@ SQL_SELECT::test_quick_select(THD *thd,
     free_root(&alloc,MYF(0));			// Return memory & allocator
     thd->mem_root= param.old_root;
     thd->no_errors=0;
+    if (thd->killed || thd->is_error())
+    {
+      delete quick;
+      quick= NULL;
+      returnval= ERROR;
+    }
   }
 
   DBUG_EXECUTE("info", print_quick(quick, &needed_reg););

sql/sql_select.cc

@@ -2014,6 +2014,7 @@ bool JOIN::make_range_rowid_filters()
     tab->table->force_index= force_index_save;
     if (rc == SQL_SELECT::ERROR || thd->is_error())
     {
+      delete sel;
       DBUG_RETURN(true); /* Fatal error */
     }
     /*
@@ -2039,8 +2040,6 @@ bool JOIN::make_range_rowid_filters()
         continue;
     }
   no_filter:
-    if (sel->quick)
-      delete sel->quick;
     delete sel;
   }
 
@@ -2058,7 +2057,9 @@ bool JOIN::make_range_rowid_filters()
     rowid container employed by the filter. On success it lets the table engine
     know that what rowid filter will be used when accessing the table rows.
 
-  @retval false  always
+  @retval
+    false OK
+    true  Error, query should abort
 */
 
 bool

storage/innobase/include/srw_lock.h

@@ -70,10 +70,13 @@ inline void pthread_mutex_wrapper<true>::wr_lock()
 { if (!wr_lock_try()) wr_wait(); }
 # endif
 
+template<bool spinloop> class ssux_lock_impl;
+
 /** Futex-based mutex */
 template<bool spinloop>
 class srw_mutex_impl final
 {
+  friend ssux_lock_impl<spinloop>;
   /** The lock word, containing HOLDER + 1 if the lock is being held,
   plus the number of waiters */
   std::atomic<uint32_t> lock;
@@ -95,6 +98,8 @@ class srw_mutex_impl final
   inline void wait(uint32_t lk);
   /** Wake up one wait() thread */
   void wake();
+  /** Wake up all wait() threads */
+  inline void wake_all();
 public:
   /** @return whether the mutex is being held or waited for */
   bool is_locked_or_waiting() const
@@ -207,27 +212,25 @@ class ssux_lock_impl
   /** @return whether the lock is being held or waited for */
   bool is_vacant() const { return !is_locked_or_waiting(); }
 #endif /* !DBUG_OFF */
-
-  bool rd_lock_try()
+private:
+  /** Try to acquire a shared latch.
+  @return the lock word value if the latch was not acquired
+  @retval 0  if the latch was acquired */
+  uint32_t rd_lock_try_low()
   {
     uint32_t lk= 0;
     while (!readers.compare_exchange_weak(lk, lk + 1,
                                           std::memory_order_acquire,
                                           std::memory_order_relaxed))
       if (lk & WRITER)
-        return false;
-    return true;
+        return lk;
+    return 0;
   }
+public:
 
-  bool u_lock_try()
-  {
-    if (!writer.wr_lock_try())
-      return false;
-    IF_DBUG_ASSERT(uint32_t lk=,)
-    readers.fetch_add(1, std::memory_order_acquire);
-    DBUG_ASSERT(lk < WRITER - 1);
-    return true;
-  }
+  bool rd_lock_try() { return rd_lock_try_low() == 0; }
+
+  bool u_lock_try() { return writer.wr_lock_try(); }
 
   bool wr_lock_try()
   {
@@ -246,9 +249,6 @@ class ssux_lock_impl
   void u_lock()
   {
     writer.wr_lock();
-    IF_DBUG_ASSERT(uint32_t lk=,)
-    readers.fetch_add(1, std::memory_order_acquire);
-    DBUG_ASSERT(lk < WRITER - 1);
   }
   void wr_lock()
   {
@@ -270,15 +270,15 @@ class ssux_lock_impl
   void u_wr_upgrade()
   {
     DBUG_ASSERT(writer.is_locked());
-    uint32_t lk= readers.fetch_add(WRITER - 1, std::memory_order_acquire);
-    if (lk != 1)
-      wr_wait(lk - 1);
+    uint32_t lk= readers.fetch_add(WRITER, std::memory_order_acquire);
+    if (lk)
+      wr_wait(lk);
   }
   void wr_u_downgrade()
   {
     DBUG_ASSERT(writer.is_locked());
     DBUG_ASSERT(is_write_locked());
-    readers.store(1, std::memory_order_release);
+    readers.store(0, std::memory_order_release);
     /* Note: Any pending rd_lock() will not be woken up until u_unlock() */
   }
 
@@ -291,10 +291,6 @@ class ssux_lock_impl
   }
   void u_unlock()
   {
-    IF_DBUG_ASSERT(uint32_t lk=,)
-    readers.fetch_sub(1, std::memory_order_release);
-    DBUG_ASSERT(lk);
-    DBUG_ASSERT(lk < WRITER);
     writer.wr_unlock();
   }
   void wr_unlock()
@@ -404,7 +400,7 @@ typedef srw_spin_lock_low srw_spin_lock;
 class ssux_lock
 {
   PSI_rwlock *pfs_psi;
-  ssux_lock_impl<false> lock;
+  ssux_lock_impl<true> lock;
 
   ATTRIBUTE_NOINLINE void psi_rd_lock(const char *file, unsigned line);
   ATTRIBUTE_NOINLINE void psi_wr_lock(const char *file, unsigned line);

storage/innobase/include/sux_lock.h

@@ -285,7 +285,7 @@ class sux_lock final
 typedef sux_lock<ssux_lock_impl<true>> block_lock;
 
 #ifndef UNIV_PFS_RWLOCK
-typedef sux_lock<ssux_lock_impl<false>> index_lock;
+typedef sux_lock<ssux_lock_impl<true>> index_lock;
 #else
 typedef sux_lock<ssux_lock> index_lock;
 

storage/innobase/sync/srw_lock.cc

@@ -191,6 +191,13 @@ void srw_mutex_impl<spinloop>::wake()
   pthread_mutex_unlock(&mutex);
 }
 template<bool spinloop>
+inline void srw_mutex_impl<spinloop>::wake_all()
+{
+  pthread_mutex_lock(&mutex);
+  pthread_cond_broadcast(&cond);
+  pthread_mutex_unlock(&mutex);
+}
+template<bool spinloop>
 void ssux_lock_impl<spinloop>::wake()
 {
   pthread_mutex_lock(&writer.mutex);
@@ -207,6 +214,8 @@ inline void srw_mutex_impl<spinloop>::wait(uint32_t lk)
 { WaitOnAddress(&lock, &lk, 4, INFINITE); }
 template<bool spinloop>
 void srw_mutex_impl<spinloop>::wake() { WakeByAddressSingle(&lock); }
+template<bool spinloop>
+inline void srw_mutex_impl<spinloop>::wake_all() { WakeByAddressAll(&lock); }
 
 template<bool spinloop>
 inline void ssux_lock_impl<spinloop>::wait(uint32_t lk)
@@ -244,6 +253,8 @@ inline void srw_mutex_impl<spinloop>::wait(uint32_t lk)
 { SRW_FUTEX(&lock, WAIT, lk); }
 template<bool spinloop>
 void srw_mutex_impl<spinloop>::wake() { SRW_FUTEX(&lock, WAKE, 1); }
+template<bool spinloop>
+void srw_mutex_impl<spinloop>::wake_all() { SRW_FUTEX(&lock, WAKE, INT_MAX); }
 
 template<bool spinloop>
 inline void ssux_lock_impl<spinloop>::wait(uint32_t lk)
@@ -304,9 +315,8 @@ void srw_mutex_impl<spinloop>::wait_and_lock()
     for (auto spin= srv_n_spin_wait_rounds;;)
     {
       DBUG_ASSERT(~HOLDER & lk);
-      if (lk & HOLDER)
       lk= lock.load(std::memory_order_relaxed);
-      else
+      if (!(lk & HOLDER))
       {
 #ifdef IF_NOT_FETCH_OR_GOTO
         static_assert(HOLDER == (1U << 31), "compatibility");
@@ -316,10 +326,10 @@ void srw_mutex_impl<spinloop>::wait_and_lock()
         if (!((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) & HOLDER))
           goto acquired;
 #endif
-        srw_pause(delay);
       }
       if (!--spin)
         break;
+      srw_pause(delay);
     }
   }
 
@@ -392,14 +402,52 @@ template void ssux_lock_impl<false>::wr_wait(uint32_t);
 template<bool spinloop>
 void ssux_lock_impl<spinloop>::rd_wait()
 {
+  const unsigned delay= srw_pause_delay();
+
+  if (spinloop)
+  {
+    for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
+    {
+      srw_pause(delay);
+      if (rd_lock_try())
+        return;
+    }
+  }
+
+  /* Subscribe to writer.wake() or write.wake_all() calls by
+  concurrently executing rd_wait() or writer.wr_unlock(). */
+  uint32_t wl= 1 + writer.lock.fetch_add(1, std::memory_order_acquire);
+
   for (;;)
   {
-    writer.wr_lock();
-    bool acquired= rd_lock_try();
-    writer.wr_unlock();
-    if (acquired)
+    if (UNIV_LIKELY(writer.HOLDER & wl))
+      writer.wait(wl);
+    uint32_t lk= rd_lock_try_low();
+    if (!lk)
       break;
+    if (UNIV_UNLIKELY(lk == WRITER)) /* A wr_lock() just succeeded. */
+      /* Immediately wake up (also) wr_lock(). We may also unnecessarily
+      wake up other concurrent threads that are executing rd_wait().
+      If we invoked writer.wake() here to wake up just one thread,
+      we could wake up a rd_wait(), which then would invoke writer.wake(),
+      waking up possibly another rd_wait(), and we could end up doing
+      lots of non-productive context switching until the wr_lock()
+      is finally woken up. */
+      writer.wake_all();
+    srw_pause(delay);
+    wl= writer.lock.load(std::memory_order_acquire);
+    ut_ad(wl);
   }
+
+  /* Unsubscribe writer.wake() and writer.wake_all(). */
+  wl= writer.lock.fetch_sub(1, std::memory_order_release);
+  ut_ad(wl);
+
+  /* Wake any other threads that may be blocked in writer.wait().
+  All other waiters than this rd_wait() would end up acquiring writer.lock
+  and waking up other threads on unlock(). */
+  if (wl > 1)
+    writer.wake_all();
 }
 
 template void ssux_lock_impl<true>::rd_wait();
@@ -462,17 +510,40 @@ template<bool spinloop>
 void srw_lock_impl<spinloop>::psi_wr_lock(const char *file, unsigned line)
 {
   PSI_rwlock_locker_state state;
-  const bool nowait= lock.wr_lock_try();
+# if defined _WIN32 || defined SUX_LOCK_GENERIC
+  const bool nowait2= lock.wr_lock_try();
+# else
+  const bool nowait1= lock.writer.wr_lock_try();
+  uint32_t lk= 0;
+  const bool nowait2= nowait1 &&
+    lock.readers.compare_exchange_strong(lk, lock.WRITER,
+                                         std::memory_order_acquire,
+                                         std::memory_order_relaxed);
+# endif
   if (PSI_rwlock_locker *locker= PSI_RWLOCK_CALL(start_rwlock_wrwait)
       (&state, pfs_psi,
-       nowait ? PSI_RWLOCK_TRYWRITELOCK : PSI_RWLOCK_WRITELOCK, file, line))
+       nowait2 ? PSI_RWLOCK_TRYWRITELOCK : PSI_RWLOCK_WRITELOCK, file, line))
   {
-    if (!nowait)
+# if defined _WIN32 || defined SUX_LOCK_GENERIC
+    if (!nowait2)
       lock.wr_lock();
+# else
+    if (!nowait1)
+      lock.wr_lock();
+    else if (!nowait2)
+      lock.u_wr_upgrade();
+# endif
     PSI_RWLOCK_CALL(end_rwlock_rdwait)(locker, 0);
   }
-  else if (!nowait)
+# if defined _WIN32 || defined SUX_LOCK_GENERIC
+  else if (!nowait2)
+    lock.wr_lock();
+# else
+  else if (!nowait1)
     lock.wr_lock();
+  else if (!nowait2)
+    lock.u_wr_upgrade();
+# endif
 }
 
 void ssux_lock::psi_rd_lock(const char *file, unsigned line)
@@ -507,25 +578,48 @@ void ssux_lock::psi_u_lock(const char *file, unsigned line)
 void ssux_lock::psi_wr_lock(const char *file, unsigned line)
 {
   PSI_rwlock_locker_state state;
-  const bool nowait= lock.wr_lock_try();
+# if defined _WIN32 || defined SUX_LOCK_GENERIC
+  const bool nowait2= lock.wr_lock_try();
+# else
+  const bool nowait1= lock.writer.wr_lock_try();
+  uint32_t lk= 0;
+  const bool nowait2= nowait1 &&
+    lock.readers.compare_exchange_strong(lk, lock.WRITER,
+                                         std::memory_order_acquire,
+                                         std::memory_order_relaxed);
+# endif
   if (PSI_rwlock_locker *locker= PSI_RWLOCK_CALL(start_rwlock_wrwait)
       (&state, pfs_psi,
-       nowait ? PSI_RWLOCK_TRYEXCLUSIVELOCK : PSI_RWLOCK_EXCLUSIVELOCK,
+       nowait2 ? PSI_RWLOCK_TRYEXCLUSIVELOCK : PSI_RWLOCK_EXCLUSIVELOCK,
        file, line))
   {
-    if (!nowait)
+# if defined _WIN32 || defined SUX_LOCK_GENERIC
+    if (!nowait2)
+      lock.wr_lock();
+# else
+    if (!nowait1)
       lock.wr_lock();
+    else if (!nowait2)
+      lock.u_wr_upgrade();
+# endif
     PSI_RWLOCK_CALL(end_rwlock_rdwait)(locker, 0);
   }
-  else if (!nowait)
+# if defined _WIN32 || defined SUX_LOCK_GENERIC
+  else if (!nowait2)
     lock.wr_lock();
+# else
+  else if (!nowait1)
+    lock.wr_lock();
+  else if (!nowait2)
+    lock.u_wr_upgrade();
+# endif
 }
 
 void ssux_lock::psi_u_wr_upgrade(const char *file, unsigned line)
 {
   PSI_rwlock_locker_state state;
   DBUG_ASSERT(lock.writer.is_locked());
-  uint32_t lk= 1;
+  uint32_t lk= 0;
   const bool nowait=
     lock.readers.compare_exchange_strong(lk, ssux_lock_impl<false>::WRITER,
                                          std::memory_order_acquire,