Skip to content

M
MariaDB
Commit c5819d72 authored by Sergey Vojtovich's avatar Sergey Vojtovich
Browse files
Options

MDEV-13728 - Import MySQL 5.7 atomic operations for MSVC and Solaris 

gcc_sync.h, solaris.h, generic-msvc.h copied verbatim.
parent c178bde9
Hide whitespace changes
Inline Side-by-side
Showing with 304 additions and 261 deletions
include/atomic/gcc_builtins.h
View file @ c5819d72
#ifndef ATOMIC_GCC_BUILTINS_INCLUDED
#define ATOMIC_GCC_BUILTINS_INCLUDED
/* Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
/* Copyright (c) 2017 MariaDB Foundation
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
......@@ -16,8 +16,6 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
#if defined(HAVE_GCC_C11_ATOMICS)
#define MY_ATOMIC_MODE "gcc-atomics-smp"
#define MY_MEMORY_ORDER_RELAXED __ATOMIC_RELAXED
#define MY_MEMORY_ORDER_CONSUME __ATOMIC_CONSUME
......@@ -76,21 +74,5 @@
__atomic_compare_exchange_n((P), (E), (D), 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
#define my_atomic_casptr(P, E, D) \
__atomic_compare_exchange_n((P), (E), (D), 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
#else
#define MY_ATOMIC_MODE "gcc-builtins-smp"
#define make_atomic_load_body(S) \
ret= __sync_fetch_and_or(a, 0);
#define make_atomic_store_body(S) \
(void) __sync_lock_test_and_set(a, v);
#define make_atomic_add_body(S) \
v= __sync_fetch_and_add(a, v);
#define make_atomic_fas_body(S) \
v= __sync_lock_test_and_set(a, v);
#define make_atomic_cas_body(S) \
int ## S sav; \
int ## S cmp_val= *cmp; \
sav= __sync_val_compare_and_swap(a, cmp_val, set);\
if (!(ret= (sav == cmp_val))) *cmp= sav
#endif
#endif /* ATOMIC_GCC_BUILTINS_INCLUDED */
include/atomic/gcc_sync.h 0 → 100644
View file @ c5819d72
#ifndef GCC_SYNC_INCLUDED
#define GCC_SYNC_INCLUDED
/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
/* Old GCC __sync builtins introduced in GCC 4.1 */
static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set)
{
int32 cmp_val= *cmp;
int32 sav= __sync_val_compare_and_swap(a, cmp_val, set);
int ret= (sav == cmp_val);
if (!ret)
*cmp = sav;
return ret;
}
static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set)
{
int64 cmp_val= *cmp;
int64 sav= __sync_val_compare_and_swap(a, cmp_val, set);
int ret= (sav == cmp_val);
if (!ret)
*cmp = sav;
return ret;
}
static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set)
{
void *cmp_val= *cmp;
void *sav= __sync_val_compare_and_swap(a, cmp_val, set);
int ret= (sav == cmp_val);
if (!ret)
*cmp = sav;
return ret;
}
static inline int32 my_atomic_add32(int32 volatile *a, int32 v)
{
return __sync_fetch_and_add(a, v);
}
static inline int64 my_atomic_add64(int64 volatile *a, int64 v)
{
return __sync_fetch_and_add(a, v);
}
static inline int32 my_atomic_fas32(int32 volatile *a, int32 v)
{
return __sync_lock_test_and_set(a, v);
}
static inline int64 my_atomic_fas64(int64 volatile *a, int64 v)
{
return __sync_lock_test_and_set(a, v);
}
static inline void * my_atomic_fasptr(void * volatile *a, void * v)
{
return __sync_lock_test_and_set(a, v);
}
static inline int32 my_atomic_load32(int32 volatile *a)
{
return __sync_fetch_and_or(a, 0);
}
static inline int64 my_atomic_load64(int64 volatile *a)
{
return __sync_fetch_and_or(a, 0);
}
static inline void* my_atomic_loadptr(void * volatile *a)
{
return __sync_fetch_and_or(a, 0);
}
static inline void my_atomic_store32(int32 volatile *a, int32 v)
{
(void) __sync_lock_test_and_set(a, v);
}
static inline void my_atomic_store64(int64 volatile *a, int64 v)
{
(void) __sync_lock_test_and_set(a, v);
}
static inline void my_atomic_storeptr(void * volatile *a, void *v)
{
(void) __sync_lock_test_and_set(a, v);
}
#endif /* GCC_SYNC_INCLUDED */
include/atomic/generic-msvc.h
View file @ c5819d72
/* Copyright (c) 2006-2008 MySQL AB, 2009 Sun Microsystems, Inc.
Use is subject to license terms.
#ifndef ATOMIC_MSC_INCLUDED
#define ATOMIC_MSC_INCLUDED
/* Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
......@@ -14,79 +16,101 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
#ifndef _atomic_h_cleanup_
#define _atomic_h_cleanup_ "atomic/generic-msvc.h"
#include <windows.h>
/*
x86 compilers (both VS2003 or VS2005) never use instrinsics, but generate
function calls to kernel32 instead, even in the optimized build.
We force intrinsics as described in MSDN documentation for
_InterlockedCompareExchange.
*/
#ifdef _M_IX86
#if (_MSC_VER >= 1500)
#include <intrin.h>
#else
C_MODE_START
/*Visual Studio 2003 and earlier do not have prototypes for atomic intrinsics*/
LONG _InterlockedCompareExchange (LONG volatile *Target, LONG Value, LONG Comp);
LONGLONG _InterlockedCompareExchange64 (LONGLONG volatile *Target,
LONGLONG Value, LONGLONG Comp);
C_MODE_END
#pragma intrinsic(_InterlockedCompareExchange)
#pragma intrinsic(_InterlockedCompareExchange64)
#endif
#define InterlockedCompareExchange _InterlockedCompareExchange
#define InterlockedCompareExchange64 _InterlockedCompareExchange64
/*
No need to do something special for InterlockedCompareExchangePointer
as it is a #define to InterlockedCompareExchange. The same applies to
InterlockedExchangePointer.
*/
#endif /*_M_IX86*/
#define MY_ATOMIC_MODE "msvc-intrinsics"
/* Implement using CAS on WIN32 */
#define IL_COMP_EXCHG32(X,Y,Z) \
InterlockedCompareExchange((volatile LONG *)(X),(Y),(Z))
#define IL_COMP_EXCHG64(X,Y,Z) \
InterlockedCompareExchange64((volatile LONGLONG *)(X), \
(LONGLONG)(Y),(LONGLONG)(Z))
#define IL_COMP_EXCHGptr InterlockedCompareExchangePointer
#define make_atomic_cas_body(S) \
int ## S initial_cmp= *cmp; \
int ## S initial_a= IL_COMP_EXCHG ## S (a, set, initial_cmp); \
if (!(ret= (initial_a == initial_cmp))) *cmp= initial_a;
#ifndef _M_IX86
/* Use full set of optimised functions on WIN64 */
#define IL_EXCHG_ADD32(X,Y) \
InterlockedExchangeAdd((volatile LONG *)(X),(Y))
#define IL_EXCHG_ADD64(X,Y) \
InterlockedExchangeAdd64((volatile LONGLONG *)(X),(LONGLONG)(Y))
#define IL_EXCHG32(X,Y) \
InterlockedExchange((volatile LONG *)(X),(Y))
#define IL_EXCHG64(X,Y) \
InterlockedExchange64((volatile LONGLONG *)(X),(LONGLONG)(Y))
#define IL_EXCHGptr InterlockedExchangePointer
#define make_atomic_add_body(S) \
v= IL_EXCHG_ADD ## S (a, v)
#define make_atomic_swap_body(S) \
v= IL_EXCHG ## S (a, v)
#define make_atomic_load_body(S) \
ret= 0; /* avoid compiler warning */ \
ret= IL_COMP_EXCHG ## S (a, ret, ret);
#endif
static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set)
{
int32 initial_cmp= *cmp;
int32 initial_a= InterlockedCompareExchange((volatile LONG*)a,
set, initial_cmp);
int ret= (initial_a == initial_cmp);
if (!ret)
*cmp= initial_a;
return ret;
}
static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set)
{
int64 initial_cmp= *cmp;
int64 initial_a= InterlockedCompareExchange64((volatile LONGLONG*)a,
(LONGLONG)set,
(LONGLONG)initial_cmp);
int ret= (initial_a == initial_cmp);
if (!ret)
*cmp= initial_a;
return ret;
}
static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set)
{
void *initial_cmp= *cmp;
void *initial_a= InterlockedCompareExchangePointer(a, set, initial_cmp);
int ret= (initial_a == initial_cmp);
if (!ret)
*cmp= initial_a;
return ret;
}
static inline int32 my_atomic_add32(int32 volatile *a, int32 v)
{
return (int32)InterlockedExchangeAdd((volatile LONG*)a, v);
}
static inline int64 my_atomic_add64(int64 volatile *a, int64 v)
{
return (int64)InterlockedExchangeAdd64((volatile LONGLONG*)a, (LONGLONG)v);
}
static inline int32 my_atomic_load32(int32 volatile *a)
{
return (int32)InterlockedCompareExchange((volatile LONG *)a, 0, 0);
}
static inline int64 my_atomic_load64(int64 volatile *a)
{
return (int64)InterlockedCompareExchange64((volatile LONGLONG *)a, 0, 0);
}
static inline void* my_atomic_loadptr(void * volatile *a)
{
return InterlockedCompareExchangePointer(a, 0, 0);
}
static inline int32 my_atomic_fas32(int32 volatile *a, int32 v)
{
return (int32)InterlockedExchange((volatile LONG*)a, v);
}
static inline int64 my_atomic_fas64(int64 volatile *a, int64 v)
{
return (int64)InterlockedExchange64((volatile LONGLONG*)a, v);
}
static inline void * my_atomic_fasptr(void * volatile *a, void * v)
{
return InterlockedExchangePointer(a, v);
}
static inline void my_atomic_store32(int32 volatile *a, int32 v)
{
(void)InterlockedExchange((volatile LONG*)a, v);
}
static inline void my_atomic_store64(int64 volatile *a, int64 v)
{
(void)InterlockedExchange64((volatile LONGLONG*)a, v);
}
static inline void my_atomic_storeptr(void * volatile *a, void *v)
{
(void)InterlockedExchangePointer(a, v);
}
/*
my_yield_processor (equivalent of x86 PAUSE instruction) should be used
to improve performance on hyperthreaded CPUs. Intel recommends to use it in
spin loops also on non-HT machines to reduce power consumption (see e.g
spin loops also on non-HT machines to reduce power consumption (see e.g
http://softwarecommunity.intel.com/articles/eng/2004.htm)
Running benchmarks for spinlocks implemented with InterlockedCompareExchange
......@@ -94,35 +118,18 @@ C_MODE_END
YieldProcessor in a loop - that is, yielding longer. On Intel boxes setting
loop count in the range 200-300 brought best results.
*/
#ifndef YIELD_LOOPS
#define YIELD_LOOPS 200
#endif
static __inline int my_yield_processor()
static inline int my_yield_processor()
{
int i;
for(i=0; i<YIELD_LOOPS; i++)
for (i=0; i<YIELD_LOOPS; i++)
{
#if (_MSC_VER <= 1310)
/* On older compilers YieldProcessor is not available, use inline assembly*/
__asm { rep nop }
#else
YieldProcessor();
#endif
}
return 1;
}
#define LF_BACKOFF my_yield_processor()
#else /* cleanup */
#undef IL_EXCHG_ADD32
#undef IL_EXCHG_ADD64
#undef IL_COMP_EXCHG32
#undef IL_COMP_EXCHG64
#undef IL_COMP_EXCHGptr
#undef IL_EXCHG32
#undef IL_EXCHG64
#undef IL_EXCHGptr
#endif
#endif /* ATOMIC_MSC_INCLUDED */
include/atomic/solaris.h
View file @ c5819d72
/* Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
#ifndef ATOMIC_SOLARIS_INCLUDED
#define ATOMIC_SOLARIS_INCLUDED
/* Copyright (c) 2008, 2014, Oracle and/or its affiliates. All rights reserved.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
......@@ -13,51 +16,102 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
#ifndef _atomic_h_cleanup_
#define _atomic_h_cleanup_ "atomic/solaris.h"
#include <atomic.h>
#define MY_ATOMIC_MODE "solaris-atomic"
#if defined(__GNUC__)
#define atomic_typeof(T,V) __typeof__(V)
#else
#define atomic_typeof(T,V) T
#endif
#define uintptr_t void *
#define atomic_or_ptr_nv(X,Y) (void *)atomic_or_ulong_nv((volatile ulong_t *)X, Y)
#define make_atomic_cas_body(S) \
atomic_typeof(uint ## S ## _t, *cmp) sav; \
sav = atomic_cas_ ## S( \
(volatile uint ## S ## _t *)a, \
(uint ## S ## _t)*cmp, \
(uint ## S ## _t)set); \
if (! (ret= (sav == *cmp))) \
static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set)
{
int ret;
atomic_typeof(uint32_t, *cmp) sav;
sav= atomic_cas_32((volatile uint32_t *)a, (uint32_t)*cmp, (uint32_t)set);
ret= (sav == *cmp);
if (!ret)
*cmp= sav;
#define make_atomic_add_body(S) \
int ## S nv; /* new value */ \
nv= atomic_add_ ## S ## _nv((volatile uint ## S ## _t *)a, v); \
v= nv - v
/* ------------------------------------------------------------------------ */
#define make_atomic_load_body(S) \
ret= atomic_or_ ## S ## _nv((volatile uint ## S ## _t *)a, 0)
#define make_atomic_store_body(S) \
(void) atomic_swap_ ## S((volatile uint ## S ## _t *)a, (uint ## S ## _t)v)
#define make_atomic_fas_body(S) \
v= atomic_swap_ ## S((volatile uint ## S ## _t *)a, (uint ## S ## _t)v)
#else /* cleanup */
#undef uintptr_t
#undef atomic_or_ptr_nv
#endif
return ret;
}
static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set)
{
int ret;
atomic_typeof(uint64_t, *cmp) sav;
sav= atomic_cas_64((volatile uint64_t *)a, (uint64_t)*cmp, (uint64_t)set);
ret= (sav == *cmp);
if (!ret)
*cmp= sav;
return ret;
}
static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set)
{
int ret;
atomic_typeof(void *, *cmp) sav;
sav= atomic_cas_ptr((volatile void **)a, (void *)*cmp, (void *)set);
ret= (sav == *cmp);
if (!ret)
*cmp= sav;
return ret;
}
static inline int32 my_atomic_add32(int32 volatile *a, int32 v)
{
int32 nv= atomic_add_32_nv((volatile uint32_t *)a, v);
return nv - v;
}
static inline int64 my_atomic_add64(int64 volatile *a, int64 v)
{
int64 nv= atomic_add_64_nv((volatile uint64_t *)a, v);
return nv - v;
}
static inline int32 my_atomic_fas32(int32 volatile *a, int32 v)
{
return atomic_swap_32((volatile uint32_t *)a, (uint32_t)v);
}
static inline int64 my_atomic_fas64(int64 volatile *a, int64 v)
{
return atomic_swap_64((volatile uint64_t *)a, (uint64_t)v);
}
static inline void * my_atomic_fasptr(void * volatile *a, void * v)
{
return atomic_swap_ptr(a, v);
}
static inline int32 my_atomic_load32(int32 volatile *a)
{
return atomic_or_32_nv((volatile uint32_t *)a, 0);
}
static inline int64 my_atomic_load64(int64 volatile *a)
{
return atomic_or_64_nv((volatile uint64_t *)a, 0);
}
static inline void* my_atomic_loadptr(void * volatile *a)
{
return atomic_add_ptr_nv(a, 0);
}
static inline void my_atomic_store32(int32 volatile *a, int32 v)
{
(void) atomic_swap_32((volatile uint32_t *)a, (uint32_t)v);
}
static inline void my_atomic_store64(int64 volatile *a, int64 v)
{
(void) atomic_swap_64((volatile uint64_t *)a, (uint64_t)v);
}
static inline void my_atomic_storeptr(void * volatile *a, void *v)
{
(void) atomic_swap_ptr((volatile void **)a, (void *)v);
}
#endif /* ATOMIC_SOLARIS_INCLUDED */
include/my_atomic.h
View file @ c5819d72
......@@ -110,119 +110,13 @@
#include "atomic/generic-msvc.h"
#elif defined(HAVE_SOLARIS_ATOMIC)
#include "atomic/solaris.h"
#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_C11_ATOMICS)
#elif defined(HAVE_GCC_C11_ATOMICS)
#include "atomic/gcc_builtins.h"
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
#include "atomic/gcc_sync.h"
#endif
#ifndef HAVE_GCC_C11_ATOMICS
#ifndef make_atomic_cas_body
/* nolock.h was not able to generate even a CAS function, fall back */
#error atomic ops for this platform are not implemented
#endif
#define intptr void *
/* define missing functions by using the already generated ones */
#ifndef make_atomic_add_body
#define make_atomic_add_body(S) \
int ## S tmp=*a; \
while (!my_atomic_cas ## S(a, &tmp, tmp+v)) ; \
v=tmp;
#endif
#ifndef make_atomic_fas_body
#define make_atomic_fas_body(S) \
int ## S tmp=*a; \
while (!my_atomic_cas ## S(a, &tmp, v)) ; \
v=tmp;
#endif
#ifndef make_atomic_load_body
#define make_atomic_load_body(S) \
ret= 0; /* avoid compiler warning */ \
(void)(my_atomic_cas ## S(a, &ret, ret));
#endif
#ifndef make_atomic_store_body
#define make_atomic_store_body(S) \
(void)(my_atomic_fas ## S (a, v));
#endif
#define make_atomic_cas(S) \
static inline int my_atomic_cas ## S(int ## S volatile *a, \
int ## S *cmp, int ## S set) \
{ \
int8 ret; \
make_atomic_cas_body(S); \
return ret; \
}
#define make_atomic_add(S) \
static inline int ## S my_atomic_add ## S( \
int ## S volatile *a, int ## S v) \
{ \
make_atomic_add_body(S); \
return v; \
}
#define make_atomic_fas(S) \
static inline int ## S my_atomic_fas ## S( \
int ## S volatile *a, int ## S v) \
{ \
make_atomic_fas_body(S); \
return v; \
}
#define make_atomic_load(S) \
static inline int ## S my_atomic_load ## S(int ## S volatile *a)\
{ \
int ## S ret; \
make_atomic_load_body(S); \
return ret; \
}
#define make_atomic_store(S) \
static inline void my_atomic_store ## S( \
int ## S volatile *a, int ## S v) \
{ \
make_atomic_store_body(S); \
}
make_atomic_cas(32)
make_atomic_cas(64)
make_atomic_cas(ptr)
make_atomic_add(32)
make_atomic_add(64)
make_atomic_load(32)
make_atomic_load(64)
make_atomic_load(ptr)
make_atomic_fas(32)
make_atomic_fas(64)
make_atomic_fas(ptr)
make_atomic_store(32)
make_atomic_store(64)
make_atomic_store(ptr)
#ifdef _atomic_h_cleanup_
#include _atomic_h_cleanup_
#undef _atomic_h_cleanup_
#endif
#undef make_atomic_add
#undef make_atomic_cas
#undef make_atomic_load
#undef make_atomic_store
#undef make_atomic_fas
#undef make_atomic_add_body
#undef make_atomic_cas_body
#undef make_atomic_load_body
#undef make_atomic_store_body
#undef make_atomic_fas_body
#undef intptr
#endif
/*
the macro below defines (as an expression) the code that
will be run in spin-loops. Intel manuals recummend to have PAUSE there.
......
unittest/mysys/thr_template.c
View file @ c5819d72
......@@ -67,7 +67,7 @@ int main(int argc __attribute__((unused)), char **argv)
#define CYCLES 3000
#define THREADS 30
diag("N CPUs: %d, atomic ops: %s", my_getncpus(), MY_ATOMIC_MODE);
diag("N CPUs: %d", my_getncpus());
do_tests();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7