Skip to content

L
linux
Commit 76d4acf2 authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Options

Merge tag 'perf-kprobes-2020-12-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull perf/kprobes updates from Thomas Gleixner:
 "Make kretprobes lockless to avoid the rp->lock performance and
  potential lock ordering issues"

* tag 'perf-kprobes-2020-12-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  locking/atomics: Regenerate the atomics-check SHA1's
  kprobes: Replace rp->free_instance with freelist
  freelist: Implement lockless freelist
  asm-generic/atomic: Add try_cmpxchg() fallbacks
  kprobes: Remove kretprobe hash
  llist: Add nonatomic __llist_add() and __llist_dell_all()
parents 8a8ca83e a70a04b3
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 646 additions and 350 deletions
arch/x86/include/asm/atomic.h
View file @ 76d4acf2
...@@ -199,7 +199,7 @@ static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) ...@@ -199,7 +199,7 @@ static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
{ {
return try_cmpxchg(&v->counter, old, new); return arch_try_cmpxchg(&v->counter, old, new);
} }
#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
......
arch/x86/include/asm/atomic64_64.h
View file @ 76d4acf2
...@@ -187,7 +187,7 @@ static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) ...@@ -187,7 +187,7 @@ static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
{ {
return try_cmpxchg(&v->counter, old, new); return arch_try_cmpxchg(&v->counter, old, new);
} }
#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
......
arch/x86/include/asm/cmpxchg.h
View file @ 76d4acf2
...@@ -221,7 +221,7 @@ extern void __add_wrong_size(void) ...@@ -221,7 +221,7 @@ extern void __add_wrong_size(void)
#define __try_cmpxchg(ptr, pold, new, size) \ #define __try_cmpxchg(ptr, pold, new, size) \
__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX) __raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)
#define try_cmpxchg(ptr, pold, new) \ #define arch_try_cmpxchg(ptr, pold, new) \
__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr))) __try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))
/* /*
......
include/asm-generic/atomic-instrumented.h
View file @ 76d4acf2
...@@ -1642,148 +1642,192 @@ atomic64_dec_if_positive(atomic64_t *v) ...@@ -1642,148 +1642,192 @@ atomic64_dec_if_positive(atomic64_t *v)
#endif #endif
#if !defined(arch_xchg_relaxed) || defined(arch_xchg) #if !defined(arch_xchg_relaxed) || defined(arch_xchg)
#define xchg(ptr, ...) \ #define xchg(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_xchg(__ai_ptr, __VA_ARGS__); \ arch_xchg(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if defined(arch_xchg_acquire) #if defined(arch_xchg_acquire)
#define xchg_acquire(ptr, ...) \ #define xchg_acquire(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if defined(arch_xchg_release) #if defined(arch_xchg_release)
#define xchg_release(ptr, ...) \ #define xchg_release(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_xchg_release(__ai_ptr, __VA_ARGS__); \ arch_xchg_release(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if defined(arch_xchg_relaxed) #if defined(arch_xchg_relaxed)
#define xchg_relaxed(ptr, ...) \ #define xchg_relaxed(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) #if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg)
#define cmpxchg(ptr, ...) \ #define cmpxchg(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if defined(arch_cmpxchg_acquire) #if defined(arch_cmpxchg_acquire)
#define cmpxchg_acquire(ptr, ...) \ #define cmpxchg_acquire(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if defined(arch_cmpxchg_release) #if defined(arch_cmpxchg_release)
#define cmpxchg_release(ptr, ...) \ #define cmpxchg_release(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if defined(arch_cmpxchg_relaxed) #if defined(arch_cmpxchg_relaxed)
#define cmpxchg_relaxed(ptr, ...) \ #define cmpxchg_relaxed(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64) #if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64)
#define cmpxchg64(ptr, ...) \ #define cmpxchg64(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if defined(arch_cmpxchg64_acquire) #if defined(arch_cmpxchg64_acquire)
#define cmpxchg64_acquire(ptr, ...) \ #define cmpxchg64_acquire(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if defined(arch_cmpxchg64_release) #if defined(arch_cmpxchg64_release)
#define cmpxchg64_release(ptr, ...) \ #define cmpxchg64_release(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#if defined(arch_cmpxchg64_relaxed) #if defined(arch_cmpxchg64_relaxed)
#define cmpxchg64_relaxed(ptr, ...) \ #define cmpxchg64_relaxed(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
}) })
#endif #endif
#define cmpxchg_local(ptr, ...) \ #if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg)
({ \ #define try_cmpxchg(ptr, oldp, ...) \
typeof(ptr) __ai_ptr = (ptr); \ ({ \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ typeof(ptr) __ai_ptr = (ptr); \
arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#endif
#if defined(arch_try_cmpxchg_acquire)
#define try_cmpxchg_acquire(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#endif
#if defined(arch_try_cmpxchg_release)
#define try_cmpxchg_release(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#endif
#if defined(arch_try_cmpxchg_relaxed)
#define try_cmpxchg_relaxed(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#endif
#define cmpxchg_local(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
}) })
#define cmpxchg64_local(ptr, ...) \ #define cmpxchg64_local(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
}) })
#define sync_cmpxchg(ptr, ...) \ #define sync_cmpxchg(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
}) })
#define cmpxchg_double(ptr, ...) \ #define cmpxchg_double(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
}) })
#define cmpxchg_double_local(ptr, ...) \ #define cmpxchg_double_local(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
}) })
#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ #endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */
// 9d5e6a315fb1335d02f0ccd3655a91c3dafcc63e // 4bec382e44520f4d8267e42620054db26a659ea3
include/linux/atomic-arch-fallback.h
View file @ 76d4acf2
...@@ -9,9 +9,9 @@ ...@@ -9,9 +9,9 @@
#include <linux/compiler.h> #include <linux/compiler.h>
#ifndef arch_xchg_relaxed #ifndef arch_xchg_relaxed
#define arch_xchg_relaxed arch_xchg #define arch_xchg_acquire arch_xchg
#define arch_xchg_acquire arch_xchg #define arch_xchg_release arch_xchg
#define arch_xchg_release arch_xchg #define arch_xchg_relaxed arch_xchg
#else /* arch_xchg_relaxed */ #else /* arch_xchg_relaxed */
#ifndef arch_xchg_acquire #ifndef arch_xchg_acquire
...@@ -32,9 +32,9 @@ ...@@ -32,9 +32,9 @@
#endif /* arch_xchg_relaxed */ #endif /* arch_xchg_relaxed */
#ifndef arch_cmpxchg_relaxed #ifndef arch_cmpxchg_relaxed
#define arch_cmpxchg_relaxed arch_cmpxchg #define arch_cmpxchg_acquire arch_cmpxchg
#define arch_cmpxchg_acquire arch_cmpxchg #define arch_cmpxchg_release arch_cmpxchg
#define arch_cmpxchg_release arch_cmpxchg #define arch_cmpxchg_relaxed arch_cmpxchg
#else /* arch_cmpxchg_relaxed */ #else /* arch_cmpxchg_relaxed */
#ifndef arch_cmpxchg_acquire #ifndef arch_cmpxchg_acquire
...@@ -55,9 +55,9 @@ ...@@ -55,9 +55,9 @@
#endif /* arch_cmpxchg_relaxed */ #endif /* arch_cmpxchg_relaxed */
#ifndef arch_cmpxchg64_relaxed #ifndef arch_cmpxchg64_relaxed
#define arch_cmpxchg64_relaxed arch_cmpxchg64 #define arch_cmpxchg64_acquire arch_cmpxchg64
#define arch_cmpxchg64_acquire arch_cmpxchg64 #define arch_cmpxchg64_release arch_cmpxchg64
#define arch_cmpxchg64_release arch_cmpxchg64 #define arch_cmpxchg64_relaxed arch_cmpxchg64
#else /* arch_cmpxchg64_relaxed */ #else /* arch_cmpxchg64_relaxed */
#ifndef arch_cmpxchg64_acquire #ifndef arch_cmpxchg64_acquire
...@@ -77,6 +77,76 @@ ...@@ -77,6 +77,76 @@
#endif /* arch_cmpxchg64_relaxed */ #endif /* arch_cmpxchg64_relaxed */
#ifndef arch_try_cmpxchg_relaxed
#ifdef arch_try_cmpxchg
#define arch_try_cmpxchg_acquire arch_try_cmpxchg
#define arch_try_cmpxchg_release arch_try_cmpxchg
#define arch_try_cmpxchg_relaxed arch_try_cmpxchg
#endif /* arch_try_cmpxchg */
#ifndef arch_try_cmpxchg
#define arch_try_cmpxchg(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg */
#ifndef arch_try_cmpxchg_acquire
#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg_acquire */
#ifndef arch_try_cmpxchg_release
#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg_release */
#ifndef arch_try_cmpxchg_relaxed
#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg_relaxed */
#else /* arch_try_cmpxchg_relaxed */
#ifndef arch_try_cmpxchg_acquire
#define arch_try_cmpxchg_acquire(...) \
__atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
#endif
#ifndef arch_try_cmpxchg_release
#define arch_try_cmpxchg_release(...) \
__atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
#endif
#ifndef arch_try_cmpxchg
#define arch_try_cmpxchg(...) \
__atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
#endif
#endif /* arch_try_cmpxchg_relaxed */
#ifndef arch_atomic_read_acquire #ifndef arch_atomic_read_acquire
static __always_inline int static __always_inline int
arch_atomic_read_acquire(const atomic_t *v) arch_atomic_read_acquire(const atomic_t *v)
...@@ -2288,4 +2358,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) ...@@ -2288,4 +2358,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v)
#endif #endif
#endif /* _LINUX_ATOMIC_FALLBACK_H */ #endif /* _LINUX_ATOMIC_FALLBACK_H */
// 90cd26cfd69d2250303d654955a0cc12620fb91b // cca554917d7ea73d5e3e7397dd70c484cad9b2c4
include/linux/atomic-fallback.h
View file @ 76d4acf2
...@@ -9,9 +9,9 @@ ...@@ -9,9 +9,9 @@
#include <linux/compiler.h> #include <linux/compiler.h>
#ifndef xchg_relaxed #ifndef xchg_relaxed
#define xchg_relaxed xchg #define xchg_acquire xchg
#define xchg_acquire xchg #define xchg_release xchg
#define xchg_release xchg #define xchg_relaxed xchg
#else /* xchg_relaxed */ #else /* xchg_relaxed */
#ifndef xchg_acquire #ifndef xchg_acquire
...@@ -32,9 +32,9 @@ ...@@ -32,9 +32,9 @@
#endif /* xchg_relaxed */ #endif /* xchg_relaxed */
#ifndef cmpxchg_relaxed #ifndef cmpxchg_relaxed
#define cmpxchg_relaxed cmpxchg #define cmpxchg_acquire cmpxchg
#define cmpxchg_acquire cmpxchg #define cmpxchg_release cmpxchg
#define cmpxchg_release cmpxchg #define cmpxchg_relaxed cmpxchg
#else /* cmpxchg_relaxed */ #else /* cmpxchg_relaxed */
#ifndef cmpxchg_acquire #ifndef cmpxchg_acquire
...@@ -55,9 +55,9 @@ ...@@ -55,9 +55,9 @@
#endif /* cmpxchg_relaxed */ #endif /* cmpxchg_relaxed */
#ifndef cmpxchg64_relaxed #ifndef cmpxchg64_relaxed
#define cmpxchg64_relaxed cmpxchg64 #define cmpxchg64_acquire cmpxchg64
#define cmpxchg64_acquire cmpxchg64 #define cmpxchg64_release cmpxchg64
#define cmpxchg64_release cmpxchg64 #define cmpxchg64_relaxed cmpxchg64
#else /* cmpxchg64_relaxed */ #else /* cmpxchg64_relaxed */
#ifndef cmpxchg64_acquire #ifndef cmpxchg64_acquire
...@@ -77,6 +77,76 @@ ...@@ -77,6 +77,76 @@
#endif /* cmpxchg64_relaxed */ #endif /* cmpxchg64_relaxed */
#ifndef try_cmpxchg_relaxed
#ifdef try_cmpxchg
#define try_cmpxchg_acquire try_cmpxchg
#define try_cmpxchg_release try_cmpxchg
#define try_cmpxchg_relaxed try_cmpxchg
#endif /* try_cmpxchg */
#ifndef try_cmpxchg
#define try_cmpxchg(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = cmpxchg((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* try_cmpxchg */
#ifndef try_cmpxchg_acquire
#define try_cmpxchg_acquire(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = cmpxchg_acquire((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* try_cmpxchg_acquire */
#ifndef try_cmpxchg_release
#define try_cmpxchg_release(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = cmpxchg_release((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* try_cmpxchg_release */
#ifndef try_cmpxchg_relaxed
#define try_cmpxchg_relaxed(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = cmpxchg_relaxed((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* try_cmpxchg_relaxed */
#else /* try_cmpxchg_relaxed */
#ifndef try_cmpxchg_acquire
#define try_cmpxchg_acquire(...) \
__atomic_op_acquire(try_cmpxchg, __VA_ARGS__)
#endif
#ifndef try_cmpxchg_release
#define try_cmpxchg_release(...) \
__atomic_op_release(try_cmpxchg, __VA_ARGS__)
#endif
#ifndef try_cmpxchg
#define try_cmpxchg(...) \
__atomic_op_fence(try_cmpxchg, __VA_ARGS__)
#endif
#endif /* try_cmpxchg_relaxed */
#define arch_atomic_read atomic_read #define arch_atomic_read atomic_read
#define arch_atomic_read_acquire atomic_read_acquire #define arch_atomic_read_acquire atomic_read_acquire
...@@ -2522,4 +2592,4 @@ atomic64_dec_if_positive(atomic64_t *v) ...@@ -2522,4 +2592,4 @@ atomic64_dec_if_positive(atomic64_t *v)
#endif #endif
#endif /* _LINUX_ATOMIC_FALLBACK_H */ #endif /* _LINUX_ATOMIC_FALLBACK_H */
// 9d95b56f98d82a2a26c7b79ccdd0c47572d50a6f // d78e6c293c661c15188f0ec05bce45188c8d5892
include/linux/freelist.h 0 → 100644
View file @ 76d4acf2
/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
#ifndef FREELIST_H
#define FREELIST_H
#include <linux/atomic.h>
/*
* Copyright: cameron@moodycamel.com
*
* A simple CAS-based lock-free free list. Not the fastest thing in the world
* under heavy contention, but simple and correct (assuming nodes are never
* freed until after the free list is destroyed), and fairly speedy under low
* contention.
*
* Adapted from: https://moodycamel.com/blog/2014/solving-the-aba-problem-for-lock-free-free-lists
*/
struct freelist_node {
atomic_t refs;
struct freelist_node *next;
};
struct freelist_head {
struct freelist_node *head;
};
#define REFS_ON_FREELIST 0x80000000
#define REFS_MASK 0x7FFFFFFF
static inline void __freelist_add(struct freelist_node *node, struct freelist_head *list)
{
/*
* Since the refcount is zero, and nobody can increase it once it's
* zero (except us, and we run only one copy of this method per node at
* a time, i.e. the single thread case), then we know we can safely
* change the next pointer of the node; however, once the refcount is
* back above zero, then other threads could increase it (happens under
* heavy contention, when the refcount goes to zero in between a load
* and a refcount increment of a node in try_get, then back up to
* something non-zero, then the refcount increment is done by the other
* thread) -- so if the CAS to add the node to the actual list fails,
* decrese the refcount and leave the add operation to the next thread
* who puts the refcount back to zero (which could be us, hence the
* loop).
*/
struct freelist_node *head = READ_ONCE(list->head);
for (;;) {
WRITE_ONCE(node->next, head);
atomic_set_release(&node->refs, 1);
if (!try_cmpxchg_release(&list->head, &head, node)) {
/*
* Hmm, the add failed, but we can only try again when
* the refcount goes back to zero.
*/
if (atomic_fetch_add_release(REFS_ON_FREELIST - 1, &node->refs) == 1)
continue;
}
return;
}
}
static inline void freelist_add(struct freelist_node *node, struct freelist_head *list)
{
/*
* We know that the should-be-on-freelist bit is 0 at this point, so
* it's safe to set it using a fetch_add.
*/
if (!atomic_fetch_add_release(REFS_ON_FREELIST, &node->refs)) {
/*
* Oh look! We were the last ones referencing this node, and we
* know we want to add it to the free list, so let's do it!
*/
__freelist_add(node, list);
}
}
static inline struct freelist_node *freelist_try_get(struct freelist_head *list)
{
struct freelist_node *prev, *next, *head = smp_load_acquire(&list->head);
unsigned int refs;
while (head) {
prev = head;
refs = atomic_read(&head->refs);
if ((refs & REFS_MASK) == 0 ||
!atomic_try_cmpxchg_acquire(&head->refs, &refs, refs+1)) {
head = smp_load_acquire(&list->head);
continue;
}
/*
* Good, reference count has been incremented (it wasn't at
* zero), which means we can read the next and not worry about
* it changing between now and the time we do the CAS.
*/
next = READ_ONCE(head->next);
if (try_cmpxchg_acquire(&list->head, &head, next)) {
/*
* Yay, got the node. This means it was on the list,
* which means should-be-on-freelist must be false no
* matter the refcount (because nobody else knows it's
* been taken off yet, it can't have been put back on).
*/
WARN_ON_ONCE(atomic_read(&head->refs) & REFS_ON_FREELIST);
/*
* Decrease refcount twice, once for our ref, and once
* for the list's ref.
*/
atomic_fetch_add(-2, &head->refs);
return head;
}
/*
* OK, the head must have changed on us, but we still need to decrement
* the refcount we increased.
*/
refs = atomic_fetch_add(-1, &prev->refs);
if (refs == REFS_ON_FREELIST + 1)
__freelist_add(prev, list);
}
return NULL;
}
#endif /* FREELIST_H */
include/linux/kprobes.h
View file @ 76d4acf2
...@@ -27,6 +27,8 @@ ...@@ -27,6 +27,8 @@
#include <linux/rcupdate.h> #include <linux/rcupdate.h>
#include <linux/mutex.h> #include <linux/mutex.h>
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/refcount.h>
#include <linux/freelist.h>
#include <asm/kprobes.h> #include <asm/kprobes.h>
#ifdef CONFIG_KPROBES #ifdef CONFIG_KPROBES
...@@ -144,6 +146,11 @@ static inline int kprobe_ftrace(struct kprobe *p) ...@@ -144,6 +146,11 @@ static inline int kprobe_ftrace(struct kprobe *p)
* ignored, due to maxactive being too low. * ignored, due to maxactive being too low.
* *
*/ */
struct kretprobe_holder {
struct kretprobe *rp;
refcount_t ref;
};
struct kretprobe { struct kretprobe {
struct kprobe kp; struct kprobe kp;
kretprobe_handler_t handler; kretprobe_handler_t handler;
...@@ -151,18 +158,18 @@ struct kretprobe { ...@@ -151,18 +158,18 @@ struct kretprobe {
int maxactive; int maxactive;
int nmissed; int nmissed;
size_t data_size; size_t data_size;
struct hlist_head free_instances; struct freelist_head freelist;
raw_spinlock_t lock; struct kretprobe_holder *rph;
}; };
struct kretprobe_instance { struct kretprobe_instance {
union { union {
struct hlist_node hlist; struct freelist_node freelist;
struct rcu_head rcu; struct rcu_head rcu;
}; };
struct kretprobe *rp; struct llist_node llist;
struct kretprobe_holder *rph;
kprobe_opcode_t *ret_addr; kprobe_opcode_t *ret_addr;
struct task_struct *task;
void *fp; void *fp;
char data[]; char data[];
}; };
...@@ -221,6 +228,14 @@ unsigned long kretprobe_trampoline_handler(struct pt_regs *regs, ...@@ -221,6 +228,14 @@ unsigned long kretprobe_trampoline_handler(struct pt_regs *regs,
return ret; return ret;
} }
static nokprobe_inline struct kretprobe *get_kretprobe(struct kretprobe_instance *ri)
{
RCU_LOCKDEP_WARN(!rcu_read_lock_any_held(),
"Kretprobe is accessed from instance under preemptive context");
return READ_ONCE(ri->rph->rp);
}
#else /* CONFIG_KRETPROBES */ #else /* CONFIG_KRETPROBES */
static inline void arch_prepare_kretprobe(struct kretprobe *rp, static inline void arch_prepare_kretprobe(struct kretprobe *rp,
struct pt_regs *regs) struct pt_regs *regs)
......
include/linux/llist.h
View file @ 76d4acf2
...@@ -197,6 +197,16 @@ static inline struct llist_node *llist_next(struct llist_node *node) ...@@ -197,6 +197,16 @@ static inline struct llist_node *llist_next(struct llist_node *node)
extern bool llist_add_batch(struct llist_node *new_first, extern bool llist_add_batch(struct llist_node *new_first,
struct llist_node *new_last, struct llist_node *new_last,
struct llist_head *head); struct llist_head *head);
static inline bool __llist_add_batch(struct llist_node *new_first,
struct llist_node *new_last,
struct llist_head *head)
{
new_last->next = head->first;
head->first = new_first;
return new_last->next == NULL;
}
/** /**
* llist_add - add a new entry * llist_add - add a new entry
* @new: new entry to be added * @new: new entry to be added
...@@ -209,6 +219,11 @@ static inline bool llist_add(struct llist_node *new, struct llist_head *head) ...@@ -209,6 +219,11 @@ static inline bool llist_add(struct llist_node *new, struct llist_head *head)
return llist_add_batch(new, new, head); return llist_add_batch(new, new, head);
} }
static inline bool __llist_add(struct llist_node *new, struct llist_head *head)
{
return __llist_add_batch(new, new, head);
}
/** /**
* llist_del_all - delete all entries from lock-less list * llist_del_all - delete all entries from lock-less list
* @head: the head of lock-less list to delete all entries * @head: the head of lock-less list to delete all entries
...@@ -222,6 +237,14 @@ static inline struct llist_node *llist_del_all(struct llist_head *head) ...@@ -222,6 +237,14 @@ static inline struct llist_node *llist_del_all(struct llist_head *head)
return xchg(&head->first, NULL); return xchg(&head->first, NULL);
} }
static inline struct llist_node *__llist_del_all(struct llist_head *head)
{
struct llist_node *first = head->first;
head->first = NULL;
return first;
}
extern struct llist_node *llist_del_first(struct llist_head *head); extern struct llist_node *llist_del_first(struct llist_head *head);
struct llist_node *llist_reverse_order(struct llist_node *head); struct llist_node *llist_reverse_order(struct llist_node *head);
......
include/linux/sched.h
View file @ 76d4acf2
...@@ -1350,6 +1350,10 @@ struct task_struct { ...@@ -1350,6 +1350,10 @@ struct task_struct {
struct callback_head mce_kill_me; struct callback_head mce_kill_me;
#endif #endif
#ifdef CONFIG_KRETPROBES
struct llist_head kretprobe_instances;
#endif
/* /*
* New fields for task_struct should be added above here, so that * New fields for task_struct should be added above here, so that
* they are included in the randomized portion of task_struct. * they are included in the randomized portion of task_struct.
......
kernel/fork.c
View file @ 76d4acf2
...@@ -2183,6 +2183,10 @@ static __latent_entropy struct task_struct *copy_process( ...@@ -2183,6 +2183,10 @@ static __latent_entropy struct task_struct *copy_process(
INIT_LIST_HEAD(&p->thread_group); INIT_LIST_HEAD(&p->thread_group);
p->task_works = NULL; p->task_works = NULL;
#ifdef CONFIG_KRETPROBES
p->kretprobe_instances.first = NULL;
#endif
/* /*
* Ensure that the cgroup subsystem policies allow the new process to be * Ensure that the cgroup subsystem policies allow the new process to be
* forked. It should be noted that the new process's css_set can be changed * forked. It should be noted that the new process's css_set can be changed
......
kernel/kprobes.c
View file @ 76d4acf2
This diff is collapsed.
kernel/trace/trace_kprobe.c
View file @ 76d4acf2
...@@ -1731,7 +1731,8 @@ NOKPROBE_SYMBOL(kprobe_dispatcher); ...@@ -1731,7 +1731,8 @@ NOKPROBE_SYMBOL(kprobe_dispatcher);
static int static int
kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
{ {
struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp); struct kretprobe *rp = get_kretprobe(ri);
struct trace_kprobe *tk = container_of(rp, struct trace_kprobe, rp);
raw_cpu_inc(*tk->nhit); raw_cpu_inc(*tk->nhit);
......
scripts/atomic/gen-atomic-fallback.sh
View file @ 76d4acf2
...@@ -144,15 +144,11 @@ gen_proto_order_variants() ...@@ -144,15 +144,11 @@ gen_proto_order_variants()
printf "#endif /* ${basename}_relaxed */\n\n" printf "#endif /* ${basename}_relaxed */\n\n"
} }
gen_xchg_fallbacks() gen_order_fallbacks()
{ {
local xchg="$1"; shift local xchg="$1"; shift
cat <<EOF cat <<EOF
#ifndef ${xchg}_relaxed
#define ${xchg}_relaxed ${xchg}
#define ${xchg}_acquire ${xchg}
#define ${xchg}_release ${xchg}
#else /* ${xchg}_relaxed */
#ifndef ${xchg}_acquire #ifndef ${xchg}_acquire
#define ${xchg}_acquire(...) \\ #define ${xchg}_acquire(...) \\
...@@ -169,11 +165,62 @@ cat <<EOF ...@@ -169,11 +165,62 @@ cat <<EOF
__atomic_op_fence(${xchg}, __VA_ARGS__) __atomic_op_fence(${xchg}, __VA_ARGS__)
#endif #endif
#endif /* ${xchg}_relaxed */ EOF
}
gen_xchg_fallbacks()
{
local xchg="$1"; shift
printf "#ifndef ${xchg}_relaxed\n"
gen_basic_fallbacks ${xchg}
printf "#else /* ${xchg}_relaxed */\n"
gen_order_fallbacks ${xchg}
printf "#endif /* ${xchg}_relaxed */\n\n"
}
gen_try_cmpxchg_fallback()
{
local order="$1"; shift;
cat <<EOF
#ifndef ${ARCH}try_cmpxchg${order}
#define ${ARCH}try_cmpxchg${order}(_ptr, _oldp, _new) \\
({ \\
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\
___r = ${ARCH}cmpxchg${order}((_ptr), ___o, (_new)); \\
if (unlikely(___r != ___o)) \\
*___op = ___r; \\
likely(___r == ___o); \\
})
#endif /* ${ARCH}try_cmpxchg${order} */
EOF EOF
} }
gen_try_cmpxchg_fallbacks()
{
printf "#ifndef ${ARCH}try_cmpxchg_relaxed\n"
printf "#ifdef ${ARCH}try_cmpxchg\n"
gen_basic_fallbacks "${ARCH}try_cmpxchg"
printf "#endif /* ${ARCH}try_cmpxchg */\n\n"
for order in "" "_acquire" "_release" "_relaxed"; do
gen_try_cmpxchg_fallback "${order}"
done
printf "#else /* ${ARCH}try_cmpxchg_relaxed */\n"
gen_order_fallbacks "${ARCH}try_cmpxchg"
printf "#endif /* ${ARCH}try_cmpxchg_relaxed */\n\n"
}
cat << EOF cat << EOF
// SPDX-License-Identifier: GPL-2.0 // SPDX-License-Identifier: GPL-2.0
...@@ -191,6 +238,8 @@ for xchg in "${ARCH}xchg" "${ARCH}cmpxchg" "${ARCH}cmpxchg64"; do ...@@ -191,6 +238,8 @@ for xchg in "${ARCH}xchg" "${ARCH}cmpxchg" "${ARCH}cmpxchg64"; do
gen_xchg_fallbacks "${xchg}" gen_xchg_fallbacks "${xchg}"
done done
gen_try_cmpxchg_fallbacks
grep '^[a-z]' "$1" | while read name meta args; do grep '^[a-z]' "$1" | while read name meta args; do
gen_proto "${meta}" "${name}" "${ARCH}" "atomic" "int" ${args} gen_proto "${meta}" "${name}" "${ARCH}" "atomic" "int" ${args}
done done
......
scripts/atomic/gen-atomic-instrumented.sh
View file @ 76d4acf2
...@@ -112,14 +112,31 @@ gen_xchg() ...@@ -112,14 +112,31 @@ gen_xchg()
local xchg="$1"; shift local xchg="$1"; shift
local mult="$1"; shift local mult="$1"; shift
if [ "${xchg%${xchg#try_cmpxchg}}" = "try_cmpxchg" ] ; then
cat <<EOF
#define ${xchg}(ptr, oldp, ...) \\
({ \\
typeof(ptr) __ai_ptr = (ptr); \\
typeof(oldp) __ai_oldp = (oldp); \\
instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\
arch_${xchg}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
})
EOF
else
cat <<EOF cat <<EOF
#define ${xchg}(ptr, ...) \\ #define ${xchg}(ptr, ...) \\
({ \\ ({ \\
typeof(ptr) __ai_ptr = (ptr); \\ typeof(ptr) __ai_ptr = (ptr); \\
instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
arch_${xchg}(__ai_ptr, __VA_ARGS__); \\ arch_${xchg}(__ai_ptr, __VA_ARGS__); \\
}) })
EOF EOF
fi
} }
gen_optional_xchg() gen_optional_xchg()
...@@ -169,7 +186,7 @@ grep '^[a-z]' "$1" | while read name meta args; do ...@@ -169,7 +186,7 @@ grep '^[a-z]' "$1" | while read name meta args; do
gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
done done
for xchg in "xchg" "cmpxchg" "cmpxchg64"; do for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do
for order in "" "_acquire" "_release" "_relaxed"; do for order in "" "_acquire" "_release" "_relaxed"; do
gen_optional_xchg "${xchg}" "${order}" gen_optional_xchg "${xchg}" "${order}"
done done
......
scripts/atomic/gen-atomics.sh 100644 → 100755
View file @ 76d4acf2
File mode changed from 100644 to 100755
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7