Commit c016b0bd authored by Christoph Lameter's avatar Christoph Lameter Committed by Pekka Enberg

slub: Extract hooks for memory checkers from hotpaths

Extract the code that memory checkers and other verification tools use from
the hotpaths. Makes it easier to add new ones and reduces the disturbances
of the hotpaths.
Signed-off-by: default avatarChristoph Lameter <cl@linux-foundation.org>
Signed-off-by: default avatarPekka Enberg <penberg@kernel.org>
parent 51df1142
...@@ -790,6 +790,37 @@ static void trace(struct kmem_cache *s, struct page *page, void *object, ...@@ -790,6 +790,37 @@ static void trace(struct kmem_cache *s, struct page *page, void *object,
} }
} }
/*
* Hooks for other subsystems that check memory allocations. In a typical
* production configuration these hooks all should produce no code at all.
*/
static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
{
lockdep_trace_alloc(flags);
might_sleep_if(flags & __GFP_WAIT);
return should_failslab(s->objsize, flags, s->flags);
}
static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object)
{
kmemcheck_slab_alloc(s, flags, object, s->objsize);
kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags);
}
static inline void slab_free_hook(struct kmem_cache *s, void *x)
{
kmemleak_free_recursive(x, s->flags);
}
static inline void slab_free_hook_irq(struct kmem_cache *s, void *object)
{
kmemcheck_slab_free(s, object, s->objsize);
debug_check_no_locks_freed(object, s->objsize);
if (!(s->flags & SLAB_DEBUG_OBJECTS))
debug_check_no_obj_freed(object, s->objsize);
}
/* /*
* Tracking of fully allocated slabs for debugging purposes. * Tracking of fully allocated slabs for debugging purposes.
*/ */
...@@ -1696,10 +1727,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, ...@@ -1696,10 +1727,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
gfpflags &= gfp_allowed_mask; gfpflags &= gfp_allowed_mask;
lockdep_trace_alloc(gfpflags); if (slab_pre_alloc_hook(s, gfpflags))
might_sleep_if(gfpflags & __GFP_WAIT);
if (should_failslab(s->objsize, gfpflags, s->flags))
return NULL; return NULL;
local_irq_save(flags); local_irq_save(flags);
...@@ -1718,8 +1746,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, ...@@ -1718,8 +1746,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
if (unlikely(gfpflags & __GFP_ZERO) && object) if (unlikely(gfpflags & __GFP_ZERO) && object)
memset(object, 0, s->objsize); memset(object, 0, s->objsize);
kmemcheck_slab_alloc(s, gfpflags, object, s->objsize); slab_post_alloc_hook(s, gfpflags, object);
kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, gfpflags);
return object; return object;
} }
...@@ -1849,13 +1876,13 @@ static __always_inline void slab_free(struct kmem_cache *s, ...@@ -1849,13 +1876,13 @@ static __always_inline void slab_free(struct kmem_cache *s,
struct kmem_cache_cpu *c; struct kmem_cache_cpu *c;
unsigned long flags; unsigned long flags;
kmemleak_free_recursive(x, s->flags); slab_free_hook(s, x);
local_irq_save(flags); local_irq_save(flags);
c = __this_cpu_ptr(s->cpu_slab); c = __this_cpu_ptr(s->cpu_slab);
kmemcheck_slab_free(s, object, s->objsize);
debug_check_no_locks_freed(object, s->objsize); slab_free_hook_irq(s, x);
if (!(s->flags & SLAB_DEBUG_OBJECTS))
debug_check_no_obj_freed(object, s->objsize);
if (likely(page == c->page && c->node >= 0)) { if (likely(page == c->page && c->node >= 0)) {
set_freepointer(s, object, c->freelist); set_freepointer(s, object, c->freelist);
c->freelist = object; c->freelist = object;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment