Commit 302a3ea3 authored by Suren Baghdasaryan's avatar Suren Baghdasaryan Committed by Vlastimil Babka

mm, slab: move allocation tagging code in the alloc path into a hook

Move allocation tagging specific code in the allocation path into
alloc_tagging_slab_alloc_hook, similar to how freeing path uses
alloc_tagging_slab_free_hook. No functional changes, just code
cleanup.
Suggested-by: default avatarVlastimil Babka <vbabka@suse.cz>
Signed-off-by: default avatarSuren Baghdasaryan <surenb@google.com>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent 22a40d14
......@@ -2033,11 +2033,54 @@ prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p)
return slab_obj_exts(slab) + obj_to_index(s, slab, p);
}
#else /* CONFIG_SLAB_OBJ_EXT */
static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
gfp_t gfp, bool new_slab)
{
return 0;
}
static inline void free_slab_obj_exts(struct slab *slab)
{
}
static inline bool need_slab_obj_ext(void)
{
return false;
}
static inline struct slabobj_ext *
prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p)
{
return NULL;
}
#endif /* CONFIG_SLAB_OBJ_EXT */
#ifdef CONFIG_MEM_ALLOC_PROFILING
static inline void
alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
{
if (need_slab_obj_ext()) {
struct slabobj_ext *obj_exts;
obj_exts = prepare_slab_obj_exts_hook(s, flags, object);
/*
* Currently obj_exts is used only for allocation profiling.
* If other users appear then mem_alloc_profiling_enabled()
* check should be added before alloc_tag_add().
*/
if (likely(obj_exts))
alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size);
}
}
static inline void
alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
int objects)
{
#ifdef CONFIG_MEM_ALLOC_PROFILING
struct slabobj_ext *obj_exts;
int i;
......@@ -2053,30 +2096,13 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
alloc_tag_sub(&obj_exts[off].ref, s->size);
}
#endif
}
#else /* CONFIG_SLAB_OBJ_EXT */
static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
gfp_t gfp, bool new_slab)
{
return 0;
}
static inline void free_slab_obj_exts(struct slab *slab)
{
}
static inline bool need_slab_obj_ext(void)
{
return false;
}
#else /* CONFIG_MEM_ALLOC_PROFILING */
static inline struct slabobj_ext *
prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p)
static inline void
alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
{
return NULL;
}
static inline void
......@@ -2085,7 +2111,8 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
{
}
#endif /* CONFIG_SLAB_OBJ_EXT */
#endif /* CONFIG_MEM_ALLOC_PROFILING */
#ifdef CONFIG_MEMCG_KMEM
......@@ -3944,20 +3971,7 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
kmemleak_alloc_recursive(p[i], s->object_size, 1,
s->flags, init_flags);
kmsan_slab_alloc(s, p[i], init_flags);
#ifdef CONFIG_MEM_ALLOC_PROFILING
if (need_slab_obj_ext()) {
struct slabobj_ext *obj_exts;
obj_exts = prepare_slab_obj_exts_hook(s, flags, p[i]);
/*
* Currently obj_exts is used only for allocation profiling.
* If other users appear then mem_alloc_profiling_enabled()
* check should be added before alloc_tag_add().
*/
if (likely(obj_exts))
alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size);
}
#endif
alloc_tagging_slab_alloc_hook(s, p[i], flags);
}
return memcg_slab_post_alloc_hook(s, lru, flags, size, p);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment