Commit af537b0a authored by Christoph Lameter's avatar Christoph Lameter Committed by Pekka Enberg

slub: Use kmem_cache flags to detect if slab is in debugging mode.

The cacheline with the flags is reachable from the hot paths after the
percpu allocator changes went in. So there is no need anymore to put a
flag into each slab page. Get rid of the SlubDebug flag and use
the flags in kmem_cache instead.
Acked-by: default avatarDavid Rientjes <rientjes@google.com>
Signed-off-by: default avatarChristoph Lameter <cl@linux-foundation.org>
Signed-off-by: default avatarPekka Enberg <penberg@cs.helsinki.fi>
parent f5b801ac
...@@ -128,7 +128,6 @@ enum pageflags { ...@@ -128,7 +128,6 @@ enum pageflags {
/* SLUB */ /* SLUB */
PG_slub_frozen = PG_active, PG_slub_frozen = PG_active,
PG_slub_debug = PG_error,
}; };
#ifndef __GENERATING_BOUNDS_H #ifndef __GENERATING_BOUNDS_H
...@@ -215,7 +214,6 @@ PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked) ...@@ -215,7 +214,6 @@ PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked)
__PAGEFLAG(SlobFree, slob_free) __PAGEFLAG(SlobFree, slob_free)
__PAGEFLAG(SlubFrozen, slub_frozen) __PAGEFLAG(SlubFrozen, slub_frozen)
__PAGEFLAG(SlubDebug, slub_debug)
/* /*
* Private page markings that may be used by the filesystem that owns the page * Private page markings that may be used by the filesystem that owns the page
......
...@@ -107,11 +107,17 @@ ...@@ -107,11 +107,17 @@
* the fast path and disables lockless freelists. * the fast path and disables lockless freelists.
*/ */
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
SLAB_TRACE | SLAB_DEBUG_FREE)
static inline int kmem_cache_debug(struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_DEBUG #ifdef CONFIG_SLUB_DEBUG
#define SLABDEBUG 1 return unlikely(s->flags & SLAB_DEBUG_FLAGS);
#else #else
#define SLABDEBUG 0 return 0;
#endif #endif
}
/* /*
* Issues still to be resolved: * Issues still to be resolved:
...@@ -1157,9 +1163,6 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) ...@@ -1157,9 +1163,6 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
inc_slabs_node(s, page_to_nid(page), page->objects); inc_slabs_node(s, page_to_nid(page), page->objects);
page->slab = s; page->slab = s;
page->flags |= 1 << PG_slab; page->flags |= 1 << PG_slab;
if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON |
SLAB_STORE_USER | SLAB_TRACE))
__SetPageSlubDebug(page);
start = page_address(page); start = page_address(page);
...@@ -1186,14 +1189,13 @@ static void __free_slab(struct kmem_cache *s, struct page *page) ...@@ -1186,14 +1189,13 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
int order = compound_order(page); int order = compound_order(page);
int pages = 1 << order; int pages = 1 << order;
if (unlikely(SLABDEBUG && PageSlubDebug(page))) { if (kmem_cache_debug(s)) {
void *p; void *p;
slab_pad_check(s, page); slab_pad_check(s, page);
for_each_object(p, s, page_address(page), for_each_object(p, s, page_address(page),
page->objects) page->objects)
check_object(s, page, p, 0); check_object(s, page, p, 0);
__ClearPageSlubDebug(page);
} }
kmemcheck_free_shadow(page, compound_order(page)); kmemcheck_free_shadow(page, compound_order(page));
...@@ -1415,8 +1417,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) ...@@ -1415,8 +1417,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD); stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
} else { } else {
stat(s, DEACTIVATE_FULL); stat(s, DEACTIVATE_FULL);
if (SLABDEBUG && PageSlubDebug(page) && if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER))
(s->flags & SLAB_STORE_USER))
add_full(n, page); add_full(n, page);
} }
slab_unlock(page); slab_unlock(page);
...@@ -1624,7 +1625,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ...@@ -1624,7 +1625,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
object = c->page->freelist; object = c->page->freelist;
if (unlikely(!object)) if (unlikely(!object))
goto another_slab; goto another_slab;
if (unlikely(SLABDEBUG && PageSlubDebug(c->page))) if (kmem_cache_debug(s))
goto debug; goto debug;
c->freelist = get_freepointer(s, object); c->freelist = get_freepointer(s, object);
...@@ -1783,7 +1784,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, ...@@ -1783,7 +1784,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
stat(s, FREE_SLOWPATH); stat(s, FREE_SLOWPATH);
slab_lock(page); slab_lock(page);
if (unlikely(SLABDEBUG && PageSlubDebug(page))) if (kmem_cache_debug(s))
goto debug; goto debug;
checks_ok: checks_ok:
...@@ -3398,16 +3399,6 @@ static void validate_slab_slab(struct kmem_cache *s, struct page *page, ...@@ -3398,16 +3399,6 @@ static void validate_slab_slab(struct kmem_cache *s, struct page *page,
} else } else
printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n", printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n",
s->name, page); s->name, page);
if (s->flags & DEBUG_DEFAULT_FLAGS) {
if (!PageSlubDebug(page))
printk(KERN_ERR "SLUB %s: SlubDebug not set "
"on slab 0x%p\n", s->name, page);
} else {
if (PageSlubDebug(page))
printk(KERN_ERR "SLUB %s: SlubDebug set on "
"slab 0x%p\n", s->name, page);
}
} }
static int validate_slab_node(struct kmem_cache *s, static int validate_slab_node(struct kmem_cache *s,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment