Commit 1759415e authored by Christoph Lameter's avatar Christoph Lameter Committed by Pekka Enberg

slub: Remove CONFIG_CMPXCHG_LOCAL ifdeffery

Remove the #ifdefs. This means that the irqsafe_cpu_cmpxchg_double() is used
everywhere.

There may be performance implications since:

A. We now have to manage a transaction ID for all arches

B. The interrupt holdoff for arches not supporting CONFIG_CMPXCHG_LOCAL is reduced
to a very short irqoff section.

There are no multiple irqoff/irqon sequences as a result of this change. Even in the fallback
case we only have to do one disable and enable like before.
Signed-off-by: default avatarChristoph Lameter <cl@linux.com>
Signed-off-by: default avatarPekka Enberg <penberg@kernel.org>
parent 8dc16c6c
...@@ -37,9 +37,7 @@ enum stat_item { ...@@ -37,9 +37,7 @@ enum stat_item {
struct kmem_cache_cpu { struct kmem_cache_cpu {
void **freelist; /* Pointer to next available object */ void **freelist; /* Pointer to next available object */
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long tid; /* Globally unique transaction id */ unsigned long tid; /* Globally unique transaction id */
#endif
struct page *page; /* The slab from which we are allocating */ struct page *page; /* The slab from which we are allocating */
int node; /* The node of the page (or -1 for debug) */ int node; /* The node of the page (or -1 for debug) */
#ifdef CONFIG_SLUB_STATS #ifdef CONFIG_SLUB_STATS
......
...@@ -1551,7 +1551,6 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) ...@@ -1551,7 +1551,6 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
} }
} }
#ifdef CONFIG_CMPXCHG_LOCAL
#ifdef CONFIG_PREEMPT #ifdef CONFIG_PREEMPT
/* /*
* Calculate the next globally unique transaction for disambiguiation * Calculate the next globally unique transaction for disambiguiation
...@@ -1611,17 +1610,12 @@ static inline void note_cmpxchg_failure(const char *n, ...@@ -1611,17 +1610,12 @@ static inline void note_cmpxchg_failure(const char *n,
stat(s, CMPXCHG_DOUBLE_CPU_FAIL); stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
} }
#endif
void init_kmem_cache_cpus(struct kmem_cache *s) void init_kmem_cache_cpus(struct kmem_cache *s)
{ {
#ifdef CONFIG_CMPXCHG_LOCAL
int cpu; int cpu;
for_each_possible_cpu(cpu) for_each_possible_cpu(cpu)
per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
#endif
} }
/* /*
* Remove the cpu slab * Remove the cpu slab
...@@ -1654,9 +1648,7 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) ...@@ -1654,9 +1648,7 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
page->inuse--; page->inuse--;
} }
c->page = NULL; c->page = NULL;
#ifdef CONFIG_CMPXCHG_LOCAL
c->tid = next_tid(c->tid); c->tid = next_tid(c->tid);
#endif
unfreeze_slab(s, page, tail); unfreeze_slab(s, page, tail);
} }
...@@ -1791,7 +1783,6 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ...@@ -1791,7 +1783,6 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
{ {
void **object; void **object;
struct page *page; struct page *page;
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long flags; unsigned long flags;
local_irq_save(flags); local_irq_save(flags);
...@@ -1802,7 +1793,6 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ...@@ -1802,7 +1793,6 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
* pointer. * pointer.
*/ */
c = this_cpu_ptr(s->cpu_slab); c = this_cpu_ptr(s->cpu_slab);
#endif
#endif #endif
/* We handle __GFP_ZERO in the caller */ /* We handle __GFP_ZERO in the caller */
...@@ -1831,10 +1821,8 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ...@@ -1831,10 +1821,8 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
unlock_out: unlock_out:
slab_unlock(page); slab_unlock(page);
#ifdef CONFIG_CMPXCHG_LOCAL
c->tid = next_tid(c->tid); c->tid = next_tid(c->tid);
local_irq_restore(flags); local_irq_restore(flags);
#endif
stat(s, ALLOC_SLOWPATH); stat(s, ALLOC_SLOWPATH);
return object; return object;
...@@ -1873,9 +1861,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ...@@ -1873,9 +1861,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
} }
if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
slab_out_of_memory(s, gfpflags, node); slab_out_of_memory(s, gfpflags, node);
#ifdef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags); local_irq_restore(flags);
#endif
return NULL; return NULL;
debug: debug:
if (!alloc_debug_processing(s, page, object, addr)) if (!alloc_debug_processing(s, page, object, addr))
...@@ -1902,20 +1888,12 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, ...@@ -1902,20 +1888,12 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
{ {
void **object; void **object;
struct kmem_cache_cpu *c; struct kmem_cache_cpu *c;
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long tid; unsigned long tid;
#else
unsigned long flags;
#endif
if (slab_pre_alloc_hook(s, gfpflags)) if (slab_pre_alloc_hook(s, gfpflags))
return NULL; return NULL;
#ifndef CONFIG_CMPXCHG_LOCAL
local_irq_save(flags);
#else
redo: redo:
#endif
/* /*
* Must read kmem_cache cpu data via this cpu ptr. Preemption is * Must read kmem_cache cpu data via this cpu ptr. Preemption is
...@@ -1925,7 +1903,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, ...@@ -1925,7 +1903,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
*/ */
c = __this_cpu_ptr(s->cpu_slab); c = __this_cpu_ptr(s->cpu_slab);
#ifdef CONFIG_CMPXCHG_LOCAL
/* /*
* The transaction ids are globally unique per cpu and per operation on * The transaction ids are globally unique per cpu and per operation on
* a per cpu queue. Thus they can be guarantee that the cmpxchg_double * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
...@@ -1934,7 +1911,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, ...@@ -1934,7 +1911,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
*/ */
tid = c->tid; tid = c->tid;
barrier(); barrier();
#endif
object = c->freelist; object = c->freelist;
if (unlikely(!object || !node_match(c, node))) if (unlikely(!object || !node_match(c, node)))
...@@ -1942,7 +1918,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, ...@@ -1942,7 +1918,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
object = __slab_alloc(s, gfpflags, node, addr, c); object = __slab_alloc(s, gfpflags, node, addr, c);
else { else {
#ifdef CONFIG_CMPXCHG_LOCAL
/* /*
* The cmpxchg will only match if there was no additonal * The cmpxchg will only match if there was no additonal
* operation and if we are on the right processor. * operation and if we are on the right processor.
...@@ -1963,16 +1938,9 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, ...@@ -1963,16 +1938,9 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
note_cmpxchg_failure("slab_alloc", s, tid); note_cmpxchg_failure("slab_alloc", s, tid);
goto redo; goto redo;
} }
#else
c->freelist = get_freepointer(s, object);
#endif
stat(s, ALLOC_FASTPATH); stat(s, ALLOC_FASTPATH);
} }
#ifndef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags);
#endif
if (unlikely(gfpflags & __GFP_ZERO) && object) if (unlikely(gfpflags & __GFP_ZERO) && object)
memset(object, 0, s->objsize); memset(object, 0, s->objsize);
...@@ -2049,11 +2017,9 @@ static void __slab_free(struct kmem_cache *s, struct page *page, ...@@ -2049,11 +2017,9 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
{ {
void *prior; void *prior;
void **object = (void *)x; void **object = (void *)x;
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long flags; unsigned long flags;
local_irq_save(flags); local_irq_save(flags);
#endif
slab_lock(page); slab_lock(page);
stat(s, FREE_SLOWPATH); stat(s, FREE_SLOWPATH);
...@@ -2084,9 +2050,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, ...@@ -2084,9 +2050,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
out_unlock: out_unlock:
slab_unlock(page); slab_unlock(page);
#ifdef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags); local_irq_restore(flags);
#endif
return; return;
slab_empty: slab_empty:
...@@ -2098,9 +2062,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, ...@@ -2098,9 +2062,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
stat(s, FREE_REMOVE_PARTIAL); stat(s, FREE_REMOVE_PARTIAL);
} }
slab_unlock(page); slab_unlock(page);
#ifdef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags); local_irq_restore(flags);
#endif
stat(s, FREE_SLAB); stat(s, FREE_SLAB);
discard_slab(s, page); discard_slab(s, page);
} }
...@@ -2121,20 +2083,11 @@ static __always_inline void slab_free(struct kmem_cache *s, ...@@ -2121,20 +2083,11 @@ static __always_inline void slab_free(struct kmem_cache *s,
{ {
void **object = (void *)x; void **object = (void *)x;
struct kmem_cache_cpu *c; struct kmem_cache_cpu *c;
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long tid; unsigned long tid;
#else
unsigned long flags;
#endif
slab_free_hook(s, x); slab_free_hook(s, x);
#ifndef CONFIG_CMPXCHG_LOCAL
local_irq_save(flags);
#else
redo: redo:
#endif
/* /*
* Determine the currently cpus per cpu slab. * Determine the currently cpus per cpu slab.
...@@ -2144,15 +2097,12 @@ static __always_inline void slab_free(struct kmem_cache *s, ...@@ -2144,15 +2097,12 @@ static __always_inline void slab_free(struct kmem_cache *s,
*/ */
c = __this_cpu_ptr(s->cpu_slab); c = __this_cpu_ptr(s->cpu_slab);
#ifdef CONFIG_CMPXCHG_LOCAL
tid = c->tid; tid = c->tid;
barrier(); barrier();
#endif
if (likely(page == c->page && c->node != NUMA_NO_NODE)) { if (likely(page == c->page && c->node != NUMA_NO_NODE)) {
set_freepointer(s, object, c->freelist); set_freepointer(s, object, c->freelist);
#ifdef CONFIG_CMPXCHG_LOCAL
if (unlikely(!this_cpu_cmpxchg_double( if (unlikely(!this_cpu_cmpxchg_double(
s->cpu_slab->freelist, s->cpu_slab->tid, s->cpu_slab->freelist, s->cpu_slab->tid,
c->freelist, tid, c->freelist, tid,
...@@ -2161,16 +2111,10 @@ static __always_inline void slab_free(struct kmem_cache *s, ...@@ -2161,16 +2111,10 @@ static __always_inline void slab_free(struct kmem_cache *s,
note_cmpxchg_failure("slab_free", s, tid); note_cmpxchg_failure("slab_free", s, tid);
goto redo; goto redo;
} }
#else
c->freelist = object;
#endif
stat(s, FREE_FASTPATH); stat(s, FREE_FASTPATH);
} else } else
__slab_free(s, page, x, addr); __slab_free(s, page, x, addr);
#ifndef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags);
#endif
} }
void kmem_cache_free(struct kmem_cache *s, void *x) void kmem_cache_free(struct kmem_cache *s, void *x)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment