Commit 832f37f5 authored by Vladimir Davydov's avatar Vladimir Davydov Committed by Linus Torvalds

slub: never fail to shrink cache

SLUB's version of __kmem_cache_shrink() not only removes empty slabs, but
also tries to rearrange the partial lists to place slabs filled up most to
the head to cope with fragmentation.  To achieve that, it allocates a
temporary array of lists used to sort slabs by the number of objects in
use.  If the allocation fails, the whole procedure is aborted.

This is unacceptable for the kernel memory accounting extension of the
memory cgroup, where we want to make sure that kmem_cache_shrink()
successfully discarded empty slabs.  Although the allocation failure is
utterly unlikely with the current page allocator implementation, which
retries GFP_KERNEL allocations of order <= 2 infinitely, it is better not
to rely on that.

This patch therefore makes __kmem_cache_shrink() allocate the array on
stack instead of calling kmalloc, which may fail.  The array size is
chosen to be equal to 32, because most SLUB caches store not more than 32
objects per slab page.  Slab pages with <= 32 free objects are sorted
using the array by the number of objects in use and promoted to the head
of the partial list, while slab pages with > 32 free objects are left in
the end of the list without any ordering imposed on them.
Signed-off-by: default avatarVladimir Davydov <vdavydov@parallels.com>
Acked-by: default avatarChristoph Lameter <cl@linux.com>
Acked-by: default avatarPekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Huang Ying <ying.huang@intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 2788cf0c
...@@ -3358,11 +3358,12 @@ void kfree(const void *x) ...@@ -3358,11 +3358,12 @@ void kfree(const void *x)
} }
EXPORT_SYMBOL(kfree); EXPORT_SYMBOL(kfree);
#define SHRINK_PROMOTE_MAX 32
/* /*
* kmem_cache_shrink removes empty slabs from the partial lists and sorts * kmem_cache_shrink discards empty slabs and promotes the slabs filled
* the remaining slabs by the number of items in use. The slabs with the * up most to the head of the partial lists. New allocations will then
* most items in use come first. New allocations will then fill those up * fill those up and thus they can be removed from the partial lists.
* and thus they can be removed from the partial lists.
* *
* The slabs with the least items are placed last. This results in them * The slabs with the least items are placed last. This results in them
* being allocated from last increasing the chance that the last objects * being allocated from last increasing the chance that the last objects
...@@ -3375,51 +3376,57 @@ int __kmem_cache_shrink(struct kmem_cache *s) ...@@ -3375,51 +3376,57 @@ int __kmem_cache_shrink(struct kmem_cache *s)
struct kmem_cache_node *n; struct kmem_cache_node *n;
struct page *page; struct page *page;
struct page *t; struct page *t;
int objects = oo_objects(s->max); struct list_head discard;
struct list_head *slabs_by_inuse = struct list_head promote[SHRINK_PROMOTE_MAX];
kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL);
unsigned long flags; unsigned long flags;
if (!slabs_by_inuse)
return -ENOMEM;
flush_all(s); flush_all(s);
for_each_kmem_cache_node(s, node, n) { for_each_kmem_cache_node(s, node, n) {
if (!n->nr_partial) if (!n->nr_partial)
continue; continue;
for (i = 0; i < objects; i++) INIT_LIST_HEAD(&discard);
INIT_LIST_HEAD(slabs_by_inuse + i); for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
INIT_LIST_HEAD(promote + i);
spin_lock_irqsave(&n->list_lock, flags); spin_lock_irqsave(&n->list_lock, flags);
/* /*
* Build lists indexed by the items in use in each slab. * Build lists of slabs to discard or promote.
* *
* Note that concurrent frees may occur while we hold the * Note that concurrent frees may occur while we hold the
* list_lock. page->inuse here is the upper limit. * list_lock. page->inuse here is the upper limit.
*/ */
list_for_each_entry_safe(page, t, &n->partial, lru) { list_for_each_entry_safe(page, t, &n->partial, lru) {
list_move(&page->lru, slabs_by_inuse + page->inuse); int free = page->objects - page->inuse;
if (!page->inuse)
/* Do not reread page->inuse */
barrier();
/* We do not keep full slabs on the list */
BUG_ON(free <= 0);
if (free == page->objects) {
list_move(&page->lru, &discard);
n->nr_partial--; n->nr_partial--;
} else if (free <= SHRINK_PROMOTE_MAX)
list_move(&page->lru, promote + free - 1);
} }
/* /*
* Rebuild the partial list with the slabs filled up most * Promote the slabs filled up most to the head of the
* first and the least used slabs at the end. * partial list.
*/ */
for (i = objects - 1; i > 0; i--) for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
list_splice(slabs_by_inuse + i, n->partial.prev); list_splice(promote + i, &n->partial);
spin_unlock_irqrestore(&n->list_lock, flags); spin_unlock_irqrestore(&n->list_lock, flags);
/* Release empty slabs */ /* Release empty slabs */
list_for_each_entry_safe(page, t, slabs_by_inuse, lru) list_for_each_entry_safe(page, t, &discard, lru)
discard_slab(s, page); discard_slab(s, page);
} }
kfree(slabs_by_inuse);
return 0; return 0;
} }
...@@ -4686,12 +4693,9 @@ static ssize_t shrink_show(struct kmem_cache *s, char *buf) ...@@ -4686,12 +4693,9 @@ static ssize_t shrink_show(struct kmem_cache *s, char *buf)
static ssize_t shrink_store(struct kmem_cache *s, static ssize_t shrink_store(struct kmem_cache *s,
const char *buf, size_t length) const char *buf, size_t length)
{ {
if (buf[0] == '1') { if (buf[0] == '1')
int rc = kmem_cache_shrink(s); kmem_cache_shrink(s);
else
if (rc)
return rc;
} else
return -EINVAL; return -EINVAL;
return length; return length;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment