Commit f3bb3043 authored by Vladimir Davydov's avatar Vladimir Davydov Committed by Linus Torvalds

memcg: don't call memcg_update_all_caches if new cache id fits

memcg_update_all_caches grows arrays of per-memcg caches, so we only need
to call it when memcg_limited_groups_array_size is increased.  However,
currently we invoke it each time a new kmem-active memory cgroup is
created.  Then it just iterates over all slab_caches and does nothing
(memcg_update_cache_size returns immediately).

This patch fixes this insanity.  In the meantime it moves the code dealing
with id allocations to separate functions, memcg_alloc_cache_id and
memcg_free_cache_id.
Signed-off-by: default avatarVladimir Davydov <vdavydov@parallels.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarMichal Hocko <mhocko@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Cc: Glauber Costa <glommer@gmail.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 33a690c4
...@@ -649,11 +649,13 @@ int memcg_limited_groups_array_size; ...@@ -649,11 +649,13 @@ int memcg_limited_groups_array_size;
struct static_key memcg_kmem_enabled_key; struct static_key memcg_kmem_enabled_key;
EXPORT_SYMBOL(memcg_kmem_enabled_key); EXPORT_SYMBOL(memcg_kmem_enabled_key);
static void memcg_free_cache_id(int id);
static void disarm_kmem_keys(struct mem_cgroup *memcg) static void disarm_kmem_keys(struct mem_cgroup *memcg)
{ {
if (memcg_kmem_is_active(memcg)) { if (memcg_kmem_is_active(memcg)) {
static_key_slow_dec(&memcg_kmem_enabled_key); static_key_slow_dec(&memcg_kmem_enabled_key);
ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id); memcg_free_cache_id(memcg->kmemcg_id);
} }
/* /*
* This check can't live in kmem destruction function, * This check can't live in kmem destruction function,
...@@ -2906,19 +2908,44 @@ int memcg_cache_id(struct mem_cgroup *memcg) ...@@ -2906,19 +2908,44 @@ int memcg_cache_id(struct mem_cgroup *memcg)
return memcg ? memcg->kmemcg_id : -1; return memcg ? memcg->kmemcg_id : -1;
} }
static size_t memcg_caches_array_size(int num_groups) static int memcg_alloc_cache_id(void)
{ {
ssize_t size; int id, size;
if (num_groups <= 0) int err;
return 0;
id = ida_simple_get(&kmem_limited_groups,
0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
if (id < 0)
return id;
if (id < memcg_limited_groups_array_size)
return id;
/*
* There's no space for the new id in memcg_caches arrays,
* so we have to grow them.
*/
size = 2 * num_groups; size = 2 * (id + 1);
if (size < MEMCG_CACHES_MIN_SIZE) if (size < MEMCG_CACHES_MIN_SIZE)
size = MEMCG_CACHES_MIN_SIZE; size = MEMCG_CACHES_MIN_SIZE;
else if (size > MEMCG_CACHES_MAX_SIZE) else if (size > MEMCG_CACHES_MAX_SIZE)
size = MEMCG_CACHES_MAX_SIZE; size = MEMCG_CACHES_MAX_SIZE;
return size; mutex_lock(&memcg_slab_mutex);
err = memcg_update_all_caches(size);
mutex_unlock(&memcg_slab_mutex);
if (err) {
ida_simple_remove(&kmem_limited_groups, id);
return err;
}
return id;
}
static void memcg_free_cache_id(int id)
{
ida_simple_remove(&kmem_limited_groups, id);
} }
/* /*
...@@ -2928,22 +2955,19 @@ static size_t memcg_caches_array_size(int num_groups) ...@@ -2928,22 +2955,19 @@ static size_t memcg_caches_array_size(int num_groups)
*/ */
void memcg_update_array_size(int num) void memcg_update_array_size(int num)
{ {
if (num > memcg_limited_groups_array_size) memcg_limited_groups_array_size = num;
memcg_limited_groups_array_size = memcg_caches_array_size(num);
} }
int memcg_update_cache_size(struct kmem_cache *s, int num_groups) int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
{ {
struct memcg_cache_params *cur_params = s->memcg_params; struct memcg_cache_params *cur_params = s->memcg_params;
struct memcg_cache_params *new_params;
size_t size;
int i;
VM_BUG_ON(!is_root_cache(s)); VM_BUG_ON(!is_root_cache(s));
if (num_groups > memcg_limited_groups_array_size) { size = num_groups * sizeof(void *);
int i;
struct memcg_cache_params *new_params;
ssize_t size = memcg_caches_array_size(num_groups);
size *= sizeof(void *);
size += offsetof(struct memcg_cache_params, memcg_caches); size += offsetof(struct memcg_cache_params, memcg_caches);
new_params = kzalloc(size, GFP_KERNEL); new_params = kzalloc(size, GFP_KERNEL);
...@@ -2980,7 +3004,6 @@ int memcg_update_cache_size(struct kmem_cache *s, int num_groups) ...@@ -2980,7 +3004,6 @@ int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
rcu_assign_pointer(s->memcg_params, new_params); rcu_assign_pointer(s->memcg_params, new_params);
if (cur_params) if (cur_params)
kfree_rcu(cur_params, rcu_head); kfree_rcu(cur_params, rcu_head);
}
return 0; return 0;
} }
...@@ -4181,23 +4204,12 @@ static int __memcg_activate_kmem(struct mem_cgroup *memcg, ...@@ -4181,23 +4204,12 @@ static int __memcg_activate_kmem(struct mem_cgroup *memcg,
if (err) if (err)
goto out; goto out;
memcg_id = ida_simple_get(&kmem_limited_groups, memcg_id = memcg_alloc_cache_id();
0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
if (memcg_id < 0) { if (memcg_id < 0) {
err = memcg_id; err = memcg_id;
goto out; goto out;
} }
/*
* Make sure we have enough space for this cgroup in each root cache's
* memcg_params.
*/
mutex_lock(&memcg_slab_mutex);
err = memcg_update_all_caches(memcg_id + 1);
mutex_unlock(&memcg_slab_mutex);
if (err)
goto out_rmid;
memcg->kmemcg_id = memcg_id; memcg->kmemcg_id = memcg_id;
INIT_LIST_HEAD(&memcg->memcg_slab_caches); INIT_LIST_HEAD(&memcg->memcg_slab_caches);
...@@ -4218,10 +4230,6 @@ static int __memcg_activate_kmem(struct mem_cgroup *memcg, ...@@ -4218,10 +4230,6 @@ static int __memcg_activate_kmem(struct mem_cgroup *memcg,
out: out:
memcg_resume_kmem_account(); memcg_resume_kmem_account();
return err; return err;
out_rmid:
ida_simple_remove(&kmem_limited_groups, memcg_id);
goto out;
} }
static int memcg_activate_kmem(struct mem_cgroup *memcg, static int memcg_activate_kmem(struct mem_cgroup *memcg,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment