Commit dc26532a authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds

cgroup: rstat: punt root-level optimization to individual controllers

Current users of the rstat code can source root-level statistics from
the native counters of their respective subsystem, allowing them to
forego aggregation at the root level.  This optimization is currently
implemented inside the generic rstat code, which doesn't track the root
cgroup and doesn't invoke the subsystem flush callbacks on it.

However, the memory controller cannot do this optimization, because
cgroup1 breaks out memory specifically for the local level, including at
the root level.  In preparation for the memory controller switching to
rstat, move the optimization from rstat core to the controllers.

Afterwards, rstat will always track the root cgroup for changes and
invoke the subsystem callbacks on it; and it's up to the subsystem to
special-case and skip aggregation of the root cgroup if it can source
this information through other, cheaper means.

This is the case for the io controller and the cgroup base stats.  In
their respective flush callbacks, check whether the parent is the root
cgroup, and if so, skip the unnecessary upward propagation.

The extra cost of tracking the root cgroup is negligible: on stat
changes, we actually remove a branch that checks for the root.  The
queueing for a flush touches only per-cpu data, and only the first stat
change since a flush requires a (per-cpu) lock.

Link: https://lkml.kernel.org/r/20210209163304.77088-6-hannes@cmpxchg.orgSigned-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarTejun Heo <tj@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Michal Koutný <mkoutny@suse.com>
Cc: Roman Gushchin <guro@fb.com>
Cc: Shakeel Butt <shakeelb@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent a7df69b8
...@@ -764,6 +764,10 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu) ...@@ -764,6 +764,10 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
struct blkcg *blkcg = css_to_blkcg(css); struct blkcg *blkcg = css_to_blkcg(css);
struct blkcg_gq *blkg; struct blkcg_gq *blkg;
/* Root-level stats are sourced from system-wide IO stats */
if (!cgroup_parent(css->cgroup))
return;
rcu_read_lock(); rcu_read_lock();
hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
...@@ -786,8 +790,8 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu) ...@@ -786,8 +790,8 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
blkg_iostat_add(&bisc->last, &delta); blkg_iostat_add(&bisc->last, &delta);
u64_stats_update_end(&blkg->iostat.sync); u64_stats_update_end(&blkg->iostat.sync);
/* propagate global delta to parent */ /* propagate global delta to parent (unless that's root) */
if (parent) { if (parent && parent->parent) {
u64_stats_update_begin(&parent->iostat.sync); u64_stats_update_begin(&parent->iostat.sync);
blkg_iostat_set(&delta, &blkg->iostat.cur); blkg_iostat_set(&delta, &blkg->iostat.cur);
blkg_iostat_sub(&delta, &blkg->iostat.last); blkg_iostat_sub(&delta, &blkg->iostat.last);
...@@ -801,10 +805,11 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu) ...@@ -801,10 +805,11 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
} }
/* /*
* The rstat algorithms intentionally don't handle the root cgroup to avoid * We source root cgroup stats from the system-wide stats to avoid
* incurring overhead when no cgroups are defined. For that reason, * tracking the same information twice and incurring overhead when no
* cgroup_rstat_flush in blkcg_print_stat does not actually fill out the * cgroups are defined. For that reason, cgroup_rstat_flush in
* iostat in the root cgroup's blkcg_gq. * blkcg_print_stat does not actually fill out the iostat in the root
* cgroup's blkcg_gq.
* *
* However, we would like to re-use the printing code between the root and * However, we would like to re-use the printing code between the root and
* non-root cgroups to the extent possible. For that reason, we simulate * non-root cgroups to the extent possible. For that reason, we simulate
......
...@@ -25,13 +25,8 @@ static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu) ...@@ -25,13 +25,8 @@ static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)
void cgroup_rstat_updated(struct cgroup *cgrp, int cpu) void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
{ {
raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu); raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu);
struct cgroup *parent;
unsigned long flags; unsigned long flags;
/* nothing to do for root */
if (!cgroup_parent(cgrp))
return;
/* /*
* Speculative already-on-list test. This may race leading to * Speculative already-on-list test. This may race leading to
* temporary inaccuracies, which is fine. * temporary inaccuracies, which is fine.
...@@ -46,10 +41,10 @@ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu) ...@@ -46,10 +41,10 @@ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
raw_spin_lock_irqsave(cpu_lock, flags); raw_spin_lock_irqsave(cpu_lock, flags);
/* put @cgrp and all ancestors on the corresponding updated lists */ /* put @cgrp and all ancestors on the corresponding updated lists */
for (parent = cgroup_parent(cgrp); parent; while (true) {
cgrp = parent, parent = cgroup_parent(cgrp)) {
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu); struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
struct cgroup_rstat_cpu *prstatc = cgroup_rstat_cpu(parent, cpu); struct cgroup *parent = cgroup_parent(cgrp);
struct cgroup_rstat_cpu *prstatc;
/* /*
* Both additions and removals are bottom-up. If a cgroup * Both additions and removals are bottom-up. If a cgroup
...@@ -58,8 +53,17 @@ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu) ...@@ -58,8 +53,17 @@ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
if (rstatc->updated_next) if (rstatc->updated_next)
break; break;
/* Root has no parent to link it to, but mark it busy */
if (!parent) {
rstatc->updated_next = cgrp;
break;
}
prstatc = cgroup_rstat_cpu(parent, cpu);
rstatc->updated_next = prstatc->updated_children; rstatc->updated_next = prstatc->updated_children;
prstatc->updated_children = cgrp; prstatc->updated_children = cgrp;
cgrp = parent;
} }
raw_spin_unlock_irqrestore(cpu_lock, flags); raw_spin_unlock_irqrestore(cpu_lock, flags);
...@@ -113,23 +117,26 @@ static struct cgroup *cgroup_rstat_cpu_pop_updated(struct cgroup *pos, ...@@ -113,23 +117,26 @@ static struct cgroup *cgroup_rstat_cpu_pop_updated(struct cgroup *pos,
*/ */
if (rstatc->updated_next) { if (rstatc->updated_next) {
struct cgroup *parent = cgroup_parent(pos); struct cgroup *parent = cgroup_parent(pos);
struct cgroup_rstat_cpu *prstatc = cgroup_rstat_cpu(parent, cpu);
struct cgroup_rstat_cpu *nrstatc; if (parent) {
struct cgroup **nextp; struct cgroup_rstat_cpu *prstatc;
struct cgroup **nextp;
nextp = &prstatc->updated_children;
while (true) { prstatc = cgroup_rstat_cpu(parent, cpu);
nrstatc = cgroup_rstat_cpu(*nextp, cpu); nextp = &prstatc->updated_children;
if (*nextp == pos) while (true) {
break; struct cgroup_rstat_cpu *nrstatc;
WARN_ON_ONCE(*nextp == parent); nrstatc = cgroup_rstat_cpu(*nextp, cpu);
nextp = &nrstatc->updated_next; if (*nextp == pos)
break;
WARN_ON_ONCE(*nextp == parent);
nextp = &nrstatc->updated_next;
}
*nextp = rstatc->updated_next;
} }
*nextp = rstatc->updated_next;
rstatc->updated_next = NULL; rstatc->updated_next = NULL;
return pos; return pos;
} }
...@@ -309,11 +316,15 @@ static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat, ...@@ -309,11 +316,15 @@ static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat,
static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu) static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
{ {
struct cgroup *parent = cgroup_parent(cgrp);
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu); struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
struct cgroup *parent = cgroup_parent(cgrp);
struct cgroup_base_stat cur, delta; struct cgroup_base_stat cur, delta;
unsigned seq; unsigned seq;
/* Root-level stats are sourced from system-wide CPU stats */
if (!parent)
return;
/* fetch the current per-cpu values */ /* fetch the current per-cpu values */
do { do {
seq = __u64_stats_fetch_begin(&rstatc->bsync); seq = __u64_stats_fetch_begin(&rstatc->bsync);
...@@ -326,8 +337,8 @@ static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu) ...@@ -326,8 +337,8 @@ static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
cgroup_base_stat_add(&cgrp->bstat, &delta); cgroup_base_stat_add(&cgrp->bstat, &delta);
cgroup_base_stat_add(&rstatc->last_bstat, &delta); cgroup_base_stat_add(&rstatc->last_bstat, &delta);
/* propagate global delta to parent */ /* propagate global delta to parent (unless that's root) */
if (parent) { if (cgroup_parent(parent)) {
delta = cgrp->bstat; delta = cgrp->bstat;
cgroup_base_stat_sub(&delta, &cgrp->last_bstat); cgroup_base_stat_sub(&delta, &cgrp->last_bstat);
cgroup_base_stat_add(&parent->bstat, &delta); cgroup_base_stat_add(&parent->bstat, &delta);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment