Commit 7b9e9b58 authored by Paul E. McKenney's avatar Paul E. McKenney

srcu: Make srcu_funnel_gp_start() cache ->mynode in snp_leaf

Currently, the srcu_funnel_gp_start() walks its local variable snp up the
tree and reloads sdp->mynode whenever it is necessary to check whether
it is still at the leaf srcu_node level.  This works, but is a bit more
obtuse than absolutely necessary.  In addition, upcoming commits will
dynamically size srcu_struct structures, in which case sdp->mynode will
no longer necessarily be a constant, and this commit helps prepare for
that dynamic sizing.
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 95ebe80d
...@@ -632,20 +632,21 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, ...@@ -632,20 +632,21 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
{ {
unsigned long flags; unsigned long flags;
int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs); int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);
struct srcu_node *snp = sdp->mynode; struct srcu_node *snp;
struct srcu_node *snp_leaf = sdp->mynode;
unsigned long snp_seq; unsigned long snp_seq;
/* Each pass through the loop does one level of the srcu_node tree. */ /* Each pass through the loop does one level of the srcu_node tree. */
for (; snp != NULL; snp = snp->srcu_parent) { for (snp = snp_leaf; snp != NULL; snp = snp->srcu_parent) {
if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != sdp->mynode) if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != snp_leaf)
return; /* GP already done and CBs recorded. */ return; /* GP already done and CBs recorded. */
spin_lock_irqsave_rcu_node(snp, flags); spin_lock_irqsave_rcu_node(snp, flags);
if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) { if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) {
snp_seq = snp->srcu_have_cbs[idx]; snp_seq = snp->srcu_have_cbs[idx];
if (snp == sdp->mynode && snp_seq == s) if (snp == snp_leaf && snp_seq == s)
snp->srcu_data_have_cbs[idx] |= sdp->grpmask; snp->srcu_data_have_cbs[idx] |= sdp->grpmask;
spin_unlock_irqrestore_rcu_node(snp, flags); spin_unlock_irqrestore_rcu_node(snp, flags);
if (snp == sdp->mynode && snp_seq != s) { if (snp == snp_leaf && snp_seq != s) {
srcu_schedule_cbs_sdp(sdp, do_norm srcu_schedule_cbs_sdp(sdp, do_norm
? SRCU_INTERVAL ? SRCU_INTERVAL
: 0); : 0);
...@@ -656,7 +657,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, ...@@ -656,7 +657,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
return; return;
} }
snp->srcu_have_cbs[idx] = s; snp->srcu_have_cbs[idx] = s;
if (snp == sdp->mynode) if (snp == snp_leaf)
snp->srcu_data_have_cbs[idx] |= sdp->grpmask; snp->srcu_data_have_cbs[idx] |= sdp->grpmask;
if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s)) if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))
WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s); WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment