Commit 560d4bc0 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Ingo Molnar

rcu: Further cleanups of use of lastcomp

Now that a copy of the rsp->completed flag is available in all
rcu_node structures, make full use of it.  It is still
legitimate to access rsp->completed while holding the root
rcu_node structure's lock, however.

Also, tighten up force_quiescent_state()'s checks for end of
current grace period.
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1258170699933-git-send-email->
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 8e9aa8f0
...@@ -817,7 +817,7 @@ cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp) ...@@ -817,7 +817,7 @@ cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp)
rnp = rdp->mynode; rnp = rdp->mynode;
spin_lock_irqsave(&rnp->lock, flags); spin_lock_irqsave(&rnp->lock, flags);
if (lastcomp != ACCESS_ONCE(rsp->completed)) { if (lastcomp != rnp->completed) {
/* /*
* Someone beat us to it for this grace period, so leave. * Someone beat us to it for this grace period, so leave.
...@@ -935,7 +935,6 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp) ...@@ -935,7 +935,6 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
{ {
unsigned long flags; unsigned long flags;
long lastcomp;
unsigned long mask; unsigned long mask;
struct rcu_data *rdp = rsp->rda[cpu]; struct rcu_data *rdp = rsp->rda[cpu];
struct rcu_node *rnp; struct rcu_node *rnp;
...@@ -971,7 +970,6 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) ...@@ -971,7 +970,6 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
spin_unlock(&rnp->lock); /* irqs remain disabled. */ spin_unlock(&rnp->lock); /* irqs remain disabled. */
rnp = rnp->parent; rnp = rnp->parent;
} while (rnp != NULL); } while (rnp != NULL);
lastcomp = rsp->completed;
spin_unlock_irqrestore(&rsp->onofflock, flags); spin_unlock_irqrestore(&rsp->onofflock, flags);
...@@ -1145,7 +1143,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp, ...@@ -1145,7 +1143,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
mask = 0; mask = 0;
spin_lock_irqsave(&rnp->lock, flags); spin_lock_irqsave(&rnp->lock, flags);
if (rsp->completed != lastcomp) { if (rnp->completed != lastcomp) {
spin_unlock_irqrestore(&rnp->lock, flags); spin_unlock_irqrestore(&rnp->lock, flags);
return 1; return 1;
} }
...@@ -1159,7 +1157,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp, ...@@ -1159,7 +1157,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,
if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu])) if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu]))
mask |= bit; mask |= bit;
} }
if (mask != 0 && rsp->completed == lastcomp) { if (mask != 0 && rnp->completed == lastcomp) {
/* cpu_quiet_msk() releases rnp->lock. */ /* cpu_quiet_msk() releases rnp->lock. */
cpu_quiet_msk(mask, rsp, rnp, flags); cpu_quiet_msk(mask, rsp, rnp, flags);
...@@ -1196,7 +1194,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) ...@@ -1196,7 +1194,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
lastcomp = rsp->gpnum - 1; lastcomp = rsp->gpnum - 1;
signaled = rsp->signaled; signaled = rsp->signaled;
rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS; rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
if (lastcomp == rsp->gpnum) { if(!rcu_gp_in_progress(rsp)) {
rsp->n_force_qs_ngp++; rsp->n_force_qs_ngp++;
spin_unlock(&rnp->lock); spin_unlock(&rnp->lock);
goto unlock_ret; /* no GP in progress, time updated. */ goto unlock_ret; /* no GP in progress, time updated. */
...@@ -1224,7 +1222,8 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) ...@@ -1224,7 +1222,8 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
/* Update state, record completion counter. */ /* Update state, record completion counter. */
forcenow = 0; forcenow = 0;
spin_lock(&rnp->lock); spin_lock(&rnp->lock);
if (lastcomp == rsp->completed && if (lastcomp + 1 == rsp->gpnum &&
lastcomp == rsp->completed &&
rsp->signaled == signaled) { rsp->signaled == signaled) {
rsp->signaled = RCU_FORCE_QS; rsp->signaled = RCU_FORCE_QS;
rsp->completed_fqs = lastcomp; rsp->completed_fqs = lastcomp;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment