Commit f26f9aff authored by Mike Galbraith's avatar Mike Galbraith Committed by Ingo Molnar

Sched: fix skip_clock_update optimization

idle_balance() drops/retakes rq->lock, leaving the previous task
vulnerable to set_tsk_need_resched().  Clear it after we return
from balancing instead, and in setup_thread_stack() as well, so
no successfully descheduled or never scheduled task has it set.

Need resched confused the skip_clock_update logic, which assumes
that the next call to update_rq_clock() will come nearly immediately
after being set.  Make the optimization robust against the waking
a sleeper before it sucessfully deschedules case by checking that
the current task has not been dequeued before setting the flag,
since it is that useless clock update we're trying to save, and
clear unconditionally in schedule() proper instead of conditionally
in put_prev_task().
Signed-off-by: default avatarMike Galbraith <efault@gmx.de>
Reported-by: default avatarBjoern B. Brandenburg <bbb.lst@gmail.com>
Tested-by: default avatarYong Zhang <yong.zhang0@gmail.com>
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Cc: stable@kernel.org
LKML-Reference: <1291802742.1417.9.camel@marge.simson.net>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 0f004f5a
...@@ -273,6 +273,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) ...@@ -273,6 +273,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
setup_thread_stack(tsk, orig); setup_thread_stack(tsk, orig);
clear_user_return_notifier(tsk); clear_user_return_notifier(tsk);
clear_tsk_need_resched(tsk);
stackend = end_of_stack(tsk); stackend = end_of_stack(tsk);
*stackend = STACK_END_MAGIC; /* for overflow detection */ *stackend = STACK_END_MAGIC; /* for overflow detection */
......
...@@ -641,17 +641,18 @@ static void sched_irq_time_avg_update(struct rq *rq, u64 irq_time); ...@@ -641,17 +641,18 @@ static void sched_irq_time_avg_update(struct rq *rq, u64 irq_time);
inline void update_rq_clock(struct rq *rq) inline void update_rq_clock(struct rq *rq)
{ {
if (!rq->skip_clock_update) {
int cpu = cpu_of(rq); int cpu = cpu_of(rq);
u64 irq_time; u64 irq_time;
if (rq->skip_clock_update)
return;
rq->clock = sched_clock_cpu(cpu); rq->clock = sched_clock_cpu(cpu);
irq_time = irq_time_cpu(cpu); irq_time = irq_time_cpu(cpu);
if (rq->clock - irq_time > rq->clock_task) if (rq->clock - irq_time > rq->clock_task)
rq->clock_task = rq->clock - irq_time; rq->clock_task = rq->clock - irq_time;
sched_irq_time_avg_update(rq, irq_time); sched_irq_time_avg_update(rq, irq_time);
}
} }
/* /*
...@@ -2129,7 +2130,7 @@ static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) ...@@ -2129,7 +2130,7 @@ static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
* A queue event has occurred, and we're going to schedule. In * A queue event has occurred, and we're going to schedule. In
* this case, we can save a useless back to back clock update. * this case, we can save a useless back to back clock update.
*/ */
if (test_tsk_need_resched(rq->curr)) if (rq->curr->se.on_rq && test_tsk_need_resched(rq->curr))
rq->skip_clock_update = 1; rq->skip_clock_update = 1;
} }
...@@ -3973,7 +3974,6 @@ static void put_prev_task(struct rq *rq, struct task_struct *prev) ...@@ -3973,7 +3974,6 @@ static void put_prev_task(struct rq *rq, struct task_struct *prev)
{ {
if (prev->se.on_rq) if (prev->se.on_rq)
update_rq_clock(rq); update_rq_clock(rq);
rq->skip_clock_update = 0;
prev->sched_class->put_prev_task(rq, prev); prev->sched_class->put_prev_task(rq, prev);
} }
...@@ -4031,7 +4031,6 @@ asmlinkage void __sched schedule(void) ...@@ -4031,7 +4031,6 @@ asmlinkage void __sched schedule(void)
hrtick_clear(rq); hrtick_clear(rq);
raw_spin_lock_irq(&rq->lock); raw_spin_lock_irq(&rq->lock);
clear_tsk_need_resched(prev);
switch_count = &prev->nivcsw; switch_count = &prev->nivcsw;
if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
...@@ -4063,6 +4062,8 @@ asmlinkage void __sched schedule(void) ...@@ -4063,6 +4062,8 @@ asmlinkage void __sched schedule(void)
put_prev_task(rq, prev); put_prev_task(rq, prev);
next = pick_next_task(rq); next = pick_next_task(rq);
clear_tsk_need_resched(prev);
rq->skip_clock_update = 0;
if (likely(prev != next)) { if (likely(prev != next)) {
sched_info_switch(prev, next); sched_info_switch(prev, next);
...@@ -4071,6 +4072,7 @@ asmlinkage void __sched schedule(void) ...@@ -4071,6 +4072,7 @@ asmlinkage void __sched schedule(void)
rq->nr_switches++; rq->nr_switches++;
rq->curr = next; rq->curr = next;
++*switch_count; ++*switch_count;
WARN_ON_ONCE(test_tsk_need_resched(next));
context_switch(rq, prev, next); /* unlocks the rq */ context_switch(rq, prev, next); /* unlocks the rq */
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment