Commit 9881b024 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar

sched/clock: Delay switching sched_clock to stable

Currently we switch to the stable sched_clock if we guess the TSC is
usable, and then switch back to the unstable path if it turns out TSC
isn't stable during SMP bringup after all.

Delay switching to the stable path until after SMP bringup is
complete. This way we'll avoid switching during the time we detect the
worst of the TSC offences.
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 555570d7
...@@ -2515,6 +2515,10 @@ extern u64 sched_clock_cpu(int cpu); ...@@ -2515,6 +2515,10 @@ extern u64 sched_clock_cpu(int cpu);
extern void sched_clock_init(void); extern void sched_clock_init(void);
#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
static inline void sched_clock_init_late(void)
{
}
static inline void sched_clock_tick(void) static inline void sched_clock_tick(void)
{ {
} }
...@@ -2537,6 +2541,7 @@ static inline u64 local_clock(void) ...@@ -2537,6 +2541,7 @@ static inline u64 local_clock(void)
return sched_clock(); return sched_clock();
} }
#else #else
extern void sched_clock_init_late(void);
/* /*
* Architectures can set this to 1 if they have specified * Architectures can set this to 1 if they have specified
* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig, * CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig,
......
...@@ -625,7 +625,6 @@ asmlinkage __visible void __init start_kernel(void) ...@@ -625,7 +625,6 @@ asmlinkage __visible void __init start_kernel(void)
numa_policy_init(); numa_policy_init();
if (late_time_init) if (late_time_init)
late_time_init(); late_time_init();
sched_clock_init();
calibrate_delay(); calibrate_delay();
pidmap_init(); pidmap_init();
anon_vma_init(); anon_vma_init();
......
...@@ -77,6 +77,11 @@ EXPORT_SYMBOL_GPL(sched_clock); ...@@ -77,6 +77,11 @@ EXPORT_SYMBOL_GPL(sched_clock);
__read_mostly int sched_clock_running; __read_mostly int sched_clock_running;
void sched_clock_init(void)
{
sched_clock_running = 1;
}
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
static DEFINE_STATIC_KEY_FALSE(__sched_clock_stable); static DEFINE_STATIC_KEY_FALSE(__sched_clock_stable);
static int __sched_clock_stable_early; static int __sched_clock_stable_early;
...@@ -96,12 +101,18 @@ void set_sched_clock_stable(void) ...@@ -96,12 +101,18 @@ void set_sched_clock_stable(void)
{ {
__sched_clock_stable_early = 1; __sched_clock_stable_early = 1;
smp_mb(); /* matches sched_clock_init() */ smp_mb(); /* matches sched_clock_init_late() */
if (!sched_clock_running)
return;
__set_sched_clock_stable(); /*
* This really should only be called early (before
* sched_clock_init_late()) when guestimating our sched_clock() is
* solid.
*
* After that we test stability and we can negate our guess using
* clear_sched_clock_stable, possibly from a watchdog.
*/
if (WARN_ON_ONCE(sched_clock_running == 2))
__set_sched_clock_stable();
} }
static void __clear_sched_clock_stable(struct work_struct *work) static void __clear_sched_clock_stable(struct work_struct *work)
...@@ -117,12 +128,10 @@ void clear_sched_clock_stable(void) ...@@ -117,12 +128,10 @@ void clear_sched_clock_stable(void)
{ {
__sched_clock_stable_early = 0; __sched_clock_stable_early = 0;
smp_mb(); /* matches sched_clock_init() */ smp_mb(); /* matches sched_clock_init_late() */
if (!sched_clock_running)
return;
schedule_work(&sched_clock_work); if (sched_clock_running == 2)
schedule_work(&sched_clock_work);
} }
struct sched_clock_data { struct sched_clock_data {
...@@ -143,20 +152,9 @@ static inline struct sched_clock_data *cpu_sdc(int cpu) ...@@ -143,20 +152,9 @@ static inline struct sched_clock_data *cpu_sdc(int cpu)
return &per_cpu(sched_clock_data, cpu); return &per_cpu(sched_clock_data, cpu);
} }
void sched_clock_init(void) void sched_clock_init_late(void)
{ {
u64 ktime_now = ktime_to_ns(ktime_get()); sched_clock_running = 2;
int cpu;
for_each_possible_cpu(cpu) {
struct sched_clock_data *scd = cpu_sdc(cpu);
scd->tick_raw = 0;
scd->tick_gtod = ktime_now;
scd->clock = ktime_now;
}
sched_clock_running = 1;
/* /*
* Ensure that it is impossible to not do a static_key update. * Ensure that it is impossible to not do a static_key update.
...@@ -362,11 +360,6 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); ...@@ -362,11 +360,6 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ #else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
void sched_clock_init(void)
{
sched_clock_running = 1;
}
u64 sched_clock_cpu(int cpu) u64 sched_clock_cpu(int cpu)
{ {
if (unlikely(!sched_clock_running)) if (unlikely(!sched_clock_running))
...@@ -374,6 +367,7 @@ u64 sched_clock_cpu(int cpu) ...@@ -374,6 +367,7 @@ u64 sched_clock_cpu(int cpu)
return sched_clock(); return sched_clock();
} }
#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ #endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
/* /*
......
...@@ -7498,6 +7498,7 @@ void __init sched_init_smp(void) ...@@ -7498,6 +7498,7 @@ void __init sched_init_smp(void)
init_sched_dl_class(); init_sched_dl_class();
sched_init_smt(); sched_init_smt();
sched_clock_init_late();
sched_smp_initialized = true; sched_smp_initialized = true;
} }
...@@ -7513,6 +7514,7 @@ early_initcall(migration_init); ...@@ -7513,6 +7514,7 @@ early_initcall(migration_init);
void __init sched_init_smp(void) void __init sched_init_smp(void)
{ {
sched_init_granularity(); sched_init_granularity();
sched_clock_init_late();
} }
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
...@@ -7556,6 +7558,8 @@ void __init sched_init(void) ...@@ -7556,6 +7558,8 @@ void __init sched_init(void)
int i, j; int i, j;
unsigned long alloc_size = 0, ptr; unsigned long alloc_size = 0, ptr;
sched_clock_init();
for (i = 0; i < WAIT_TABLE_SIZE; i++) for (i = 0; i < WAIT_TABLE_SIZE; i++)
init_waitqueue_head(bit_wait_table + i); init_waitqueue_head(bit_wait_table + i);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment