Commit 3a95398f authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull NOHZ updates from Thomas Gleixner:
 "A few updates to the nohz infrastructure:

   - recursion protection for context tracking

   - make the TIF_NOHZ inheritance smarter

   - isolate cpus which belong to the NOHZ full set"

* 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  nohz: Set isolcpus when nohz_full is set
  nohz: Add tick_nohz_full_add_cpus_to() API
  context_tracking: Inherit TIF_NOHZ through forks instead of context switches
  context_tracking: Protect against recursion
parents 43224b96 8cb9764f
...@@ -14,8 +14,6 @@ extern void context_tracking_enter(enum ctx_state state); ...@@ -14,8 +14,6 @@ extern void context_tracking_enter(enum ctx_state state);
extern void context_tracking_exit(enum ctx_state state); extern void context_tracking_exit(enum ctx_state state);
extern void context_tracking_user_enter(void); extern void context_tracking_user_enter(void);
extern void context_tracking_user_exit(void); extern void context_tracking_user_exit(void);
extern void __context_tracking_task_switch(struct task_struct *prev,
struct task_struct *next);
static inline void user_enter(void) static inline void user_enter(void)
{ {
...@@ -51,19 +49,11 @@ static inline void exception_exit(enum ctx_state prev_ctx) ...@@ -51,19 +49,11 @@ static inline void exception_exit(enum ctx_state prev_ctx)
} }
} }
static inline void context_tracking_task_switch(struct task_struct *prev,
struct task_struct *next)
{
if (context_tracking_is_enabled())
__context_tracking_task_switch(prev, next);
}
#else #else
static inline void user_enter(void) { } static inline void user_enter(void) { }
static inline void user_exit(void) { } static inline void user_exit(void) { }
static inline enum ctx_state exception_enter(void) { return 0; } static inline enum ctx_state exception_enter(void) { return 0; }
static inline void exception_exit(enum ctx_state prev_ctx) { } static inline void exception_exit(enum ctx_state prev_ctx) { }
static inline void context_tracking_task_switch(struct task_struct *prev,
struct task_struct *next) { }
#endif /* !CONFIG_CONTEXT_TRACKING */ #endif /* !CONFIG_CONTEXT_TRACKING */
......
...@@ -12,6 +12,7 @@ struct context_tracking { ...@@ -12,6 +12,7 @@ struct context_tracking {
* may be further optimized using static keys. * may be further optimized using static keys.
*/ */
bool active; bool active;
int recursion;
enum ctx_state { enum ctx_state {
CONTEXT_KERNEL = 0, CONTEXT_KERNEL = 0,
CONTEXT_USER, CONTEXT_USER,
......
...@@ -2599,6 +2599,9 @@ static inline unsigned long wait_task_inactive(struct task_struct *p, ...@@ -2599,6 +2599,9 @@ static inline unsigned long wait_task_inactive(struct task_struct *p,
} }
#endif #endif
#define tasklist_empty() \
list_empty(&init_task.tasks)
#define next_task(p) \ #define next_task(p) \
list_entry_rcu((p)->tasks.next, struct task_struct, tasks) list_entry_rcu((p)->tasks.next, struct task_struct, tasks)
......
...@@ -134,6 +134,12 @@ static inline bool tick_nohz_full_cpu(int cpu) ...@@ -134,6 +134,12 @@ static inline bool tick_nohz_full_cpu(int cpu)
return cpumask_test_cpu(cpu, tick_nohz_full_mask); return cpumask_test_cpu(cpu, tick_nohz_full_mask);
} }
static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask)
{
if (tick_nohz_full_enabled())
cpumask_or(mask, mask, tick_nohz_full_mask);
}
extern void __tick_nohz_full_check(void); extern void __tick_nohz_full_check(void);
extern void tick_nohz_full_kick(void); extern void tick_nohz_full_kick(void);
extern void tick_nohz_full_kick_cpu(int cpu); extern void tick_nohz_full_kick_cpu(int cpu);
...@@ -142,6 +148,7 @@ extern void __tick_nohz_task_switch(struct task_struct *tsk); ...@@ -142,6 +148,7 @@ extern void __tick_nohz_task_switch(struct task_struct *tsk);
#else #else
static inline bool tick_nohz_full_enabled(void) { return false; } static inline bool tick_nohz_full_enabled(void) { return false; }
static inline bool tick_nohz_full_cpu(int cpu) { return false; } static inline bool tick_nohz_full_cpu(int cpu) { return false; }
static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) { }
static inline void __tick_nohz_full_check(void) { } static inline void __tick_nohz_full_check(void) { }
static inline void tick_nohz_full_kick_cpu(int cpu) { } static inline void tick_nohz_full_kick_cpu(int cpu) { }
static inline void tick_nohz_full_kick(void) { } static inline void tick_nohz_full_kick(void) { }
......
...@@ -30,12 +30,23 @@ EXPORT_SYMBOL_GPL(context_tracking_enabled); ...@@ -30,12 +30,23 @@ EXPORT_SYMBOL_GPL(context_tracking_enabled);
DEFINE_PER_CPU(struct context_tracking, context_tracking); DEFINE_PER_CPU(struct context_tracking, context_tracking);
EXPORT_SYMBOL_GPL(context_tracking); EXPORT_SYMBOL_GPL(context_tracking);
void context_tracking_cpu_set(int cpu) static bool context_tracking_recursion_enter(void)
{ {
if (!per_cpu(context_tracking.active, cpu)) { int recursion;
per_cpu(context_tracking.active, cpu) = true;
static_key_slow_inc(&context_tracking_enabled); recursion = __this_cpu_inc_return(context_tracking.recursion);
} if (recursion == 1)
return true;
WARN_ONCE((recursion < 1), "Invalid context tracking recursion value %d\n", recursion);
__this_cpu_dec(context_tracking.recursion);
return false;
}
static void context_tracking_recursion_exit(void)
{
__this_cpu_dec(context_tracking.recursion);
} }
/** /**
...@@ -75,6 +86,9 @@ void context_tracking_enter(enum ctx_state state) ...@@ -75,6 +86,9 @@ void context_tracking_enter(enum ctx_state state)
WARN_ON_ONCE(!current->mm); WARN_ON_ONCE(!current->mm);
local_irq_save(flags); local_irq_save(flags);
if (!context_tracking_recursion_enter())
goto out_irq_restore;
if ( __this_cpu_read(context_tracking.state) != state) { if ( __this_cpu_read(context_tracking.state) != state) {
if (__this_cpu_read(context_tracking.active)) { if (__this_cpu_read(context_tracking.active)) {
/* /*
...@@ -105,6 +119,8 @@ void context_tracking_enter(enum ctx_state state) ...@@ -105,6 +119,8 @@ void context_tracking_enter(enum ctx_state state)
*/ */
__this_cpu_write(context_tracking.state, state); __this_cpu_write(context_tracking.state, state);
} }
context_tracking_recursion_exit();
out_irq_restore:
local_irq_restore(flags); local_irq_restore(flags);
} }
NOKPROBE_SYMBOL(context_tracking_enter); NOKPROBE_SYMBOL(context_tracking_enter);
...@@ -139,6 +155,9 @@ void context_tracking_exit(enum ctx_state state) ...@@ -139,6 +155,9 @@ void context_tracking_exit(enum ctx_state state)
return; return;
local_irq_save(flags); local_irq_save(flags);
if (!context_tracking_recursion_enter())
goto out_irq_restore;
if (__this_cpu_read(context_tracking.state) == state) { if (__this_cpu_read(context_tracking.state) == state) {
if (__this_cpu_read(context_tracking.active)) { if (__this_cpu_read(context_tracking.active)) {
/* /*
...@@ -153,6 +172,8 @@ void context_tracking_exit(enum ctx_state state) ...@@ -153,6 +172,8 @@ void context_tracking_exit(enum ctx_state state)
} }
__this_cpu_write(context_tracking.state, CONTEXT_KERNEL); __this_cpu_write(context_tracking.state, CONTEXT_KERNEL);
} }
context_tracking_recursion_exit();
out_irq_restore:
local_irq_restore(flags); local_irq_restore(flags);
} }
NOKPROBE_SYMBOL(context_tracking_exit); NOKPROBE_SYMBOL(context_tracking_exit);
...@@ -164,24 +185,26 @@ void context_tracking_user_exit(void) ...@@ -164,24 +185,26 @@ void context_tracking_user_exit(void)
} }
NOKPROBE_SYMBOL(context_tracking_user_exit); NOKPROBE_SYMBOL(context_tracking_user_exit);
/** void __init context_tracking_cpu_set(int cpu)
* __context_tracking_task_switch - context switch the syscall callbacks
* @prev: the task that is being switched out
* @next: the task that is being switched in
*
* The context tracking uses the syscall slow path to implement its user-kernel
* boundaries probes on syscalls. This way it doesn't impact the syscall fast
* path on CPUs that don't do context tracking.
*
* But we need to clear the flag on the previous task because it may later
* migrate to some CPU that doesn't do the context tracking. As such the TIF
* flag may not be desired there.
*/
void __context_tracking_task_switch(struct task_struct *prev,
struct task_struct *next)
{ {
clear_tsk_thread_flag(prev, TIF_NOHZ); static __initdata bool initialized = false;
set_tsk_thread_flag(next, TIF_NOHZ);
if (!per_cpu(context_tracking.active, cpu)) {
per_cpu(context_tracking.active, cpu) = true;
static_key_slow_inc(&context_tracking_enabled);
}
if (initialized)
return;
/*
* Set TIF_NOHZ to init/0 and let it propagate to all tasks through fork
* This assumes that init is the only task at this early boot stage.
*/
set_tsk_thread_flag(&init_task, TIF_NOHZ);
WARN_ON_ONCE(!tasklist_empty());
initialized = true;
} }
#ifdef CONFIG_CONTEXT_TRACKING_FORCE #ifdef CONFIG_CONTEXT_TRACKING_FORCE
......
...@@ -2374,7 +2374,6 @@ context_switch(struct rq *rq, struct task_struct *prev, ...@@ -2374,7 +2374,6 @@ context_switch(struct rq *rq, struct task_struct *prev,
*/ */
spin_release(&rq->lock.dep_map, 1, _THIS_IP_); spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
context_tracking_task_switch(prev, next);
/* Here we just switch the register state and the stack. */ /* Here we just switch the register state and the stack. */
switch_to(prev, next, prev); switch_to(prev, next, prev);
barrier(); barrier();
...@@ -7068,6 +7067,9 @@ void __init sched_init_smp(void) ...@@ -7068,6 +7067,9 @@ void __init sched_init_smp(void)
alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL); alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
alloc_cpumask_var(&fallback_doms, GFP_KERNEL); alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
/* nohz_full won't take effect without isolating the cpus. */
tick_nohz_full_add_cpus_to(cpu_isolated_map);
sched_init_numa(); sched_init_numa();
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment