Commit 07793144 authored by Frederic Weisbecker's avatar Frederic Weisbecker

Merge branch 'nohz/printk-v8' into irq/core

Conflicts:
	kernel/irq_work.c

Add support for printk in full dynticks CPU.

* Don't stop tick with irq works pending. This
fix is generally useful and concerns archs that
can't raise self IPIs.

* Flush irq works before CPU offlining.

* Introduce "lazy" irq works that can wait for the
next tick to be executed, unless it's stopped.

* Implement klogd wake up using irq work. This
removes the ad-hoc printk_tick()/printk_needs_cpu()
hooks and make it working even in dynticks mode.
Signed-off-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
parents f7c819c0 74876a98
...@@ -3,6 +3,20 @@ ...@@ -3,6 +3,20 @@
#include <linux/llist.h> #include <linux/llist.h>
/*
* An entry can be in one of four states:
*
* free NULL, 0 -> {claimed} : free to be used
* claimed NULL, 3 -> {pending} : claimed to be enqueued
* pending next, 3 -> {busy} : queued, pending callback
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
*/
#define IRQ_WORK_PENDING 1UL
#define IRQ_WORK_BUSY 2UL
#define IRQ_WORK_FLAGS 3UL
#define IRQ_WORK_LAZY 4UL /* Doesn't want IPI, wait for tick */
struct irq_work { struct irq_work {
unsigned long flags; unsigned long flags;
struct llist_node llnode; struct llist_node llnode;
...@@ -20,4 +34,10 @@ void irq_work_queue(struct irq_work *work); ...@@ -20,4 +34,10 @@ void irq_work_queue(struct irq_work *work);
void irq_work_run(void); void irq_work_run(void);
void irq_work_sync(struct irq_work *work); void irq_work_sync(struct irq_work *work);
#ifdef CONFIG_IRQ_WORK
bool irq_work_needs_cpu(void);
#else
static bool irq_work_needs_cpu(void) { return false; }
#endif
#endif /* _LINUX_IRQ_WORK_H */ #endif /* _LINUX_IRQ_WORK_H */
...@@ -98,9 +98,6 @@ int no_printk(const char *fmt, ...) ...@@ -98,9 +98,6 @@ int no_printk(const char *fmt, ...)
extern asmlinkage __printf(1, 2) extern asmlinkage __printf(1, 2)
void early_printk(const char *fmt, ...); void early_printk(const char *fmt, ...);
extern int printk_needs_cpu(int cpu);
extern void printk_tick(void);
#ifdef CONFIG_PRINTK #ifdef CONFIG_PRINTK
asmlinkage __printf(5, 0) asmlinkage __printf(5, 0)
int vprintk_emit(int facility, int level, int vprintk_emit(int facility, int level,
......
...@@ -8,6 +8,8 @@ ...@@ -8,6 +8,8 @@
#include <linux/clockchips.h> #include <linux/clockchips.h>
#include <linux/irqflags.h> #include <linux/irqflags.h>
#include <linux/percpu.h>
#include <linux/hrtimer.h>
#ifdef CONFIG_GENERIC_CLOCKEVENTS #ifdef CONFIG_GENERIC_CLOCKEVENTS
...@@ -122,13 +124,26 @@ static inline int tick_oneshot_mode_active(void) { return 0; } ...@@ -122,13 +124,26 @@ static inline int tick_oneshot_mode_active(void) { return 0; }
#endif /* !CONFIG_GENERIC_CLOCKEVENTS */ #endif /* !CONFIG_GENERIC_CLOCKEVENTS */
# ifdef CONFIG_NO_HZ # ifdef CONFIG_NO_HZ
DECLARE_PER_CPU(struct tick_sched, tick_cpu_sched);
static inline int tick_nohz_tick_stopped(void)
{
return __this_cpu_read(tick_cpu_sched.tick_stopped);
}
extern void tick_nohz_idle_enter(void); extern void tick_nohz_idle_enter(void);
extern void tick_nohz_idle_exit(void); extern void tick_nohz_idle_exit(void);
extern void tick_nohz_irq_exit(void); extern void tick_nohz_irq_exit(void);
extern ktime_t tick_nohz_get_sleep_length(void); extern ktime_t tick_nohz_get_sleep_length(void);
extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
# else
# else /* !CONFIG_NO_HZ */
static inline int tick_nohz_tick_stopped(void)
{
return 0;
}
static inline void tick_nohz_idle_enter(void) { } static inline void tick_nohz_idle_enter(void) { }
static inline void tick_nohz_idle_exit(void) { } static inline void tick_nohz_idle_exit(void) { }
......
...@@ -1259,6 +1259,7 @@ config HOTPLUG ...@@ -1259,6 +1259,7 @@ config HOTPLUG
config PRINTK config PRINTK
default y default y
bool "Enable support for printk" if EXPERT bool "Enable support for printk" if EXPERT
select IRQ_WORK
help help
This option enables normal printk support. Removing it This option enables normal printk support. Removing it
eliminates most of the message strings from the kernel image eliminates most of the message strings from the kernel image
......
...@@ -12,22 +12,15 @@ ...@@ -12,22 +12,15 @@
#include <linux/percpu.h> #include <linux/percpu.h>
#include <linux/hardirq.h> #include <linux/hardirq.h>
#include <linux/irqflags.h> #include <linux/irqflags.h>
#include <linux/sched.h>
#include <linux/tick.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <asm/processor.h> #include <asm/processor.h>
/*
* An entry can be in one of four states:
*
* free NULL, 0 -> {claimed} : free to be used
* claimed NULL, 3 -> {pending} : claimed to be enqueued
* pending next, 3 -> {busy} : queued, pending callback
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
*/
#define IRQ_WORK_PENDING 1UL
#define IRQ_WORK_BUSY 2UL
#define IRQ_WORK_FLAGS 3UL
static DEFINE_PER_CPU(struct llist_head, irq_work_list); static DEFINE_PER_CPU(struct llist_head, irq_work_list);
static DEFINE_PER_CPU(int, irq_work_raised);
/* /*
* Claim the entry so that no one else will poke at it. * Claim the entry so that no one else will poke at it.
...@@ -70,8 +63,6 @@ void __weak arch_irq_work_raise(void) ...@@ -70,8 +63,6 @@ void __weak arch_irq_work_raise(void)
*/ */
void irq_work_queue(struct irq_work *work) void irq_work_queue(struct irq_work *work)
{ {
bool empty;
/* Only queue if not already pending */ /* Only queue if not already pending */
if (!irq_work_claim(work)) if (!irq_work_claim(work))
return; return;
...@@ -79,30 +70,55 @@ void irq_work_queue(struct irq_work *work) ...@@ -79,30 +70,55 @@ void irq_work_queue(struct irq_work *work)
/* Queue the entry and raise the IPI if needed. */ /* Queue the entry and raise the IPI if needed. */
preempt_disable(); preempt_disable();
empty = llist_add(&work->llnode, &__get_cpu_var(irq_work_list)); llist_add(&work->llnode, &__get_cpu_var(irq_work_list));
/* The list was empty, raise self-interrupt to start processing. */
if (empty) /*
* If the work is not "lazy" or the tick is stopped, raise the irq
* work interrupt (if supported by the arch), otherwise, just wait
* for the next tick.
*/
if (!(work->flags & IRQ_WORK_LAZY) || tick_nohz_tick_stopped()) {
if (!this_cpu_cmpxchg(irq_work_raised, 0, 1))
arch_irq_work_raise(); arch_irq_work_raise();
}
preempt_enable(); preempt_enable();
} }
EXPORT_SYMBOL_GPL(irq_work_queue); EXPORT_SYMBOL_GPL(irq_work_queue);
/* bool irq_work_needs_cpu(void)
* Run the irq_work entries on this cpu. Requires to be ran from hardirq
* context with local IRQs disabled.
*/
void irq_work_run(void)
{ {
struct llist_head *this_list;
this_list = &__get_cpu_var(irq_work_list);
if (llist_empty(this_list))
return false;
/* All work should have been flushed before going offline */
WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
return true;
}
static void __irq_work_run(void)
{
unsigned long flags;
struct irq_work *work; struct irq_work *work;
struct llist_head *this_list; struct llist_head *this_list;
struct llist_node *llnode; struct llist_node *llnode;
/*
* Reset the "raised" state right before we check the list because
* an NMI may enqueue after we find the list empty from the runner.
*/
__this_cpu_write(irq_work_raised, 0);
barrier();
this_list = &__get_cpu_var(irq_work_list); this_list = &__get_cpu_var(irq_work_list);
if (llist_empty(this_list)) if (llist_empty(this_list))
return; return;
BUG_ON(!in_irq());
BUG_ON(!irqs_disabled()); BUG_ON(!irqs_disabled());
llnode = llist_del_all(this_list); llnode = llist_del_all(this_list);
...@@ -118,15 +134,27 @@ void irq_work_run(void) ...@@ -118,15 +134,27 @@ void irq_work_run(void)
* to claim that work don't rely on us to handle their data * to claim that work don't rely on us to handle their data
* while we are in the middle of the func. * while we are in the middle of the func.
*/ */
xchg(&work->flags, IRQ_WORK_BUSY); flags = work->flags & ~IRQ_WORK_PENDING;
xchg(&work->flags, flags);
work->func(work); work->func(work);
/* /*
* Clear the BUSY bit and return to the free state if * Clear the BUSY bit and return to the free state if
* no-one else claimed it meanwhile. * no-one else claimed it meanwhile.
*/ */
(void)cmpxchg(&work->flags, IRQ_WORK_BUSY, 0); (void)cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY);
} }
} }
/*
* Run the irq_work entries on this cpu. Requires to be ran from hardirq
* context with local IRQs disabled.
*/
void irq_work_run(void)
{
BUG_ON(!in_irq());
__irq_work_run();
}
EXPORT_SYMBOL_GPL(irq_work_run); EXPORT_SYMBOL_GPL(irq_work_run);
/* /*
...@@ -141,3 +169,35 @@ void irq_work_sync(struct irq_work *work) ...@@ -141,3 +169,35 @@ void irq_work_sync(struct irq_work *work)
cpu_relax(); cpu_relax();
} }
EXPORT_SYMBOL_GPL(irq_work_sync); EXPORT_SYMBOL_GPL(irq_work_sync);
#ifdef CONFIG_HOTPLUG_CPU
static int irq_work_cpu_notify(struct notifier_block *self,
unsigned long action, void *hcpu)
{
long cpu = (long)hcpu;
switch (action) {
case CPU_DYING:
/* Called from stop_machine */
if (WARN_ON_ONCE(cpu != smp_processor_id()))
break;
__irq_work_run();
break;
default:
break;
}
return NOTIFY_OK;
}
static struct notifier_block cpu_notify;
static __init int irq_work_init_cpu_notifier(void)
{
cpu_notify.notifier_call = irq_work_cpu_notify;
cpu_notify.priority = 0;
register_cpu_notifier(&cpu_notify);
return 0;
}
device_initcall(irq_work_init_cpu_notifier);
#endif /* CONFIG_HOTPLUG_CPU */
...@@ -42,6 +42,7 @@ ...@@ -42,6 +42,7 @@
#include <linux/notifier.h> #include <linux/notifier.h>
#include <linux/rculist.h> #include <linux/rculist.h>
#include <linux/poll.h> #include <linux/poll.h>
#include <linux/irq_work.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
...@@ -1967,30 +1968,32 @@ int is_console_locked(void) ...@@ -1967,30 +1968,32 @@ int is_console_locked(void)
static DEFINE_PER_CPU(int, printk_pending); static DEFINE_PER_CPU(int, printk_pending);
static DEFINE_PER_CPU(char [PRINTK_BUF_SIZE], printk_sched_buf); static DEFINE_PER_CPU(char [PRINTK_BUF_SIZE], printk_sched_buf);
void printk_tick(void) static void wake_up_klogd_work_func(struct irq_work *irq_work)
{ {
if (__this_cpu_read(printk_pending)) {
int pending = __this_cpu_xchg(printk_pending, 0); int pending = __this_cpu_xchg(printk_pending, 0);
if (pending & PRINTK_PENDING_SCHED) { if (pending & PRINTK_PENDING_SCHED) {
char *buf = __get_cpu_var(printk_sched_buf); char *buf = __get_cpu_var(printk_sched_buf);
printk(KERN_WARNING "[sched_delayed] %s", buf); printk(KERN_WARNING "[sched_delayed] %s", buf);
} }
if (pending & PRINTK_PENDING_WAKEUP) if (pending & PRINTK_PENDING_WAKEUP)
wake_up_interruptible(&log_wait); wake_up_interruptible(&log_wait);
}
} }
int printk_needs_cpu(int cpu) static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) = {
{ .func = wake_up_klogd_work_func,
if (cpu_is_offline(cpu)) .flags = IRQ_WORK_LAZY,
printk_tick(); };
return __this_cpu_read(printk_pending);
}
void wake_up_klogd(void) void wake_up_klogd(void)
{ {
if (waitqueue_active(&log_wait)) preempt_disable();
if (waitqueue_active(&log_wait)) {
this_cpu_or(printk_pending, PRINTK_PENDING_WAKEUP); this_cpu_or(printk_pending, PRINTK_PENDING_WAKEUP);
irq_work_queue(&__get_cpu_var(wake_up_klogd_work));
}
preempt_enable();
} }
static void console_cont_flush(char *text, size_t size) static void console_cont_flush(char *text, size_t size)
...@@ -2471,6 +2474,7 @@ int printk_sched(const char *fmt, ...) ...@@ -2471,6 +2474,7 @@ int printk_sched(const char *fmt, ...)
va_end(args); va_end(args);
__this_cpu_or(printk_pending, PRINTK_PENDING_SCHED); __this_cpu_or(printk_pending, PRINTK_PENDING_SCHED);
irq_work_queue(&__get_cpu_var(wake_up_klogd_work));
local_irq_restore(flags); local_irq_restore(flags);
return r; return r;
......
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
#include <linux/profile.h> #include <linux/profile.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/irq_work.h>
#include <asm/irq_regs.h> #include <asm/irq_regs.h>
...@@ -28,7 +29,7 @@ ...@@ -28,7 +29,7 @@
/* /*
* Per cpu nohz control structure * Per cpu nohz control structure
*/ */
static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched); DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
/* /*
* The time, when the last jiffy update happened. Protected by jiffies_lock. * The time, when the last jiffy update happened. Protected by jiffies_lock.
...@@ -331,8 +332,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, ...@@ -331,8 +332,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
time_delta = timekeeping_max_deferment(); time_delta = timekeeping_max_deferment();
} while (read_seqretry(&jiffies_lock, seq)); } while (read_seqretry(&jiffies_lock, seq));
if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || printk_needs_cpu(cpu) || if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) ||
arch_needs_cpu(cpu)) { arch_needs_cpu(cpu) || irq_work_needs_cpu()) {
next_jiffies = last_jiffies + 1; next_jiffies = last_jiffies + 1;
delta_jiffies = 1; delta_jiffies = 1;
} else { } else {
......
...@@ -1351,7 +1351,6 @@ void update_process_times(int user_tick) ...@@ -1351,7 +1351,6 @@ void update_process_times(int user_tick)
account_process_tick(p, user_tick); account_process_tick(p, user_tick);
run_local_timers(); run_local_timers();
rcu_check_callbacks(cpu, user_tick); rcu_check_callbacks(cpu, user_tick);
printk_tick();
#ifdef CONFIG_IRQ_WORK #ifdef CONFIG_IRQ_WORK
if (in_irq()) if (in_irq())
irq_work_run(); irq_work_run();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment