Commit 3491caf2 authored by Christian Borntraeger's avatar Christian Borntraeger Committed by Paolo Bonzini

KVM: halt_polling: provide a way to qualify wakeups during poll

Some wakeups should not be considered a sucessful poll. For example on
s390 I/O interrupts are usually floating, which means that _ALL_ CPUs
would be considered runnable - letting all vCPUs poll all the time for
transactional like workload, even if one vCPU would be enough.
This can result in huge CPU usage for large guests.
This patch lets architectures provide a way to qualify wakeups if they
should be considered a good/bad wakeups in regard to polls.

For s390 the implementation will fence of halt polling for anything but
known good, single vCPU events. The s390 implementation for floating
interrupts does a wakeup for one vCPU, but the interrupt will be delivered
by whatever CPU checks first for a pending interrupt. We prefer the
woken up CPU by marking the poll of this CPU as "good" poll.
This code will also mark several other wakeup reasons like IPI or
expired timers as "good". This will of course also mark some events as
not sucessful. As  KVM on z runs always as a 2nd level hypervisor,
we prefer to not poll, unless we are really sure, though.

This patch successfully limits the CPU usage for cases like uperf 1byte
transactional ping pong workload or wakeup heavy workload like OLTP
while still providing a proper speedup.

This also introduced a new vcpu stat "halt_poll_no_tuning" that marks
wakeups that are considered not good for polling.
Signed-off-by: default avatarChristian Borntraeger <borntraeger@de.ibm.com>
Acked-by: Radim Krčmář <rkrcmar@redhat.com> (for an earlier version)
Cc: David Matlack <dmatlack@google.com>
Cc: Wanpeng Li <kernellwp@gmail.com>
[Rename config symbol. - Paolo]
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent d7e1633a
...@@ -187,6 +187,7 @@ struct kvm_vm_stat { ...@@ -187,6 +187,7 @@ struct kvm_vm_stat {
struct kvm_vcpu_stat { struct kvm_vcpu_stat {
u32 halt_successful_poll; u32 halt_successful_poll;
u32 halt_attempted_poll; u32 halt_attempted_poll;
u32 halt_poll_invalid;
u32 halt_wakeup; u32 halt_wakeup;
u32 hvc_exit_stat; u32 hvc_exit_stat;
u64 wfe_exit_stat; u64 wfe_exit_stat;
...@@ -282,6 +283,7 @@ static inline void kvm_arch_hardware_unsetup(void) {} ...@@ -282,6 +283,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {}
static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
static inline void kvm_arm_init_debug(void) {} static inline void kvm_arm_init_debug(void) {}
static inline void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) {} static inline void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) {}
......
...@@ -293,6 +293,7 @@ struct kvm_vm_stat { ...@@ -293,6 +293,7 @@ struct kvm_vm_stat {
struct kvm_vcpu_stat { struct kvm_vcpu_stat {
u32 halt_successful_poll; u32 halt_successful_poll;
u32 halt_attempted_poll; u32 halt_attempted_poll;
u32 halt_poll_invalid;
u32 halt_wakeup; u32 halt_wakeup;
u32 hvc_exit_stat; u32 hvc_exit_stat;
u64 wfe_exit_stat; u64 wfe_exit_stat;
...@@ -357,6 +358,7 @@ static inline void kvm_arch_hardware_unsetup(void) {} ...@@ -357,6 +358,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {}
static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
void kvm_arm_init_debug(void); void kvm_arm_init_debug(void);
void kvm_arm_setup_debug(struct kvm_vcpu *vcpu); void kvm_arm_setup_debug(struct kvm_vcpu *vcpu);
......
...@@ -122,6 +122,7 @@ struct kvm_vcpu_stat { ...@@ -122,6 +122,7 @@ struct kvm_vcpu_stat {
u32 flush_dcache_exits; u32 flush_dcache_exits;
u32 halt_successful_poll; u32 halt_successful_poll;
u32 halt_attempted_poll; u32 halt_attempted_poll;
u32 halt_poll_invalid;
u32 halt_wakeup; u32 halt_wakeup;
}; };
...@@ -812,5 +813,6 @@ static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} ...@@ -812,5 +813,6 @@ static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
#endif /* __MIPS_KVM_HOST_H__ */ #endif /* __MIPS_KVM_HOST_H__ */
...@@ -56,6 +56,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { ...@@ -56,6 +56,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
{ "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU }, { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU }, { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU },
{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), KVM_STAT_VCPU }, { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), KVM_STAT_VCPU },
{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid), KVM_STAT_VCPU },
{ "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU }, { "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU },
{NULL} {NULL}
}; };
......
...@@ -116,6 +116,7 @@ struct kvm_vcpu_stat { ...@@ -116,6 +116,7 @@ struct kvm_vcpu_stat {
u32 ext_intr_exits; u32 ext_intr_exits;
u32 halt_successful_poll; u32 halt_successful_poll;
u32 halt_attempted_poll; u32 halt_attempted_poll;
u32 halt_poll_invalid;
u32 halt_wakeup; u32 halt_wakeup;
u32 dbell_exits; u32 dbell_exits;
u32 gdbell_exits; u32 gdbell_exits;
...@@ -727,5 +728,6 @@ static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} ...@@ -727,5 +728,6 @@ static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
static inline void kvm_arch_exit(void) {} static inline void kvm_arch_exit(void) {}
static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
#endif /* __POWERPC_KVM_HOST_H__ */ #endif /* __POWERPC_KVM_HOST_H__ */
...@@ -54,6 +54,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { ...@@ -54,6 +54,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
{ "queue_intr", VCPU_STAT(queue_intr) }, { "queue_intr", VCPU_STAT(queue_intr) },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll), }, { "halt_successful_poll", VCPU_STAT(halt_successful_poll), },
{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), }, { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), },
{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
{ "halt_wakeup", VCPU_STAT(halt_wakeup) }, { "halt_wakeup", VCPU_STAT(halt_wakeup) },
{ "pf_storage", VCPU_STAT(pf_storage) }, { "pf_storage", VCPU_STAT(pf_storage) },
{ "sp_storage", VCPU_STAT(sp_storage) }, { "sp_storage", VCPU_STAT(sp_storage) },
......
...@@ -64,6 +64,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { ...@@ -64,6 +64,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
{ "ext_intr", VCPU_STAT(ext_intr_exits) }, { "ext_intr", VCPU_STAT(ext_intr_exits) },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
{ "halt_wakeup", VCPU_STAT(halt_wakeup) }, { "halt_wakeup", VCPU_STAT(halt_wakeup) },
{ "doorbell", VCPU_STAT(dbell_exits) }, { "doorbell", VCPU_STAT(dbell_exits) },
{ "guest doorbell", VCPU_STAT(gdbell_exits) }, { "guest doorbell", VCPU_STAT(gdbell_exits) },
......
...@@ -247,6 +247,7 @@ struct kvm_vcpu_stat { ...@@ -247,6 +247,7 @@ struct kvm_vcpu_stat {
u32 exit_instruction; u32 exit_instruction;
u32 halt_successful_poll; u32 halt_successful_poll;
u32 halt_attempted_poll; u32 halt_attempted_poll;
u32 halt_poll_invalid;
u32 halt_wakeup; u32 halt_wakeup;
u32 instruction_lctl; u32 instruction_lctl;
u32 instruction_lctlg; u32 instruction_lctlg;
...@@ -696,4 +697,6 @@ static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, ...@@ -696,4 +697,6 @@ static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu);
#endif #endif
...@@ -28,6 +28,7 @@ config KVM ...@@ -28,6 +28,7 @@ config KVM
select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQCHIP
select HAVE_KVM_IRQFD select HAVE_KVM_IRQFD
select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_IRQ_ROUTING
select HAVE_KVM_INVALID_WAKEUPS
select SRCU select SRCU
select KVM_VFIO select KVM_VFIO
---help--- ---help---
......
...@@ -977,6 +977,11 @@ int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) ...@@ -977,6 +977,11 @@ int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
{ {
/*
* We cannot move this into the if, as the CPU might be already
* in kvm_vcpu_block without having the waitqueue set (polling)
*/
vcpu->valid_wakeup = true;
if (swait_active(&vcpu->wq)) { if (swait_active(&vcpu->wq)) {
/* /*
* The vcpu gave up the cpu voluntarily, mark it as a good * The vcpu gave up the cpu voluntarily, mark it as a good
......
...@@ -65,6 +65,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { ...@@ -65,6 +65,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
{ "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
{ "halt_wakeup", VCPU_STAT(halt_wakeup) }, { "halt_wakeup", VCPU_STAT(halt_wakeup) },
{ "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
{ "instruction_lctl", VCPU_STAT(instruction_lctl) }, { "instruction_lctl", VCPU_STAT(instruction_lctl) },
...@@ -2992,6 +2993,11 @@ static inline unsigned long nonhyp_mask(int i) ...@@ -2992,6 +2993,11 @@ static inline unsigned long nonhyp_mask(int i)
return 0x0000ffffffffffffUL >> (nonhyp_fai << 4); return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
} }
void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
{
vcpu->valid_wakeup = false;
}
static int __init kvm_s390_init(void) static int __init kvm_s390_init(void)
{ {
int i; int i;
......
...@@ -803,6 +803,7 @@ struct kvm_vcpu_stat { ...@@ -803,6 +803,7 @@ struct kvm_vcpu_stat {
u32 halt_exits; u32 halt_exits;
u32 halt_successful_poll; u32 halt_successful_poll;
u32 halt_attempted_poll; u32 halt_attempted_poll;
u32 halt_poll_invalid;
u32 halt_wakeup; u32 halt_wakeup;
u32 request_irq_exits; u32 request_irq_exits;
u32 irq_exits; u32 irq_exits;
...@@ -1342,5 +1343,6 @@ void kvm_set_msi_irq(struct kvm_kernel_irq_routing_entry *e, ...@@ -1342,5 +1343,6 @@ void kvm_set_msi_irq(struct kvm_kernel_irq_routing_entry *e,
static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
#endif /* _ASM_X86_KVM_HOST_H */ #endif /* _ASM_X86_KVM_HOST_H */
...@@ -161,6 +161,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { ...@@ -161,6 +161,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
{ "halt_exits", VCPU_STAT(halt_exits) }, { "halt_exits", VCPU_STAT(halt_exits) },
{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
{ "halt_wakeup", VCPU_STAT(halt_wakeup) }, { "halt_wakeup", VCPU_STAT(halt_wakeup) },
{ "hypercalls", VCPU_STAT(hypercalls) }, { "hypercalls", VCPU_STAT(hypercalls) },
{ "request_irq", VCPU_STAT(request_irq_exits) }, { "request_irq", VCPU_STAT(request_irq_exits) },
......
...@@ -229,6 +229,7 @@ struct kvm_vcpu { ...@@ -229,6 +229,7 @@ struct kvm_vcpu {
sigset_t sigset; sigset_t sigset;
struct kvm_vcpu_stat stat; struct kvm_vcpu_stat stat;
unsigned int halt_poll_ns; unsigned int halt_poll_ns;
bool valid_wakeup;
#ifdef CONFIG_HAS_IOMEM #ifdef CONFIG_HAS_IOMEM
int mmio_needed; int mmio_needed;
...@@ -1196,4 +1197,18 @@ int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq, ...@@ -1196,4 +1197,18 @@ int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
uint32_t guest_irq, bool set); uint32_t guest_irq, bool set);
#endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */ #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */
#ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS
/* If we wakeup during the poll time, was it a sucessful poll? */
static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu)
{
return vcpu->valid_wakeup;
}
#else
static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu)
{
return true;
}
#endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */
#endif #endif
...@@ -38,22 +38,25 @@ TRACE_EVENT(kvm_userspace_exit, ...@@ -38,22 +38,25 @@ TRACE_EVENT(kvm_userspace_exit,
); );
TRACE_EVENT(kvm_vcpu_wakeup, TRACE_EVENT(kvm_vcpu_wakeup,
TP_PROTO(__u64 ns, bool waited), TP_PROTO(__u64 ns, bool waited, bool valid),
TP_ARGS(ns, waited), TP_ARGS(ns, waited, valid),
TP_STRUCT__entry( TP_STRUCT__entry(
__field( __u64, ns ) __field( __u64, ns )
__field( bool, waited ) __field( bool, waited )
__field( bool, valid )
), ),
TP_fast_assign( TP_fast_assign(
__entry->ns = ns; __entry->ns = ns;
__entry->waited = waited; __entry->waited = waited;
__entry->valid = valid;
), ),
TP_printk("%s time %lld ns", TP_printk("%s time %lld ns, polling %s",
__entry->waited ? "wait" : "poll", __entry->waited ? "wait" : "poll",
__entry->ns) __entry->ns,
__entry->valid ? "valid" : "invalid")
); );
#if defined(CONFIG_HAVE_KVM_IRQFD) #if defined(CONFIG_HAVE_KVM_IRQFD)
......
...@@ -41,6 +41,9 @@ config KVM_VFIO ...@@ -41,6 +41,9 @@ config KVM_VFIO
config HAVE_KVM_ARCH_TLB_FLUSH_ALL config HAVE_KVM_ARCH_TLB_FLUSH_ALL
bool bool
config HAVE_KVM_INVALID_WAKEUPS
bool
config KVM_GENERIC_DIRTYLOG_READ_PROTECT config KVM_GENERIC_DIRTYLOG_READ_PROTECT
bool bool
......
...@@ -2028,6 +2028,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) ...@@ -2028,6 +2028,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
*/ */
if (kvm_vcpu_check_block(vcpu) < 0) { if (kvm_vcpu_check_block(vcpu) < 0) {
++vcpu->stat.halt_successful_poll; ++vcpu->stat.halt_successful_poll;
if (!vcpu_valid_wakeup(vcpu))
++vcpu->stat.halt_poll_invalid;
goto out; goto out;
} }
cur = ktime_get(); cur = ktime_get();
...@@ -2057,7 +2059,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) ...@@ -2057,7 +2059,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
if (block_ns <= vcpu->halt_poll_ns) if (block_ns <= vcpu->halt_poll_ns)
; ;
/* we had a long block, shrink polling */ /* we had a long block, shrink polling */
else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns) else if (!vcpu_valid_wakeup(vcpu) ||
(vcpu->halt_poll_ns && block_ns > halt_poll_ns))
shrink_halt_poll_ns(vcpu); shrink_halt_poll_ns(vcpu);
/* we had a short halt and our poll time is too small */ /* we had a short halt and our poll time is too small */
else if (vcpu->halt_poll_ns < halt_poll_ns && else if (vcpu->halt_poll_ns < halt_poll_ns &&
...@@ -2066,7 +2069,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) ...@@ -2066,7 +2069,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
} else } else
vcpu->halt_poll_ns = 0; vcpu->halt_poll_ns = 0;
trace_kvm_vcpu_wakeup(block_ns, waited); trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
kvm_arch_vcpu_block_finish(vcpu);
} }
EXPORT_SYMBOL_GPL(kvm_vcpu_block); EXPORT_SYMBOL_GPL(kvm_vcpu_block);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment