Commit f4990264 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'x86-hyperv-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86/hyper-v update from Thomas Gleixner:
 "Add fast hypercall support for guest running on the Microsoft HyperV(isor)"

* 'x86-hyperv-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/hyper-v: Fix wrong merge conflict resolution
  x86/hyper-v: Check for VP_INVAL in hyperv_flush_tlb_others()
  x86/hyper-v: Check cpumask_to_vpset() return value in hyperv_flush_tlb_others_ex()
  x86/hyper-v: Trace PV IPI send
  x86/hyper-v: Use cheaper HVCALL_SEND_IPI hypercall when possible
  x86/hyper-v: Use 'fast' hypercall for HVCALL_SEND_IPI
  x86/hyper-v: Implement hv_do_fast_hypercall16
  x86/hyper-v: Use cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE} hypercalls when possible
parents 27a52501 be0e16ce
...@@ -31,6 +31,8 @@ ...@@ -31,6 +31,8 @@
#include <asm/mshyperv.h> #include <asm/mshyperv.h>
#include <asm/apic.h> #include <asm/apic.h>
#include <asm/trace/hyperv.h>
static struct apic orig_apic; static struct apic orig_apic;
static u64 hv_apic_icr_read(void) static u64 hv_apic_icr_read(void)
...@@ -99,6 +101,9 @@ static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector) ...@@ -99,6 +101,9 @@ static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector)
int nr_bank = 0; int nr_bank = 0;
int ret = 1; int ret = 1;
if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
return false;
local_irq_save(flags); local_irq_save(flags);
arg = (struct ipi_arg_ex **)this_cpu_ptr(hyperv_pcpu_input_arg); arg = (struct ipi_arg_ex **)this_cpu_ptr(hyperv_pcpu_input_arg);
...@@ -130,10 +135,10 @@ static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector) ...@@ -130,10 +135,10 @@ static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector)
static bool __send_ipi_mask(const struct cpumask *mask, int vector) static bool __send_ipi_mask(const struct cpumask *mask, int vector)
{ {
int cur_cpu, vcpu; int cur_cpu, vcpu;
struct ipi_arg_non_ex **arg; struct ipi_arg_non_ex ipi_arg;
struct ipi_arg_non_ex *ipi_arg;
int ret = 1; int ret = 1;
unsigned long flags;
trace_hyperv_send_ipi_mask(mask, vector);
if (cpumask_empty(mask)) if (cpumask_empty(mask))
return true; return true;
...@@ -144,40 +149,43 @@ static bool __send_ipi_mask(const struct cpumask *mask, int vector) ...@@ -144,40 +149,43 @@ static bool __send_ipi_mask(const struct cpumask *mask, int vector)
if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR)) if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR))
return false; return false;
if ((ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) /*
return __send_ipi_mask_ex(mask, vector); * From the supplied CPU set we need to figure out if we can get away
* with cheaper HVCALL_SEND_IPI hypercall. This is possible when the
local_irq_save(flags); * highest VP number in the set is < 64. As VP numbers are usually in
arg = (struct ipi_arg_non_ex **)this_cpu_ptr(hyperv_pcpu_input_arg); * ascending order and match Linux CPU ids, here is an optimization:
* we check the VP number for the highest bit in the supplied set first
ipi_arg = *arg; * so we can quickly find out if using HVCALL_SEND_IPI_EX hypercall is
if (unlikely(!ipi_arg)) * a must. We will also check all VP numbers when walking the supplied
goto ipi_mask_done; * CPU set to remain correct in all cases.
*/
if (hv_cpu_number_to_vp_number(cpumask_last(mask)) >= 64)
goto do_ex_hypercall;
ipi_arg->vector = vector; ipi_arg.vector = vector;
ipi_arg->reserved = 0; ipi_arg.cpu_mask = 0;
ipi_arg->cpu_mask = 0;
for_each_cpu(cur_cpu, mask) { for_each_cpu(cur_cpu, mask) {
vcpu = hv_cpu_number_to_vp_number(cur_cpu); vcpu = hv_cpu_number_to_vp_number(cur_cpu);
if (vcpu == VP_INVAL) if (vcpu == VP_INVAL)
goto ipi_mask_done; return false;
/* /*
* This particular version of the IPI hypercall can * This particular version of the IPI hypercall can
* only target upto 64 CPUs. * only target upto 64 CPUs.
*/ */
if (vcpu >= 64) if (vcpu >= 64)
goto ipi_mask_done; goto do_ex_hypercall;
__set_bit(vcpu, (unsigned long *)&ipi_arg->cpu_mask); __set_bit(vcpu, (unsigned long *)&ipi_arg.cpu_mask);
} }
ret = hv_do_hypercall(HVCALL_SEND_IPI, ipi_arg, NULL); ret = hv_do_fast_hypercall16(HVCALL_SEND_IPI, ipi_arg.vector,
ipi_arg.cpu_mask);
ipi_mask_done:
local_irq_restore(flags);
return ((ret == 0) ? true : false); return ((ret == 0) ? true : false);
do_ex_hypercall:
return __send_ipi_mask_ex(mask, vector);
} }
static bool __send_ipi_one(int cpu, int vector) static bool __send_ipi_one(int cpu, int vector)
...@@ -233,9 +241,6 @@ static void hv_send_ipi_self(int vector) ...@@ -233,9 +241,6 @@ static void hv_send_ipi_self(int vector)
void __init hv_apic_init(void) void __init hv_apic_init(void)
{ {
if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) { if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) {
if ((ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
pr_info("Hyper-V: Using ext hypercalls for IPI\n");
else
pr_info("Hyper-V: Using IPI hypercalls\n"); pr_info("Hyper-V: Using IPI hypercalls\n");
/* /*
* Set the IPI entry points. * Set the IPI entry points.
......
...@@ -16,6 +16,8 @@ ...@@ -16,6 +16,8 @@
/* Each gva in gva_list encodes up to 4096 pages to flush */ /* Each gva in gva_list encodes up to 4096 pages to flush */
#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE) #define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
const struct flush_tlb_info *info);
/* /*
* Fills in gva_list starting from offset. Returns the number of items added. * Fills in gva_list starting from offset. Returns the number of items added.
...@@ -93,10 +95,29 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus, ...@@ -93,10 +95,29 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus,
if (cpumask_equal(cpus, cpu_present_mask)) { if (cpumask_equal(cpus, cpu_present_mask)) {
flush->flags |= HV_FLUSH_ALL_PROCESSORS; flush->flags |= HV_FLUSH_ALL_PROCESSORS;
} else { } else {
/*
* From the supplied CPU set we need to figure out if we can get
* away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE}
* hypercalls. This is possible when the highest VP number in
* the set is < 64. As VP numbers are usually in ascending order
* and match Linux CPU ids, here is an optimization: we check
* the VP number for the highest bit in the supplied set first
* so we can quickly find out if using *_EX hypercalls is a
* must. We will also check all VP numbers when walking the
* supplied CPU set to remain correct in all cases.
*/
if (hv_cpu_number_to_vp_number(cpumask_last(cpus)) >= 64)
goto do_ex_hypercall;
for_each_cpu(cpu, cpus) { for_each_cpu(cpu, cpus) {
vcpu = hv_cpu_number_to_vp_number(cpu); vcpu = hv_cpu_number_to_vp_number(cpu);
if (vcpu >= 64) if (vcpu == VP_INVAL) {
local_irq_restore(flags);
goto do_native; goto do_native;
}
if (vcpu >= 64)
goto do_ex_hypercall;
__set_bit(vcpu, (unsigned long *) __set_bit(vcpu, (unsigned long *)
&flush->processor_mask); &flush->processor_mask);
...@@ -123,7 +144,12 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus, ...@@ -123,7 +144,12 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus,
status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST, status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
gva_n, 0, flush, NULL); gva_n, 0, flush, NULL);
} }
goto check_status;
do_ex_hypercall:
status = hyperv_flush_tlb_others_ex(cpus, info);
check_status:
local_irq_restore(flags); local_irq_restore(flags);
if (!(status & HV_HYPERCALL_RESULT_MASK)) if (!(status & HV_HYPERCALL_RESULT_MASK))
...@@ -132,35 +158,22 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus, ...@@ -132,35 +158,22 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus,
native_flush_tlb_others(cpus, info); native_flush_tlb_others(cpus, info);
} }
static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus, static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
const struct flush_tlb_info *info) const struct flush_tlb_info *info)
{ {
int nr_bank = 0, max_gvas, gva_n; int nr_bank = 0, max_gvas, gva_n;
struct hv_tlb_flush_ex **flush_pcpu; struct hv_tlb_flush_ex **flush_pcpu;
struct hv_tlb_flush_ex *flush; struct hv_tlb_flush_ex *flush;
u64 status = U64_MAX; u64 status;
unsigned long flags;
trace_hyperv_mmu_flush_tlb_others(cpus, info);
if (!hv_hypercall_pg) if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
goto do_native; return U64_MAX;
if (cpumask_empty(cpus))
return;
local_irq_save(flags);
flush_pcpu = (struct hv_tlb_flush_ex **) flush_pcpu = (struct hv_tlb_flush_ex **)
this_cpu_ptr(hyperv_pcpu_input_arg); this_cpu_ptr(hyperv_pcpu_input_arg);
flush = *flush_pcpu; flush = *flush_pcpu;
if (unlikely(!flush)) {
local_irq_restore(flags);
goto do_native;
}
if (info->mm) { if (info->mm) {
/* /*
* AddressSpace argument must match the CR3 with PCID bits * AddressSpace argument must match the CR3 with PCID bits
...@@ -176,15 +189,10 @@ static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus, ...@@ -176,15 +189,10 @@ static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
flush->hv_vp_set.valid_bank_mask = 0; flush->hv_vp_set.valid_bank_mask = 0;
if (!cpumask_equal(cpus, cpu_present_mask)) {
flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K; flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus); nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus);
} if (nr_bank < 0)
return U64_MAX;
if (!nr_bank) {
flush->hv_vp_set.format = HV_GENERIC_SET_ALL;
flush->flags |= HV_FLUSH_ALL_PROCESSORS;
}
/* /*
* We can flush not more than max_gvas with one hypercall. Flush the * We can flush not more than max_gvas with one hypercall. Flush the
...@@ -213,12 +221,7 @@ static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus, ...@@ -213,12 +221,7 @@ static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
gva_n, nr_bank, flush, NULL); gva_n, nr_bank, flush, NULL);
} }
local_irq_restore(flags); return status;
if (!(status & HV_HYPERCALL_RESULT_MASK))
return;
do_native:
native_flush_tlb_others(cpus, info);
} }
void hyperv_setup_mmu_ops(void) void hyperv_setup_mmu_ops(void)
...@@ -226,11 +229,6 @@ void hyperv_setup_mmu_ops(void) ...@@ -226,11 +229,6 @@ void hyperv_setup_mmu_ops(void)
if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED)) if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
return; return;
if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) {
pr_info("Using hypercall for remote TLB flush\n"); pr_info("Using hypercall for remote TLB flush\n");
pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others; pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
} else {
pr_info("Using ext hypercall for remote TLB flush\n");
pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others_ex;
}
} }
...@@ -194,6 +194,40 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) ...@@ -194,6 +194,40 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
return hv_status; return hv_status;
} }
/* Fast hypercall with 16 bytes of input */
static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
{
u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT;
#ifdef CONFIG_X86_64
{
__asm__ __volatile__("mov %4, %%r8\n"
CALL_NOSPEC
: "=a" (hv_status), ASM_CALL_CONSTRAINT,
"+c" (control), "+d" (input1)
: "r" (input2),
THUNK_TARGET(hv_hypercall_pg)
: "cc", "r8", "r9", "r10", "r11");
}
#else
{
u32 input1_hi = upper_32_bits(input1);
u32 input1_lo = lower_32_bits(input1);
u32 input2_hi = upper_32_bits(input2);
u32 input2_lo = lower_32_bits(input2);
__asm__ __volatile__ (CALL_NOSPEC
: "=A"(hv_status),
"+c"(input1_lo), ASM_CALL_CONSTRAINT
: "A" (control), "b" (input1_hi),
"D"(input2_hi), "S"(input2_lo),
THUNK_TARGET(hv_hypercall_pg)
: "cc");
}
#endif
return hv_status;
}
/* /*
* Rep hypercalls. Callers of this functions are supposed to ensure that * Rep hypercalls. Callers of this functions are supposed to ensure that
* rep_count and varhead_size comply with Hyper-V hypercall definition. * rep_count and varhead_size comply with Hyper-V hypercall definition.
......
...@@ -28,6 +28,21 @@ TRACE_EVENT(hyperv_mmu_flush_tlb_others, ...@@ -28,6 +28,21 @@ TRACE_EVENT(hyperv_mmu_flush_tlb_others,
__entry->addr, __entry->end) __entry->addr, __entry->end)
); );
TRACE_EVENT(hyperv_send_ipi_mask,
TP_PROTO(const struct cpumask *cpus,
int vector),
TP_ARGS(cpus, vector),
TP_STRUCT__entry(
__field(unsigned int, ncpus)
__field(int, vector)
),
TP_fast_assign(__entry->ncpus = cpumask_weight(cpus);
__entry->vector = vector;
),
TP_printk("ncpus %d vector %x",
__entry->ncpus, __entry->vector)
);
#endif /* CONFIG_HYPERV */ #endif /* CONFIG_HYPERV */
#undef TRACE_INCLUDE_PATH #undef TRACE_INCLUDE_PATH
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment