Commit 9d0b048d authored by Suraj Jitindar Singh's avatar Suraj Jitindar Singh Committed by Michael Ellerman

KVM: PPC: Book3S HV: Invalidate TLB when nested vcpu moves physical cpu

This is only done at level 0, since only level 0 knows which physical
CPU a vcpu is running on.  This does for nested guests what L0 already
did for its own guests, which is to flush the TLB on a pCPU when it
goes to run a vCPU there, and there is another vCPU in the same VM
which previously ran on this pCPU and has now started to run on another
pCPU.  This is to handle the situation where the other vCPU touched
a mapping, moved to another pCPU and did a tlbiel (local-only tlbie)
on that new pCPU and thus left behind a stale TLB entry on this pCPU.

This introduces a limit on the the vcpu_token values used in the
H_ENTER_NESTED hcall -- they must now be less than NR_CPUS.

[paulus@ozlabs.org - made prev_cpu array be short[] to reduce
 memory consumption.]
Reviewed-by: default avatarDavid Gibson <david@gibson.dropbear.id.au>
Signed-off-by: default avatarSuraj Jitindar Singh <sjitindarsingh@gmail.com>
Signed-off-by: default avatarPaul Mackerras <paulus@ozlabs.org>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent 690ed4ca
...@@ -52,6 +52,9 @@ struct kvm_nested_guest { ...@@ -52,6 +52,9 @@ struct kvm_nested_guest {
long refcnt; /* number of pointers to this struct */ long refcnt; /* number of pointers to this struct */
struct mutex tlb_lock; /* serialize page faults and tlbies */ struct mutex tlb_lock; /* serialize page faults and tlbies */
struct kvm_nested_guest *next; struct kvm_nested_guest *next;
cpumask_t need_tlb_flush;
cpumask_t cpu_in_guest;
short prev_cpu[NR_CPUS];
}; };
/* /*
......
...@@ -2397,10 +2397,18 @@ static void kvmppc_release_hwthread(int cpu) ...@@ -2397,10 +2397,18 @@ static void kvmppc_release_hwthread(int cpu)
static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu) static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
{ {
struct kvm_nested_guest *nested = vcpu->arch.nested;
cpumask_t *cpu_in_guest;
int i; int i;
cpu = cpu_first_thread_sibling(cpu); cpu = cpu_first_thread_sibling(cpu);
cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush); if (nested) {
cpumask_set_cpu(cpu, &nested->need_tlb_flush);
cpu_in_guest = &nested->cpu_in_guest;
} else {
cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush);
cpu_in_guest = &kvm->arch.cpu_in_guest;
}
/* /*
* Make sure setting of bit in need_tlb_flush precedes * Make sure setting of bit in need_tlb_flush precedes
* testing of cpu_in_guest bits. The matching barrier on * testing of cpu_in_guest bits. The matching barrier on
...@@ -2408,13 +2416,23 @@ static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu) ...@@ -2408,13 +2416,23 @@ static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
*/ */
smp_mb(); smp_mb();
for (i = 0; i < threads_per_core; ++i) for (i = 0; i < threads_per_core; ++i)
if (cpumask_test_cpu(cpu + i, &kvm->arch.cpu_in_guest)) if (cpumask_test_cpu(cpu + i, cpu_in_guest))
smp_call_function_single(cpu + i, do_nothing, NULL, 1); smp_call_function_single(cpu + i, do_nothing, NULL, 1);
} }
static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu) static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
{ {
struct kvm_nested_guest *nested = vcpu->arch.nested;
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
int prev_cpu;
if (!cpu_has_feature(CPU_FTR_HVMODE))
return;
if (nested)
prev_cpu = nested->prev_cpu[vcpu->arch.nested_vcpu_id];
else
prev_cpu = vcpu->arch.prev_cpu;
/* /*
* With radix, the guest can do TLB invalidations itself, * With radix, the guest can do TLB invalidations itself,
...@@ -2428,12 +2446,46 @@ static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu) ...@@ -2428,12 +2446,46 @@ static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
* ran to flush the TLB. The TLB is shared between threads, * ran to flush the TLB. The TLB is shared between threads,
* so we use a single bit in .need_tlb_flush for all 4 threads. * so we use a single bit in .need_tlb_flush for all 4 threads.
*/ */
if (vcpu->arch.prev_cpu != pcpu) { if (prev_cpu != pcpu) {
if (vcpu->arch.prev_cpu >= 0 && if (prev_cpu >= 0 &&
cpu_first_thread_sibling(vcpu->arch.prev_cpu) != cpu_first_thread_sibling(prev_cpu) !=
cpu_first_thread_sibling(pcpu)) cpu_first_thread_sibling(pcpu))
radix_flush_cpu(kvm, vcpu->arch.prev_cpu, vcpu); radix_flush_cpu(kvm, prev_cpu, vcpu);
vcpu->arch.prev_cpu = pcpu; if (nested)
nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu;
else
vcpu->arch.prev_cpu = pcpu;
}
}
static void kvmppc_radix_check_need_tlb_flush(struct kvm *kvm, int pcpu,
struct kvm_nested_guest *nested)
{
cpumask_t *need_tlb_flush;
int lpid;
if (!cpu_has_feature(CPU_FTR_HVMODE))
return;
if (cpu_has_feature(CPU_FTR_ARCH_300))
pcpu &= ~0x3UL;
if (nested) {
lpid = nested->shadow_lpid;
need_tlb_flush = &nested->need_tlb_flush;
} else {
lpid = kvm->arch.lpid;
need_tlb_flush = &kvm->arch.need_tlb_flush;
}
mtspr(SPRN_LPID, lpid);
isync();
smp_mb();
if (cpumask_test_cpu(pcpu, need_tlb_flush)) {
radix__local_flush_tlb_lpid_guest(lpid);
/* Clear the bit after the TLB flush */
cpumask_clear_cpu(pcpu, need_tlb_flush);
} }
} }
...@@ -3127,8 +3179,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) ...@@ -3127,8 +3179,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
spin_unlock(&core_info.vc[sub]->lock); spin_unlock(&core_info.vc[sub]->lock);
if (kvm_is_radix(vc->kvm)) { if (kvm_is_radix(vc->kvm)) {
int tmp = pcpu;
/* /*
* Do we need to flush the process scoped TLB for the LPAR? * Do we need to flush the process scoped TLB for the LPAR?
* *
...@@ -3139,17 +3189,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) ...@@ -3139,17 +3189,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
* *
* Hash must be flushed in realmode in order to use tlbiel. * Hash must be flushed in realmode in order to use tlbiel.
*/ */
mtspr(SPRN_LPID, vc->kvm->arch.lpid); kvmppc_radix_check_need_tlb_flush(vc->kvm, pcpu, NULL);
isync();
if (cpu_has_feature(CPU_FTR_ARCH_300))
tmp &= ~0x3UL;
if (cpumask_test_cpu(tmp, &vc->kvm->arch.need_tlb_flush)) {
radix__local_flush_tlb_lpid_guest(vc->kvm->arch.lpid);
/* Clear the bit after the TLB flush */
cpumask_clear_cpu(tmp, &vc->kvm->arch.need_tlb_flush);
}
} }
/* /*
...@@ -3872,12 +3912,11 @@ int kvmhv_run_single_vcpu(struct kvm_run *kvm_run, ...@@ -3872,12 +3912,11 @@ int kvmhv_run_single_vcpu(struct kvm_run *kvm_run,
struct kvm_vcpu *vcpu, u64 time_limit, struct kvm_vcpu *vcpu, u64 time_limit,
unsigned long lpcr) unsigned long lpcr)
{ {
int trap, r, pcpu, pcpu0; int trap, r, pcpu;
int srcu_idx; int srcu_idx;
struct kvmppc_vcore *vc; struct kvmppc_vcore *vc;
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
struct kvm_nested_guest *nested = vcpu->arch.nested; struct kvm_nested_guest *nested = vcpu->arch.nested;
unsigned long lpid;
trace_kvmppc_run_vcpu_enter(vcpu); trace_kvmppc_run_vcpu_enter(vcpu);
...@@ -3950,22 +3989,8 @@ int kvmhv_run_single_vcpu(struct kvm_run *kvm_run, ...@@ -3950,22 +3989,8 @@ int kvmhv_run_single_vcpu(struct kvm_run *kvm_run,
vc->vcore_state = VCORE_RUNNING; vc->vcore_state = VCORE_RUNNING;
trace_kvmppc_run_core(vc, 0); trace_kvmppc_run_core(vc, 0);
lpid = vc->kvm->arch.lpid; if (cpu_has_feature(CPU_FTR_HVMODE))
if (nested) kvmppc_radix_check_need_tlb_flush(kvm, pcpu, nested);
lpid = nested->shadow_lpid;
mtspr(SPRN_LPID, lpid);
isync();
/* See comment above in kvmppc_run_core() about this */
pcpu0 = pcpu;
if (cpu_has_feature(CPU_FTR_ARCH_300))
pcpu0 &= ~0x3UL;
if (cpumask_test_cpu(pcpu0, &kvm->arch.need_tlb_flush)) {
radix__local_flush_tlb_lpid_guest(lpid);
/* Clear the bit after the TLB flush */
cpumask_clear_cpu(pcpu0, &kvm->arch.need_tlb_flush);
}
trace_hardirqs_on(); trace_hardirqs_on();
guest_enter_irqoff(); guest_enter_irqoff();
......
...@@ -168,6 +168,9 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) ...@@ -168,6 +168,9 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
if (err) if (err)
return H_PARAMETER; return H_PARAMETER;
if (l2_hv.vcpu_token >= NR_CPUS)
return H_PARAMETER;
/* translate lpid */ /* translate lpid */
l2 = kvmhv_get_nested(vcpu->kvm, l2_hv.lpid, true); l2 = kvmhv_get_nested(vcpu->kvm, l2_hv.lpid, true);
if (!l2) if (!l2)
...@@ -412,6 +415,8 @@ struct kvm_nested_guest *kvmhv_alloc_nested(struct kvm *kvm, unsigned int lpid) ...@@ -412,6 +415,8 @@ struct kvm_nested_guest *kvmhv_alloc_nested(struct kvm *kvm, unsigned int lpid)
goto out_free2; goto out_free2;
gp->shadow_lpid = shadow_lpid; gp->shadow_lpid = shadow_lpid;
memset(gp->prev_cpu, -1, sizeof(gp->prev_cpu));
return gp; return gp;
out_free2: out_free2:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment