Commit 063afacd authored by Ben Gardon's avatar Ben Gardon Committed by Paolo Bonzini

kvm: x86/mmu: Support invalidate range MMU notifier for TDP MMU

In order to interoperate correctly with the rest of KVM and other Linux
subsystems, the TDP MMU must correctly handle various MMU notifiers. Add
hooks to handle the invalidate range family of MMU notifiers.

Tested by running kvm-unit-tests and KVM selftests on an Intel Haswell
machine. This series introduced no new failures.

This series can be viewed in Gerrit at:
	https://linux-review.googlesource.com/c/virt/kvm/kvm/+/2538Signed-off-by: default avatarBen Gardon <bgardon@google.com>
Message-Id: <20201014182700.2888246-13-bgardon@google.com>
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 89c0fd49
...@@ -1497,7 +1497,14 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, ...@@ -1497,7 +1497,14 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end,
unsigned flags) unsigned flags)
{ {
return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp); int r;
r = kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
if (kvm->arch.tdp_mmu_enabled)
r |= kvm_tdp_mmu_zap_hva_range(kvm, start, end);
return r;
} }
int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
......
...@@ -52,7 +52,7 @@ bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa) ...@@ -52,7 +52,7 @@ bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa)
} }
static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
gfn_t start, gfn_t end); gfn_t start, gfn_t end, bool can_yield);
void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root) void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root)
{ {
...@@ -65,7 +65,7 @@ void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root) ...@@ -65,7 +65,7 @@ void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root)
list_del(&root->link); list_del(&root->link);
zap_gfn_range(kvm, root, 0, max_gfn); zap_gfn_range(kvm, root, 0, max_gfn, false);
free_page((unsigned long)root->spt); free_page((unsigned long)root->spt);
kmem_cache_free(mmu_page_header_cache, root); kmem_cache_free(mmu_page_header_cache, root);
...@@ -303,9 +303,14 @@ static bool tdp_mmu_iter_flush_cond_resched(struct kvm *kvm, struct tdp_iter *it ...@@ -303,9 +303,14 @@ static bool tdp_mmu_iter_flush_cond_resched(struct kvm *kvm, struct tdp_iter *it
* non-root pages mapping GFNs strictly within that range. Returns true if * non-root pages mapping GFNs strictly within that range. Returns true if
* SPTEs have been cleared and a TLB flush is needed before releasing the * SPTEs have been cleared and a TLB flush is needed before releasing the
* MMU lock. * MMU lock.
* If can_yield is true, will release the MMU lock and reschedule if the
* scheduler needs the CPU or there is contention on the MMU lock. If this
* function cannot yield, it will not release the MMU lock or reschedule and
* the caller must ensure it does not supply too large a GFN range, or the
* operation can cause a soft lockup.
*/ */
static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
gfn_t start, gfn_t end) gfn_t start, gfn_t end, bool can_yield)
{ {
struct tdp_iter iter; struct tdp_iter iter;
bool flush_needed = false; bool flush_needed = false;
...@@ -326,7 +331,10 @@ static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, ...@@ -326,7 +331,10 @@ static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
tdp_mmu_set_spte(kvm, &iter, 0); tdp_mmu_set_spte(kvm, &iter, 0);
flush_needed = tdp_mmu_iter_flush_cond_resched(kvm, &iter); if (can_yield)
flush_needed = tdp_mmu_iter_flush_cond_resched(kvm, &iter);
else
flush_needed = true;
} }
return flush_needed; return flush_needed;
} }
...@@ -349,7 +357,7 @@ bool kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, gfn_t start, gfn_t end) ...@@ -349,7 +357,7 @@ bool kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, gfn_t start, gfn_t end)
*/ */
kvm_mmu_get_root(kvm, root); kvm_mmu_get_root(kvm, root);
flush |= zap_gfn_range(kvm, root, start, end); flush |= zap_gfn_range(kvm, root, start, end, true);
kvm_mmu_put_root(kvm, root); kvm_mmu_put_root(kvm, root);
} }
...@@ -496,3 +504,65 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, ...@@ -496,3 +504,65 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code,
return ret; return ret;
} }
static int kvm_tdp_mmu_handle_hva_range(struct kvm *kvm, unsigned long start,
unsigned long end, unsigned long data,
int (*handler)(struct kvm *kvm, struct kvm_memory_slot *slot,
struct kvm_mmu_page *root, gfn_t start,
gfn_t end, unsigned long data))
{
struct kvm_memslots *slots;
struct kvm_memory_slot *memslot;
struct kvm_mmu_page *root;
int ret = 0;
int as_id;
for_each_tdp_mmu_root(kvm, root) {
/*
* Take a reference on the root so that it cannot be freed if
* this thread releases the MMU lock and yields in this loop.
*/
kvm_mmu_get_root(kvm, root);
as_id = kvm_mmu_page_as_id(root);
slots = __kvm_memslots(kvm, as_id);
kvm_for_each_memslot(memslot, slots) {
unsigned long hva_start, hva_end;
gfn_t gfn_start, gfn_end;
hva_start = max(start, memslot->userspace_addr);
hva_end = min(end, memslot->userspace_addr +
(memslot->npages << PAGE_SHIFT));
if (hva_start >= hva_end)
continue;
/*
* {gfn(page) | page intersects with [hva_start, hva_end)} =
* {gfn_start, gfn_start+1, ..., gfn_end-1}.
*/
gfn_start = hva_to_gfn_memslot(hva_start, memslot);
gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
ret |= handler(kvm, memslot, root, gfn_start,
gfn_end, data);
}
kvm_mmu_put_root(kvm, root);
}
return ret;
}
static int zap_gfn_range_hva_wrapper(struct kvm *kvm,
struct kvm_memory_slot *slot,
struct kvm_mmu_page *root, gfn_t start,
gfn_t end, unsigned long unused)
{
return zap_gfn_range(kvm, root, start, end, false);
}
int kvm_tdp_mmu_zap_hva_range(struct kvm *kvm, unsigned long start,
unsigned long end)
{
return kvm_tdp_mmu_handle_hva_range(kvm, start, end, 0,
zap_gfn_range_hva_wrapper);
}
...@@ -18,4 +18,7 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm); ...@@ -18,4 +18,7 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm);
int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code,
int map_writable, int max_level, kvm_pfn_t pfn, int map_writable, int max_level, kvm_pfn_t pfn,
bool prefault); bool prefault);
int kvm_tdp_mmu_zap_hva_range(struct kvm *kvm, unsigned long start,
unsigned long end);
#endif /* __KVM_X86_MMU_TDP_MMU_H */ #endif /* __KVM_X86_MMU_TDP_MMU_H */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment