Commit 48a8b97c authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Linus Torvalds

x86/mm: Only use tlb_remove_table() for paravirt

If we don't use paravirt; don't play unnecessary and complicated games
to free page-tables.
Suggested-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarRik van Riel <riel@surriel.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent fd1102f0
...@@ -180,7 +180,7 @@ config X86 ...@@ -180,7 +180,7 @@ config X86
select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && HAVE_PERF_EVENTS_NMI select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && HAVE_PERF_EVENTS_NMI
select HAVE_PERF_REGS select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP select HAVE_PERF_USER_STACK_DUMP
select HAVE_RCU_TABLE_FREE select HAVE_RCU_TABLE_FREE if PARAVIRT
select HAVE_RCU_TABLE_INVALIDATE if HAVE_RCU_TABLE_FREE select HAVE_RCU_TABLE_INVALIDATE if HAVE_RCU_TABLE_FREE
select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION
......
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
#include <asm/mshyperv.h> #include <asm/mshyperv.h>
#include <asm/msr.h> #include <asm/msr.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include <asm/tlb.h>
#define CREATE_TRACE_POINTS #define CREATE_TRACE_POINTS
#include <asm/trace/hyperv.h> #include <asm/trace/hyperv.h>
...@@ -231,4 +232,5 @@ void hyperv_setup_mmu_ops(void) ...@@ -231,4 +232,5 @@ void hyperv_setup_mmu_ops(void)
pr_info("Using hypercall for remote TLB flush\n"); pr_info("Using hypercall for remote TLB flush\n");
pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others; pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
pv_mmu_ops.tlb_remove_table = tlb_remove_table;
} }
...@@ -309,6 +309,11 @@ static inline void flush_tlb_others(const struct cpumask *cpumask, ...@@ -309,6 +309,11 @@ static inline void flush_tlb_others(const struct cpumask *cpumask,
PVOP_VCALL2(pv_mmu_ops.flush_tlb_others, cpumask, info); PVOP_VCALL2(pv_mmu_ops.flush_tlb_others, cpumask, info);
} }
static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table)
{
PVOP_VCALL2(pv_mmu_ops.tlb_remove_table, tlb, table);
}
static inline int paravirt_pgd_alloc(struct mm_struct *mm) static inline int paravirt_pgd_alloc(struct mm_struct *mm)
{ {
return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm); return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
......
...@@ -54,6 +54,7 @@ struct desc_struct; ...@@ -54,6 +54,7 @@ struct desc_struct;
struct task_struct; struct task_struct;
struct cpumask; struct cpumask;
struct flush_tlb_info; struct flush_tlb_info;
struct mmu_gather;
/* /*
* Wrapper type for pointers to code which uses the non-standard * Wrapper type for pointers to code which uses the non-standard
...@@ -222,6 +223,8 @@ struct pv_mmu_ops { ...@@ -222,6 +223,8 @@ struct pv_mmu_ops {
void (*flush_tlb_others)(const struct cpumask *cpus, void (*flush_tlb_others)(const struct cpumask *cpus,
const struct flush_tlb_info *info); const struct flush_tlb_info *info);
void (*tlb_remove_table)(struct mmu_gather *tlb, void *table);
/* Hooks for allocating and freeing a pagetable top-level */ /* Hooks for allocating and freeing a pagetable top-level */
int (*pgd_alloc)(struct mm_struct *mm); int (*pgd_alloc)(struct mm_struct *mm);
void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd); void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
......
...@@ -552,6 +552,9 @@ extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch); ...@@ -552,6 +552,9 @@ extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch);
#ifndef CONFIG_PARAVIRT #ifndef CONFIG_PARAVIRT
#define flush_tlb_others(mask, info) \ #define flush_tlb_others(mask, info) \
native_flush_tlb_others(mask, info) native_flush_tlb_others(mask, info)
#define paravirt_tlb_remove_table(tlb, page) \
tlb_remove_page(tlb, (void *)(page))
#endif #endif
#endif /* _ASM_X86_TLBFLUSH_H */ #endif /* _ASM_X86_TLBFLUSH_H */
...@@ -45,6 +45,7 @@ ...@@ -45,6 +45,7 @@
#include <asm/apic.h> #include <asm/apic.h>
#include <asm/apicdef.h> #include <asm/apicdef.h>
#include <asm/hypervisor.h> #include <asm/hypervisor.h>
#include <asm/tlb.h>
static int kvmapf = 1; static int kvmapf = 1;
...@@ -636,8 +637,10 @@ static void __init kvm_guest_init(void) ...@@ -636,8 +637,10 @@ static void __init kvm_guest_init(void)
if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) && if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
!kvm_para_has_hint(KVM_HINTS_REALTIME) && !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
pv_mmu_ops.flush_tlb_others = kvm_flush_tlb_others; pv_mmu_ops.flush_tlb_others = kvm_flush_tlb_others;
pv_mmu_ops.tlb_remove_table = tlb_remove_table;
}
if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
apic_set_eoi_write(kvm_guest_apic_eoi_write); apic_set_eoi_write(kvm_guest_apic_eoi_write);
......
...@@ -41,6 +41,7 @@ ...@@ -41,6 +41,7 @@
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include <asm/timer.h> #include <asm/timer.h>
#include <asm/special_insns.h> #include <asm/special_insns.h>
#include <asm/tlb.h>
/* /*
* nop stub, which must not clobber anything *including the stack* to * nop stub, which must not clobber anything *including the stack* to
...@@ -409,6 +410,7 @@ struct pv_mmu_ops pv_mmu_ops __ro_after_init = { ...@@ -409,6 +410,7 @@ struct pv_mmu_ops pv_mmu_ops __ro_after_init = {
.flush_tlb_kernel = native_flush_tlb_global, .flush_tlb_kernel = native_flush_tlb_global,
.flush_tlb_one_user = native_flush_tlb_one_user, .flush_tlb_one_user = native_flush_tlb_one_user,
.flush_tlb_others = native_flush_tlb_others, .flush_tlb_others = native_flush_tlb_others,
.tlb_remove_table = (void (*)(struct mmu_gather *, void *))tlb_remove_page,
.pgd_alloc = __paravirt_pgd_alloc, .pgd_alloc = __paravirt_pgd_alloc,
.pgd_free = paravirt_nop, .pgd_free = paravirt_nop,
......
...@@ -63,7 +63,7 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte) ...@@ -63,7 +63,7 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{ {
pgtable_page_dtor(pte); pgtable_page_dtor(pte);
paravirt_release_pte(page_to_pfn(pte)); paravirt_release_pte(page_to_pfn(pte));
tlb_remove_table(tlb, pte); paravirt_tlb_remove_table(tlb, pte);
} }
#if CONFIG_PGTABLE_LEVELS > 2 #if CONFIG_PGTABLE_LEVELS > 2
...@@ -79,21 +79,21 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) ...@@ -79,21 +79,21 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
tlb->need_flush_all = 1; tlb->need_flush_all = 1;
#endif #endif
pgtable_pmd_page_dtor(page); pgtable_pmd_page_dtor(page);
tlb_remove_table(tlb, page); paravirt_tlb_remove_table(tlb, page);
} }
#if CONFIG_PGTABLE_LEVELS > 3 #if CONFIG_PGTABLE_LEVELS > 3
void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
{ {
paravirt_release_pud(__pa(pud) >> PAGE_SHIFT); paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
tlb_remove_table(tlb, virt_to_page(pud)); paravirt_tlb_remove_table(tlb, virt_to_page(pud));
} }
#if CONFIG_PGTABLE_LEVELS > 4 #if CONFIG_PGTABLE_LEVELS > 4
void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d) void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d)
{ {
paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT); paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT);
tlb_remove_table(tlb, virt_to_page(p4d)); paravirt_tlb_remove_table(tlb, virt_to_page(p4d));
} }
#endif /* CONFIG_PGTABLE_LEVELS > 4 */ #endif /* CONFIG_PGTABLE_LEVELS > 4 */
#endif /* CONFIG_PGTABLE_LEVELS > 3 */ #endif /* CONFIG_PGTABLE_LEVELS > 3 */
......
...@@ -67,6 +67,7 @@ ...@@ -67,6 +67,7 @@
#include <asm/init.h> #include <asm/init.h>
#include <asm/pat.h> #include <asm/pat.h>
#include <asm/smp.h> #include <asm/smp.h>
#include <asm/tlb.h>
#include <asm/xen/hypercall.h> #include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h> #include <asm/xen/hypervisor.h>
...@@ -2397,6 +2398,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = { ...@@ -2397,6 +2398,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
.flush_tlb_kernel = xen_flush_tlb, .flush_tlb_kernel = xen_flush_tlb,
.flush_tlb_one_user = xen_flush_tlb_one_user, .flush_tlb_one_user = xen_flush_tlb_one_user,
.flush_tlb_others = xen_flush_tlb_others, .flush_tlb_others = xen_flush_tlb_others,
.tlb_remove_table = tlb_remove_table,
.pgd_alloc = xen_pgd_alloc, .pgd_alloc = xen_pgd_alloc,
.pgd_free = xen_pgd_free, .pgd_free = xen_pgd_free,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment