Commit 1de14c3c authored by Dave Hansen's avatar Dave Hansen Committed by Linus Torvalds

x86-32: Fix possible incomplete TLB invalidate with PAE pagetables

This patch attempts to fix:

	https://bugzilla.kernel.org/show_bug.cgi?id=56461

The symptom is a crash and messages like this:

	chrome: Corrupted page table at address 34a03000
	*pdpt = 0000000000000000 *pde = 0000000000000000
	Bad pagetable: 000f [#1] PREEMPT SMP

Ingo guesses this got introduced by commit 611ae8e3 ("x86/tlb:
enable tlb flush range support for x86") since that code started to free
unused pagetables.

On x86-32 PAE kernels, that new code has the potential to free an entire
PMD page and will clear one of the four page-directory-pointer-table
(aka pgd_t entries).

The hardware aggressively "caches" these top-level entries and invlpg
does not actually affect the CPU's copy.  If we clear one we *HAVE* to
do a full TLB flush, otherwise we might continue using a freed pmd page.
(note, we do this properly on the population side in pud_populate()).

This patch tracks whenever we clear one of these entries in the 'struct
mmu_gather', and ensures that we follow up with a full tlb flush.

BTW, I disassembled and checked that:

	if (tlb->fullmm == 0)
and
	if (!tlb->fullmm && !tlb->need_flush_all)

generate essentially the same code, so there should be zero impact there
to the !PAE case.
Signed-off-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Cc: Peter Anvin <hpa@zytor.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Artem S Tashkinov <t.artem@mailcity.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent bf81710c
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
#define tlb_flush(tlb) \ #define tlb_flush(tlb) \
{ \ { \
if (tlb->fullmm == 0) \ if (!tlb->fullmm && !tlb->need_flush_all) \
flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL); \ flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL); \
else \ else \
flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL); \ flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL); \
......
...@@ -58,6 +58,13 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte) ...@@ -58,6 +58,13 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
{ {
paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT); paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
/*
* NOTE! For PAE, any changes to the top page-directory-pointer-table
* entries need a full cr3 reload to flush.
*/
#ifdef CONFIG_X86_PAE
tlb->need_flush_all = 1;
#endif
tlb_remove_page(tlb, virt_to_page(pmd)); tlb_remove_page(tlb, virt_to_page(pmd));
} }
......
...@@ -99,7 +99,12 @@ struct mmu_gather { ...@@ -99,7 +99,12 @@ struct mmu_gather {
unsigned int need_flush : 1, /* Did free PTEs */ unsigned int need_flush : 1, /* Did free PTEs */
fast_mode : 1; /* No batching */ fast_mode : 1; /* No batching */
unsigned int fullmm; /* we are in the middle of an operation to clear
* a full mm and can make some optimizations */
unsigned int fullmm : 1,
/* we have performed an operation which
* requires a complete flush of the tlb */
need_flush_all : 1;
struct mmu_gather_batch *active; struct mmu_gather_batch *active;
struct mmu_gather_batch local; struct mmu_gather_batch local;
......
...@@ -216,6 +216,7 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm) ...@@ -216,6 +216,7 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
tlb->mm = mm; tlb->mm = mm;
tlb->fullmm = fullmm; tlb->fullmm = fullmm;
tlb->need_flush_all = 0;
tlb->start = -1UL; tlb->start = -1UL;
tlb->end = 0; tlb->end = 0;
tlb->need_flush = 0; tlb->need_flush = 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment