Commit 8c914cb7 authored by Jan Beulich's avatar Jan Beulich Committed by Linus Torvalds

[PATCH] x86_64: actively synchronize vmalloc area when registering certain callbacks

While the modular aspect of the respective i386 patch doesn't apply to
x86-64 (as the top level page directory entry is shared between modules
and the base kernel), handlers registered with register_die_notifier()
are still under similar constraints for touching ioremap()ed or
vmalloc()ed memory. The likelihood of this problem becoming visible is
of course significantly lower, as the assigned virtual addresses would
have to cross a 2**39 byte boundary. This is because the callback gets
invoked
(a) in the page fault path before the top level page table propagation
gets carried out (hence a fault to propagate the top level page table
entry/entries mapping to module's code/data would nest infinitly) and
(b) in the NMI path, where nested faults must absolutely not happen,
since otherwise the IRET from the nested fault re-enables NMIs,
potentially resulting in nested NMI occurences.
Signed-off-by: default avatarJan Beulich <jbeulich@novell.com>
Signed-off-by: default avatarAndi Kleen <ak@suse.de>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 85f9eebc
...@@ -534,6 +534,7 @@ asmlinkage __kprobes void do_nmi(struct pt_regs * regs, long error_code) ...@@ -534,6 +534,7 @@ asmlinkage __kprobes void do_nmi(struct pt_regs * regs, long error_code)
void set_nmi_callback(nmi_callback_t callback) void set_nmi_callback(nmi_callback_t callback)
{ {
vmalloc_sync_all();
rcu_assign_pointer(nmi_callback, callback); rcu_assign_pointer(nmi_callback, callback);
} }
......
...@@ -78,6 +78,8 @@ int register_die_notifier(struct notifier_block *nb) ...@@ -78,6 +78,8 @@ int register_die_notifier(struct notifier_block *nb)
{ {
int err = 0; int err = 0;
unsigned long flags; unsigned long flags;
vmalloc_sync_all();
spin_lock_irqsave(&die_notifier_lock, flags); spin_lock_irqsave(&die_notifier_lock, flags);
err = notifier_chain_register(&die_chain, nb); err = notifier_chain_register(&die_chain, nb);
spin_unlock_irqrestore(&die_notifier_lock, flags); spin_unlock_irqrestore(&die_notifier_lock, flags);
......
...@@ -264,6 +264,8 @@ static int vmalloc_fault(unsigned long address) ...@@ -264,6 +264,8 @@ static int vmalloc_fault(unsigned long address)
return -1; return -1;
if (pgd_none(*pgd)) if (pgd_none(*pgd))
set_pgd(pgd, *pgd_ref); set_pgd(pgd, *pgd_ref);
else
BUG_ON(pgd_page(*pgd) != pgd_page(*pgd_ref));
/* Below here mismatches are bugs because these lower tables /* Below here mismatches are bugs because these lower tables
are shared */ are shared */
...@@ -314,16 +316,6 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -314,16 +316,6 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
/* get the address */ /* get the address */
__asm__("movq %%cr2,%0":"=r" (address)); __asm__("movq %%cr2,%0":"=r" (address));
if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
SIGSEGV) == NOTIFY_STOP)
return;
if (likely(regs->eflags & X86_EFLAGS_IF))
local_irq_enable();
if (unlikely(page_fault_trace))
printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
tsk = current; tsk = current;
mm = tsk->mm; mm = tsk->mm;
...@@ -351,10 +343,12 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -351,10 +343,12 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
*/ */
if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) && if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
((address >= VMALLOC_START && address < VMALLOC_END))) { ((address >= VMALLOC_START && address < VMALLOC_END))) {
if (vmalloc_fault(address) < 0) if (vmalloc_fault(address) >= 0)
goto bad_area_nosemaphore; return;
return;
} }
if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
SIGSEGV) == NOTIFY_STOP)
return;
/* /*
* Don't take the mm semaphore here. If we fixup a prefetch * Don't take the mm semaphore here. If we fixup a prefetch
* fault we could otherwise deadlock. * fault we could otherwise deadlock.
...@@ -362,6 +356,17 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -362,6 +356,17 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
goto bad_area_nosemaphore; goto bad_area_nosemaphore;
} }
if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
SIGSEGV) == NOTIFY_STOP)
return;
if (likely(regs->eflags & X86_EFLAGS_IF))
local_irq_enable();
if (unlikely(page_fault_trace))
printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
if (unlikely(error_code & PF_RSVD)) if (unlikely(error_code & PF_RSVD))
pgtable_bad(address, regs, error_code); pgtable_bad(address, regs, error_code);
...@@ -571,6 +576,48 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -571,6 +576,48 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
return; return;
} }
DEFINE_SPINLOCK(pgd_lock);
struct page *pgd_list;
void vmalloc_sync_all(void)
{
/* Note that races in the updates of insync and start aren't
problematic:
insync can only get set bits added, and updates to start are only
improving performance (without affecting correctness if undone). */
static DECLARE_BITMAP(insync, PTRS_PER_PGD);
static unsigned long start = VMALLOC_START & PGDIR_MASK;
unsigned long address;
for (address = start; address <= VMALLOC_END; address += PGDIR_SIZE) {
if (!test_bit(pgd_index(address), insync)) {
const pgd_t *pgd_ref = pgd_offset_k(address);
struct page *page;
if (pgd_none(*pgd_ref))
continue;
spin_lock(&pgd_lock);
for (page = pgd_list; page;
page = (struct page *)page->index) {
pgd_t *pgd;
pgd = (pgd_t *)page_address(page) + pgd_index(address);
if (pgd_none(*pgd))
set_pgd(pgd, *pgd_ref);
else
BUG_ON(pgd_page(*pgd) != pgd_page(*pgd_ref));
}
spin_unlock(&pgd_lock);
set_bit(pgd_index(address), insync);
}
if (address == start)
start = address + PGDIR_SIZE;
}
/* Check that there is no need to do the same for the modules area. */
BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
(__START_KERNEL & PGDIR_MASK)));
}
static int __init enable_pagefaulttrace(char *str) static int __init enable_pagefaulttrace(char *str)
{ {
page_fault_trace = 1; page_fault_trace = 1;
......
...@@ -45,12 +45,39 @@ static inline void pud_free (pud_t *pud) ...@@ -45,12 +45,39 @@ static inline void pud_free (pud_t *pud)
free_page((unsigned long)pud); free_page((unsigned long)pud);
} }
static inline void pgd_list_add(pgd_t *pgd)
{
struct page *page = virt_to_page(pgd);
spin_lock(&pgd_lock);
page->index = (pgoff_t)pgd_list;
if (pgd_list)
pgd_list->private = (unsigned long)&page->index;
pgd_list = page;
page->private = (unsigned long)&pgd_list;
spin_unlock(&pgd_lock);
}
static inline void pgd_list_del(pgd_t *pgd)
{
struct page *next, **pprev, *page = virt_to_page(pgd);
spin_lock(&pgd_lock);
next = (struct page *)page->index;
pprev = (struct page **)page->private;
*pprev = next;
if (next)
next->private = (unsigned long)pprev;
spin_unlock(&pgd_lock);
}
static inline pgd_t *pgd_alloc(struct mm_struct *mm) static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{ {
unsigned boundary; unsigned boundary;
pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT);
if (!pgd) if (!pgd)
return NULL; return NULL;
pgd_list_add(pgd);
/* /*
* Copy kernel pointers in from init. * Copy kernel pointers in from init.
* Could keep a freelist or slab cache of those because the kernel * Could keep a freelist or slab cache of those because the kernel
...@@ -67,6 +94,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) ...@@ -67,6 +94,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
static inline void pgd_free(pgd_t *pgd) static inline void pgd_free(pgd_t *pgd)
{ {
BUG_ON((unsigned long)pgd & (PAGE_SIZE-1)); BUG_ON((unsigned long)pgd & (PAGE_SIZE-1));
pgd_list_del(pgd);
free_page((unsigned long)pgd); free_page((unsigned long)pgd);
} }
......
...@@ -420,6 +420,10 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) ...@@ -420,6 +420,10 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val })
extern spinlock_t pgd_lock;
extern struct page *pgd_list;
void vmalloc_sync_all(void);
#endif /* !__ASSEMBLY__ */ #endif /* !__ASSEMBLY__ */
extern int kern_addr_valid(unsigned long addr); extern int kern_addr_valid(unsigned long addr);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment