Commit 5bf5f03c authored by Pravin B Shelar's avatar Pravin B Shelar Committed by Linus Torvalds

mm: fix slab->page flags corruption

Transparent huge pages can change page->flags (PG_compound_lock) without
taking Slab lock.  Since THP can not break slab pages we can safely access
compound page without taking compound lock.

Specifically this patch fixes a race between compound_unlock() and slab
functions which perform page-flags updates.  This can occur when
get_page()/put_page() is called on a page from slab.

[akpm@linux-foundation.org: tweak comment text, fix comment layout, fix label indenting]
Reported-by: default avatarAmey Bhide <abhide@nicira.com>
Signed-off-by: default avatarPravin B Shelar <pshelar@nicira.com>
Reviewed-by: default avatarChristoph Lameter <cl@linux.com>
Acked-by: default avatarAndrea Arcangeli <aarcange@redhat.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent dbda591d
......@@ -321,6 +321,7 @@ static inline int is_vmalloc_or_module_addr(const void *x)
static inline void compound_lock(struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
VM_BUG_ON(PageSlab(page));
bit_spin_lock(PG_compound_lock, &page->flags);
#endif
}
......@@ -328,6 +329,7 @@ static inline void compound_lock(struct page *page)
static inline void compound_unlock(struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
VM_BUG_ON(PageSlab(page));
bit_spin_unlock(PG_compound_lock, &page->flags);
#endif
}
......
......@@ -82,6 +82,25 @@ static void put_compound_page(struct page *page)
if (likely(page != page_head &&
get_page_unless_zero(page_head))) {
unsigned long flags;
/*
* THP can not break up slab pages so avoid taking
* compound_lock(). Slab performs non-atomic bit ops
* on page->flags for better performance. In particular
* slab_unlock() in slub used to be a hot path. It is
* still hot on arches that do not support
* this_cpu_cmpxchg_double().
*/
if (PageSlab(page_head)) {
if (PageTail(page)) {
if (put_page_testzero(page_head))
VM_BUG_ON(1);
atomic_dec(&page->_mapcount);
goto skip_lock_tail;
} else
goto skip_lock;
}
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
......@@ -92,10 +111,10 @@ static void put_compound_page(struct page *page)
if (unlikely(!PageTail(page))) {
/* __split_huge_page_refcount run before us */
compound_unlock_irqrestore(page_head, flags);
VM_BUG_ON(PageHead(page_head));
skip_lock:
if (put_page_testzero(page_head))
__put_single_page(page_head);
out_put_single:
out_put_single:
if (put_page_testzero(page))
__put_single_page(page);
return;
......@@ -115,6 +134,8 @@ static void put_compound_page(struct page *page)
VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
VM_BUG_ON(atomic_read(&page->_count) != 0);
compound_unlock_irqrestore(page_head, flags);
skip_lock_tail:
if (put_page_testzero(page_head)) {
if (PageHead(page_head))
__put_compound_page(page_head);
......@@ -162,6 +183,18 @@ bool __get_page_tail(struct page *page)
struct page *page_head = compound_trans_head(page);
if (likely(page != page_head && get_page_unless_zero(page_head))) {
/* Ref to put_compound_page() comment. */
if (PageSlab(page_head)) {
if (likely(PageTail(page))) {
__get_page_tail_foll(page, false);
return true;
} else {
put_page(page_head);
return false;
}
}
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment