Commit e180377f authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

thp: change split_huge_page_pmd() interface

Pass vma instead of mm and add address parameter.

In most cases we already have vma on the stack. We provides
split_huge_page_pmd_mm() for few cases when we have mm, but not vma.

This change is preparation to huge zero pmd splitting implementation.
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@linux.intel.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent cad7f613
...@@ -276,7 +276,7 @@ unaffected. libhugetlbfs will also work fine as usual. ...@@ -276,7 +276,7 @@ unaffected. libhugetlbfs will also work fine as usual.
== Graceful fallback == == Graceful fallback ==
Code walking pagetables but unware about huge pmds can simply call Code walking pagetables but unware about huge pmds can simply call
split_huge_page_pmd(mm, pmd) where the pmd is the one returned by split_huge_page_pmd(vma, addr, pmd) where the pmd is the one returned by
pmd_offset. It's trivial to make the code transparent hugepage aware pmd_offset. It's trivial to make the code transparent hugepage aware
by just grepping for "pmd_offset" and adding split_huge_page_pmd where by just grepping for "pmd_offset" and adding split_huge_page_pmd where
missing after pmd_offset returns the pmd. Thanks to the graceful missing after pmd_offset returns the pmd. Thanks to the graceful
...@@ -299,7 +299,7 @@ diff --git a/mm/mremap.c b/mm/mremap.c ...@@ -299,7 +299,7 @@ diff --git a/mm/mremap.c b/mm/mremap.c
return NULL; return NULL;
pmd = pmd_offset(pud, addr); pmd = pmd_offset(pud, addr);
+ split_huge_page_pmd(mm, pmd); + split_huge_page_pmd(vma, addr, pmd);
if (pmd_none_or_clear_bad(pmd)) if (pmd_none_or_clear_bad(pmd))
return NULL; return NULL;
......
...@@ -182,7 +182,7 @@ static void mark_screen_rdonly(struct mm_struct *mm) ...@@ -182,7 +182,7 @@ static void mark_screen_rdonly(struct mm_struct *mm)
if (pud_none_or_clear_bad(pud)) if (pud_none_or_clear_bad(pud))
goto out; goto out;
pmd = pmd_offset(pud, 0xA0000); pmd = pmd_offset(pud, 0xA0000);
split_huge_page_pmd(mm, pmd); split_huge_page_pmd_mm(mm, 0xA0000, pmd);
if (pmd_none_or_clear_bad(pmd)) if (pmd_none_or_clear_bad(pmd))
goto out; goto out;
pte = pte_offset_map_lock(mm, pmd, 0xA0000, &ptl); pte = pte_offset_map_lock(mm, pmd, 0xA0000, &ptl);
......
...@@ -643,7 +643,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, ...@@ -643,7 +643,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
spinlock_t *ptl; spinlock_t *ptl;
struct page *page; struct page *page;
split_huge_page_pmd(walk->mm, pmd); split_huge_page_pmd(vma, addr, pmd);
if (pmd_trans_unstable(pmd)) if (pmd_trans_unstable(pmd))
return 0; return 0;
......
...@@ -95,12 +95,14 @@ extern int handle_pte_fault(struct mm_struct *mm, ...@@ -95,12 +95,14 @@ extern int handle_pte_fault(struct mm_struct *mm,
struct vm_area_struct *vma, unsigned long address, struct vm_area_struct *vma, unsigned long address,
pte_t *pte, pmd_t *pmd, unsigned int flags); pte_t *pte, pmd_t *pmd, unsigned int flags);
extern int split_huge_page(struct page *page); extern int split_huge_page(struct page *page);
extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd); extern void __split_huge_page_pmd(struct vm_area_struct *vma,
#define split_huge_page_pmd(__mm, __pmd) \ unsigned long address, pmd_t *pmd);
#define split_huge_page_pmd(__vma, __address, __pmd) \
do { \ do { \
pmd_t *____pmd = (__pmd); \ pmd_t *____pmd = (__pmd); \
if (unlikely(pmd_trans_huge(*____pmd))) \ if (unlikely(pmd_trans_huge(*____pmd))) \
__split_huge_page_pmd(__mm, ____pmd); \ __split_huge_page_pmd(__vma, __address, \
____pmd); \
} while (0) } while (0)
#define wait_split_huge_page(__anon_vma, __pmd) \ #define wait_split_huge_page(__anon_vma, __pmd) \
do { \ do { \
...@@ -110,6 +112,8 @@ extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd); ...@@ -110,6 +112,8 @@ extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
BUG_ON(pmd_trans_splitting(*____pmd) || \ BUG_ON(pmd_trans_splitting(*____pmd) || \
pmd_trans_huge(*____pmd)); \ pmd_trans_huge(*____pmd)); \
} while (0) } while (0)
extern void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
pmd_t *pmd);
#if HPAGE_PMD_ORDER > MAX_ORDER #if HPAGE_PMD_ORDER > MAX_ORDER
#error "hugepages can't be allocated by the buddy allocator" #error "hugepages can't be allocated by the buddy allocator"
#endif #endif
...@@ -177,10 +181,12 @@ static inline int split_huge_page(struct page *page) ...@@ -177,10 +181,12 @@ static inline int split_huge_page(struct page *page)
{ {
return 0; return 0;
} }
#define split_huge_page_pmd(__mm, __pmd) \ #define split_huge_page_pmd(__vma, __address, __pmd) \
do { } while (0) do { } while (0)
#define wait_split_huge_page(__anon_vma, __pmd) \ #define wait_split_huge_page(__anon_vma, __pmd) \
do { } while (0) do { } while (0)
#define split_huge_page_pmd_mm(__mm, __address, __pmd) \
do { } while (0)
#define compound_trans_head(page) compound_head(page) #define compound_trans_head(page) compound_head(page)
static inline int hugepage_madvise(struct vm_area_struct *vma, static inline int hugepage_madvise(struct vm_area_struct *vma,
unsigned long *vm_flags, int advice) unsigned long *vm_flags, int advice)
......
...@@ -2475,9 +2475,14 @@ static int khugepaged(void *none) ...@@ -2475,9 +2475,14 @@ static int khugepaged(void *none)
return 0; return 0;
} }
void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd) void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmd)
{ {
struct page *page; struct page *page;
unsigned long haddr = address & HPAGE_PMD_MASK;
struct mm_struct *mm = vma->vm_mm;
BUG_ON(vma->vm_start > haddr || vma->vm_end < haddr + HPAGE_PMD_SIZE);
spin_lock(&mm->page_table_lock); spin_lock(&mm->page_table_lock);
if (unlikely(!pmd_trans_huge(*pmd))) { if (unlikely(!pmd_trans_huge(*pmd))) {
...@@ -2495,6 +2500,16 @@ void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd) ...@@ -2495,6 +2500,16 @@ void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd)
BUG_ON(pmd_trans_huge(*pmd)); BUG_ON(pmd_trans_huge(*pmd));
} }
void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
pmd_t *pmd)
{
struct vm_area_struct *vma;
vma = find_vma(mm, address);
BUG_ON(vma == NULL);
split_huge_page_pmd(vma, address, pmd);
}
static void split_huge_page_address(struct mm_struct *mm, static void split_huge_page_address(struct mm_struct *mm,
unsigned long address) unsigned long address)
{ {
...@@ -2509,7 +2524,7 @@ static void split_huge_page_address(struct mm_struct *mm, ...@@ -2509,7 +2524,7 @@ static void split_huge_page_address(struct mm_struct *mm,
* Caller holds the mmap_sem write mode, so a huge pmd cannot * Caller holds the mmap_sem write mode, so a huge pmd cannot
* materialize from under us. * materialize from under us.
*/ */
split_huge_page_pmd(mm, pmd); split_huge_page_pmd_mm(mm, address, pmd);
} }
void __vma_adjust_trans_huge(struct vm_area_struct *vma, void __vma_adjust_trans_huge(struct vm_area_struct *vma,
......
...@@ -1243,7 +1243,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, ...@@ -1243,7 +1243,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
BUG(); BUG();
} }
#endif #endif
split_huge_page_pmd(vma->vm_mm, pmd); split_huge_page_pmd(vma, addr, pmd);
} else if (zap_huge_pmd(tlb, vma, pmd, addr)) } else if (zap_huge_pmd(tlb, vma, pmd, addr))
goto next; goto next;
/* fall through */ /* fall through */
...@@ -1512,7 +1512,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, ...@@ -1512,7 +1512,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
} }
if (pmd_trans_huge(*pmd)) { if (pmd_trans_huge(*pmd)) {
if (flags & FOLL_SPLIT) { if (flags & FOLL_SPLIT) {
split_huge_page_pmd(mm, pmd); split_huge_page_pmd(vma, address, pmd);
goto split_fallthrough; goto split_fallthrough;
} }
spin_lock(&mm->page_table_lock); spin_lock(&mm->page_table_lock);
......
...@@ -511,7 +511,7 @@ static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud, ...@@ -511,7 +511,7 @@ static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
pmd = pmd_offset(pud, addr); pmd = pmd_offset(pud, addr);
do { do {
next = pmd_addr_end(addr, end); next = pmd_addr_end(addr, end);
split_huge_page_pmd(vma->vm_mm, pmd); split_huge_page_pmd(vma, addr, pmd);
if (pmd_none_or_trans_huge_or_clear_bad(pmd)) if (pmd_none_or_trans_huge_or_clear_bad(pmd))
continue; continue;
if (check_pte_range(vma, pmd, addr, next, nodes, if (check_pte_range(vma, pmd, addr, next, nodes,
......
...@@ -90,7 +90,7 @@ static inline void change_pmd_range(struct vm_area_struct *vma, pud_t *pud, ...@@ -90,7 +90,7 @@ static inline void change_pmd_range(struct vm_area_struct *vma, pud_t *pud,
next = pmd_addr_end(addr, end); next = pmd_addr_end(addr, end);
if (pmd_trans_huge(*pmd)) { if (pmd_trans_huge(*pmd)) {
if (next - addr != HPAGE_PMD_SIZE) if (next - addr != HPAGE_PMD_SIZE)
split_huge_page_pmd(vma->vm_mm, pmd); split_huge_page_pmd(vma, addr, pmd);
else if (change_huge_pmd(vma, pmd, addr, newprot)) else if (change_huge_pmd(vma, pmd, addr, newprot))
continue; continue;
/* fall through */ /* fall through */
......
...@@ -182,7 +182,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma, ...@@ -182,7 +182,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
need_flush = true; need_flush = true;
continue; continue;
} else if (!err) { } else if (!err) {
split_huge_page_pmd(vma->vm_mm, old_pmd); split_huge_page_pmd(vma, old_addr, old_pmd);
} }
VM_BUG_ON(pmd_trans_huge(*old_pmd)); VM_BUG_ON(pmd_trans_huge(*old_pmd));
} }
......
...@@ -58,7 +58,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, ...@@ -58,7 +58,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
if (!walk->pte_entry) if (!walk->pte_entry)
continue; continue;
split_huge_page_pmd(walk->mm, pmd); split_huge_page_pmd_mm(walk->mm, addr, pmd);
if (pmd_none_or_trans_huge_or_clear_bad(pmd)) if (pmd_none_or_trans_huge_or_clear_bad(pmd))
goto again; goto again;
err = walk_pte_range(pmd, addr, next, walk); err = walk_pte_range(pmd, addr, next, walk);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment