Commit 78ca0e67 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

thp: lazy huge zero page allocation

Instead of allocating huge zero page on hugepage_init() we can postpone it
until first huge zero page map. It saves memory if THP is not in use.

cmpxchg() is used to avoid race on huge_zero_pfn initialization.
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@linux.intel.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 80371957
...@@ -160,22 +160,24 @@ static int start_khugepaged(void) ...@@ -160,22 +160,24 @@ static int start_khugepaged(void)
return err; return err;
} }
static int __init init_huge_zero_page(void) static int init_huge_zero_pfn(void)
{ {
struct page *hpage; struct page *hpage;
unsigned long pfn;
hpage = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, hpage = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
HPAGE_PMD_ORDER); HPAGE_PMD_ORDER);
if (!hpage) if (!hpage)
return -ENOMEM; return -ENOMEM;
pfn = page_to_pfn(hpage);
huge_zero_pfn = page_to_pfn(hpage); if (cmpxchg(&huge_zero_pfn, 0, pfn))
__free_page(hpage);
return 0; return 0;
} }
static inline bool is_huge_zero_pfn(unsigned long pfn) static inline bool is_huge_zero_pfn(unsigned long pfn)
{ {
return pfn == huge_zero_pfn; return huge_zero_pfn && pfn == huge_zero_pfn;
} }
static inline bool is_huge_zero_pmd(pmd_t pmd) static inline bool is_huge_zero_pmd(pmd_t pmd)
...@@ -564,10 +566,6 @@ static int __init hugepage_init(void) ...@@ -564,10 +566,6 @@ static int __init hugepage_init(void)
if (err) if (err)
return err; return err;
err = init_huge_zero_page();
if (err)
goto out;
err = khugepaged_slab_init(); err = khugepaged_slab_init();
if (err) if (err)
goto out; goto out;
...@@ -590,8 +588,6 @@ static int __init hugepage_init(void) ...@@ -590,8 +588,6 @@ static int __init hugepage_init(void)
return 0; return 0;
out: out:
if (huge_zero_pfn)
__free_page(pfn_to_page(huge_zero_pfn));
hugepage_exit_sysfs(hugepage_kobj); hugepage_exit_sysfs(hugepage_kobj);
return err; return err;
} }
...@@ -735,6 +731,10 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -735,6 +731,10 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
return VM_FAULT_OOM; return VM_FAULT_OOM;
if (!(flags & FAULT_FLAG_WRITE)) { if (!(flags & FAULT_FLAG_WRITE)) {
pgtable_t pgtable; pgtable_t pgtable;
if (unlikely(!huge_zero_pfn && init_huge_zero_pfn())) {
count_vm_event(THP_FAULT_FALLBACK);
goto out;
}
pgtable = pte_alloc_one(mm, haddr); pgtable = pte_alloc_one(mm, haddr);
if (unlikely(!pgtable)) if (unlikely(!pgtable))
return VM_FAULT_OOM; return VM_FAULT_OOM;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment