Commit c672c7f2 authored by Mike Kravetz's avatar Mike Kravetz Committed by Linus Torvalds

mm/hugetlb: expose hugetlb fault mutex for use by fallocate

hugetlb page faults are currently synchronized by the table of mutexes
(htlb_fault_mutex_table).  fallocate code will need to synchronize with
the page fault code when it allocates or deletes pages.  Expose
interfaces so that fallocate operations can be synchronized with page
faults.  Minor name changes to be more consistent with other global
hugetlb symbols.
Signed-off-by: default avatarMike Kravetz <mike.kravetz@oracle.com>
Reviewed-by: default avatarNaoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Acked-by: default avatarHillf Danton <hillf.zj@alibaba-inc.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Aneesh Kumar <aneesh.kumar@linux.vnet.ibm.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Michal Hocko <mhocko@suse.cz>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent feba16e2
...@@ -88,6 +88,11 @@ int dequeue_hwpoisoned_huge_page(struct page *page); ...@@ -88,6 +88,11 @@ int dequeue_hwpoisoned_huge_page(struct page *page);
bool isolate_huge_page(struct page *page, struct list_head *list); bool isolate_huge_page(struct page *page, struct list_head *list);
void putback_active_hugepage(struct page *page); void putback_active_hugepage(struct page *page);
void free_huge_page(struct page *page); void free_huge_page(struct page *page);
extern struct mutex *hugetlb_fault_mutex_table;
u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
struct vm_area_struct *vma,
struct address_space *mapping,
pgoff_t idx, unsigned long address);
#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud); pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud);
......
...@@ -64,7 +64,7 @@ DEFINE_SPINLOCK(hugetlb_lock); ...@@ -64,7 +64,7 @@ DEFINE_SPINLOCK(hugetlb_lock);
* prevent spurious OOMs when the hugepage pool is fully utilized. * prevent spurious OOMs when the hugepage pool is fully utilized.
*/ */
static int num_fault_mutexes; static int num_fault_mutexes;
static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp; struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
/* Forward declaration */ /* Forward declaration */
static int hugetlb_acct_memory(struct hstate *h, long delta); static int hugetlb_acct_memory(struct hstate *h, long delta);
...@@ -2482,7 +2482,7 @@ static void __exit hugetlb_exit(void) ...@@ -2482,7 +2482,7 @@ static void __exit hugetlb_exit(void)
} }
kobject_put(hugepages_kobj); kobject_put(hugepages_kobj);
kfree(htlb_fault_mutex_table); kfree(hugetlb_fault_mutex_table);
} }
module_exit(hugetlb_exit); module_exit(hugetlb_exit);
...@@ -2515,12 +2515,12 @@ static int __init hugetlb_init(void) ...@@ -2515,12 +2515,12 @@ static int __init hugetlb_init(void)
#else #else
num_fault_mutexes = 1; num_fault_mutexes = 1;
#endif #endif
htlb_fault_mutex_table = hugetlb_fault_mutex_table =
kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL); kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
BUG_ON(!htlb_fault_mutex_table); BUG_ON(!hugetlb_fault_mutex_table);
for (i = 0; i < num_fault_mutexes; i++) for (i = 0; i < num_fault_mutexes; i++)
mutex_init(&htlb_fault_mutex_table[i]); mutex_init(&hugetlb_fault_mutex_table[i]);
return 0; return 0;
} }
module_init(hugetlb_init); module_init(hugetlb_init);
...@@ -3454,7 +3454,7 @@ static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3454,7 +3454,7 @@ static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
} }
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm, u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
struct vm_area_struct *vma, struct vm_area_struct *vma,
struct address_space *mapping, struct address_space *mapping,
pgoff_t idx, unsigned long address) pgoff_t idx, unsigned long address)
...@@ -3479,7 +3479,7 @@ static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm, ...@@ -3479,7 +3479,7 @@ static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
* For uniprocesor systems we always use a single mutex, so just * For uniprocesor systems we always use a single mutex, so just
* return 0 and avoid the hashing overhead. * return 0 and avoid the hashing overhead.
*/ */
static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm, u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
struct vm_area_struct *vma, struct vm_area_struct *vma,
struct address_space *mapping, struct address_space *mapping,
pgoff_t idx, unsigned long address) pgoff_t idx, unsigned long address)
...@@ -3527,8 +3527,8 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3527,8 +3527,8 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* get spurious allocation failures if two CPUs race to instantiate * get spurious allocation failures if two CPUs race to instantiate
* the same page in the page cache. * the same page in the page cache.
*/ */
hash = fault_mutex_hash(h, mm, vma, mapping, idx, address); hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address);
mutex_lock(&htlb_fault_mutex_table[hash]); mutex_lock(&hugetlb_fault_mutex_table[hash]);
entry = huge_ptep_get(ptep); entry = huge_ptep_get(ptep);
if (huge_pte_none(entry)) { if (huge_pte_none(entry)) {
...@@ -3613,7 +3613,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3613,7 +3613,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
put_page(pagecache_page); put_page(pagecache_page);
} }
out_mutex: out_mutex:
mutex_unlock(&htlb_fault_mutex_table[hash]); mutex_unlock(&hugetlb_fault_mutex_table[hash]);
/* /*
* Generally it's safe to hold refcount during waiting page lock. But * Generally it's safe to hold refcount during waiting page lock. But
* here we just wait to defer the next page fault to avoid busy loop and * here we just wait to defer the next page fault to avoid busy loop and
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment