Commit b969c4ab authored by Mel Gorman's avatar Mel Gorman Committed by Linus Torvalds

mm: compaction: determine if dirty pages can be migrated without blocking within ->migratepage

Asynchronous compaction is used when allocating transparent hugepages to
avoid blocking for long periods of time.  Due to reports of stalling,
there was a debate on disabling synchronous compaction but this severely
impacted allocation success rates.  Part of the reason was that many dirty
pages are skipped in asynchronous compaction by the following check;

	if (PageDirty(page) && !sync &&
		mapping->a_ops->migratepage != migrate_page)
			rc = -EBUSY;

This skips over all mapping aops using buffer_migrate_page() even though
it is possible to migrate some of these pages without blocking.  This
patch updates the ->migratepage callback with a "sync" parameter.  It is
the responsibility of the callback to fail gracefully if migration would
block.
Signed-off-by: default avatarMel Gorman <mgorman@suse.de>
Reviewed-by: default avatarRik van Riel <riel@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Andy Isaacson <adi@hexapodia.org>
Cc: Nai Xia <nai.xia@gmail.com>
Cc: Johannes Weiner <jweiner@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 7335084d
...@@ -872,7 +872,7 @@ static int btree_submit_bio_hook(struct inode *inode, int rw, struct bio *bio, ...@@ -872,7 +872,7 @@ static int btree_submit_bio_hook(struct inode *inode, int rw, struct bio *bio,
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
static int btree_migratepage(struct address_space *mapping, static int btree_migratepage(struct address_space *mapping,
struct page *newpage, struct page *page) struct page *newpage, struct page *page, bool sync)
{ {
/* /*
* we can't safely write a btree page from here, * we can't safely write a btree page from here,
...@@ -887,7 +887,7 @@ static int btree_migratepage(struct address_space *mapping, ...@@ -887,7 +887,7 @@ static int btree_migratepage(struct address_space *mapping,
if (page_has_private(page) && if (page_has_private(page) &&
!try_to_release_page(page, GFP_KERNEL)) !try_to_release_page(page, GFP_KERNEL))
return -EAGAIN; return -EAGAIN;
return migrate_page(mapping, newpage, page); return migrate_page(mapping, newpage, page, sync);
} }
#endif #endif
......
...@@ -583,7 +583,8 @@ static int hugetlbfs_set_page_dirty(struct page *page) ...@@ -583,7 +583,8 @@ static int hugetlbfs_set_page_dirty(struct page *page)
} }
static int hugetlbfs_migrate_page(struct address_space *mapping, static int hugetlbfs_migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page) struct page *newpage, struct page *page,
bool sync)
{ {
int rc; int rc;
......
...@@ -332,7 +332,7 @@ void nfs_commit_release_pages(struct nfs_write_data *data); ...@@ -332,7 +332,7 @@ void nfs_commit_release_pages(struct nfs_write_data *data);
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
extern int nfs_migrate_page(struct address_space *, extern int nfs_migrate_page(struct address_space *,
struct page *, struct page *); struct page *, struct page *, bool);
#else #else
#define nfs_migrate_page NULL #define nfs_migrate_page NULL
#endif #endif
......
...@@ -1688,7 +1688,7 @@ int nfs_wb_page(struct inode *inode, struct page *page) ...@@ -1688,7 +1688,7 @@ int nfs_wb_page(struct inode *inode, struct page *page)
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
int nfs_migrate_page(struct address_space *mapping, struct page *newpage, int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
struct page *page) struct page *page, bool sync)
{ {
/* /*
* If PagePrivate is set, then the page is currently associated with * If PagePrivate is set, then the page is currently associated with
...@@ -1703,7 +1703,7 @@ int nfs_migrate_page(struct address_space *mapping, struct page *newpage, ...@@ -1703,7 +1703,7 @@ int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
nfs_fscache_release_page(page, GFP_KERNEL); nfs_fscache_release_page(page, GFP_KERNEL);
return migrate_page(mapping, newpage, page); return migrate_page(mapping, newpage, page, sync);
} }
#endif #endif
......
...@@ -609,9 +609,12 @@ struct address_space_operations { ...@@ -609,9 +609,12 @@ struct address_space_operations {
loff_t offset, unsigned long nr_segs); loff_t offset, unsigned long nr_segs);
int (*get_xip_mem)(struct address_space *, pgoff_t, int, int (*get_xip_mem)(struct address_space *, pgoff_t, int,
void **, unsigned long *); void **, unsigned long *);
/* migrate the contents of a page to the specified target */ /*
* migrate the contents of a page to the specified target. If sync
* is false, it must not block.
*/
int (*migratepage) (struct address_space *, int (*migratepage) (struct address_space *,
struct page *, struct page *); struct page *, struct page *, bool);
int (*launder_page) (struct page *); int (*launder_page) (struct page *);
int (*is_partially_uptodate) (struct page *, read_descriptor_t *, int (*is_partially_uptodate) (struct page *, read_descriptor_t *,
unsigned long); unsigned long);
...@@ -2537,7 +2540,7 @@ extern int generic_check_addressable(unsigned, u64); ...@@ -2537,7 +2540,7 @@ extern int generic_check_addressable(unsigned, u64);
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
extern int buffer_migrate_page(struct address_space *, extern int buffer_migrate_page(struct address_space *,
struct page *, struct page *); struct page *, struct page *, bool);
#else #else
#define buffer_migrate_page NULL #define buffer_migrate_page NULL
#endif #endif
......
...@@ -11,7 +11,7 @@ typedef struct page *new_page_t(struct page *, unsigned long private, int **); ...@@ -11,7 +11,7 @@ typedef struct page *new_page_t(struct page *, unsigned long private, int **);
extern void putback_lru_pages(struct list_head *l); extern void putback_lru_pages(struct list_head *l);
extern int migrate_page(struct address_space *, extern int migrate_page(struct address_space *,
struct page *, struct page *); struct page *, struct page *, bool);
extern int migrate_pages(struct list_head *l, new_page_t x, extern int migrate_pages(struct list_head *l, new_page_t x,
unsigned long private, bool offlining, unsigned long private, bool offlining,
bool sync); bool sync);
......
...@@ -216,6 +216,55 @@ void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, ...@@ -216,6 +216,55 @@ void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
pte_unmap_unlock(ptep, ptl); pte_unmap_unlock(ptep, ptl);
} }
#ifdef CONFIG_BLOCK
/* Returns true if all buffers are successfully locked */
static bool buffer_migrate_lock_buffers(struct buffer_head *head, bool sync)
{
struct buffer_head *bh = head;
/* Simple case, sync compaction */
if (sync) {
do {
get_bh(bh);
lock_buffer(bh);
bh = bh->b_this_page;
} while (bh != head);
return true;
}
/* async case, we cannot block on lock_buffer so use trylock_buffer */
do {
get_bh(bh);
if (!trylock_buffer(bh)) {
/*
* We failed to lock the buffer and cannot stall in
* async migration. Release the taken locks
*/
struct buffer_head *failed_bh = bh;
put_bh(failed_bh);
bh = head;
while (bh != failed_bh) {
unlock_buffer(bh);
put_bh(bh);
bh = bh->b_this_page;
}
return false;
}
bh = bh->b_this_page;
} while (bh != head);
return true;
}
#else
static inline bool buffer_migrate_lock_buffers(struct buffer_head *head,
bool sync)
{
return true;
}
#endif /* CONFIG_BLOCK */
/* /*
* Replace the page in the mapping. * Replace the page in the mapping.
* *
...@@ -225,7 +274,8 @@ void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, ...@@ -225,7 +274,8 @@ void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
* 3 for pages with a mapping and PagePrivate/PagePrivate2 set. * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
*/ */
static int migrate_page_move_mapping(struct address_space *mapping, static int migrate_page_move_mapping(struct address_space *mapping,
struct page *newpage, struct page *page) struct page *newpage, struct page *page,
struct buffer_head *head, bool sync)
{ {
int expected_count; int expected_count;
void **pslot; void **pslot;
...@@ -254,6 +304,19 @@ static int migrate_page_move_mapping(struct address_space *mapping, ...@@ -254,6 +304,19 @@ static int migrate_page_move_mapping(struct address_space *mapping,
return -EAGAIN; return -EAGAIN;
} }
/*
* In the async migration case of moving a page with buffers, lock the
* buffers using trylock before the mapping is moved. If the mapping
* was moved, we later failed to lock the buffers and could not move
* the mapping back due to an elevated page count, we would have to
* block waiting on other references to be dropped.
*/
if (!sync && head && !buffer_migrate_lock_buffers(head, sync)) {
page_unfreeze_refs(page, expected_count);
spin_unlock_irq(&mapping->tree_lock);
return -EAGAIN;
}
/* /*
* Now we know that no one else is looking at the page. * Now we know that no one else is looking at the page.
*/ */
...@@ -409,13 +472,13 @@ EXPORT_SYMBOL(fail_migrate_page); ...@@ -409,13 +472,13 @@ EXPORT_SYMBOL(fail_migrate_page);
* Pages are locked upon entry and exit. * Pages are locked upon entry and exit.
*/ */
int migrate_page(struct address_space *mapping, int migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page) struct page *newpage, struct page *page, bool sync)
{ {
int rc; int rc;
BUG_ON(PageWriteback(page)); /* Writeback must be complete */ BUG_ON(PageWriteback(page)); /* Writeback must be complete */
rc = migrate_page_move_mapping(mapping, newpage, page); rc = migrate_page_move_mapping(mapping, newpage, page, NULL, sync);
if (rc) if (rc)
return rc; return rc;
...@@ -432,28 +495,28 @@ EXPORT_SYMBOL(migrate_page); ...@@ -432,28 +495,28 @@ EXPORT_SYMBOL(migrate_page);
* exist. * exist.
*/ */
int buffer_migrate_page(struct address_space *mapping, int buffer_migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page) struct page *newpage, struct page *page, bool sync)
{ {
struct buffer_head *bh, *head; struct buffer_head *bh, *head;
int rc; int rc;
if (!page_has_buffers(page)) if (!page_has_buffers(page))
return migrate_page(mapping, newpage, page); return migrate_page(mapping, newpage, page, sync);
head = page_buffers(page); head = page_buffers(page);
rc = migrate_page_move_mapping(mapping, newpage, page); rc = migrate_page_move_mapping(mapping, newpage, page, head, sync);
if (rc) if (rc)
return rc; return rc;
bh = head; /*
do { * In the async case, migrate_page_move_mapping locked the buffers
get_bh(bh); * with an IRQ-safe spinlock held. In the sync case, the buffers
lock_buffer(bh); * need to be locked now
bh = bh->b_this_page; */
if (sync)
} while (bh != head); BUG_ON(!buffer_migrate_lock_buffers(head, sync));
ClearPagePrivate(page); ClearPagePrivate(page);
set_page_private(newpage, page_private(page)); set_page_private(newpage, page_private(page));
...@@ -530,10 +593,13 @@ static int writeout(struct address_space *mapping, struct page *page) ...@@ -530,10 +593,13 @@ static int writeout(struct address_space *mapping, struct page *page)
* Default handling if a filesystem does not provide a migration function. * Default handling if a filesystem does not provide a migration function.
*/ */
static int fallback_migrate_page(struct address_space *mapping, static int fallback_migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page) struct page *newpage, struct page *page, bool sync)
{ {
if (PageDirty(page)) if (PageDirty(page)) {
if (!sync)
return -EBUSY;
return writeout(mapping, page); return writeout(mapping, page);
}
/* /*
* Buffers may be managed in a filesystem specific way. * Buffers may be managed in a filesystem specific way.
...@@ -543,7 +609,7 @@ static int fallback_migrate_page(struct address_space *mapping, ...@@ -543,7 +609,7 @@ static int fallback_migrate_page(struct address_space *mapping,
!try_to_release_page(page, GFP_KERNEL)) !try_to_release_page(page, GFP_KERNEL))
return -EAGAIN; return -EAGAIN;
return migrate_page(mapping, newpage, page); return migrate_page(mapping, newpage, page, sync);
} }
/* /*
...@@ -579,29 +645,18 @@ static int move_to_new_page(struct page *newpage, struct page *page, ...@@ -579,29 +645,18 @@ static int move_to_new_page(struct page *newpage, struct page *page,
mapping = page_mapping(page); mapping = page_mapping(page);
if (!mapping) if (!mapping)
rc = migrate_page(mapping, newpage, page); rc = migrate_page(mapping, newpage, page, sync);
else { else if (mapping->a_ops->migratepage)
/* /*
* Do not writeback pages if !sync and migratepage is * Most pages have a mapping and most filesystems provide a
* not pointing to migrate_page() which is nonblocking * migratepage callback. Anonymous pages are part of swap
* (swapcache/tmpfs uses migratepage = migrate_page). * space which also has its own migratepage callback. This
* is the most common path for page migration.
*/ */
if (PageDirty(page) && !sync && rc = mapping->a_ops->migratepage(mapping,
mapping->a_ops->migratepage != migrate_page) newpage, page, sync);
rc = -EBUSY; else
else if (mapping->a_ops->migratepage) rc = fallback_migrate_page(mapping, newpage, page, sync);
/*
* Most pages have a mapping and most filesystems
* should provide a migration function. Anonymous
* pages are part of swap space which also has its
* own migration function. This is the most common
* path for page migration.
*/
rc = mapping->a_ops->migratepage(mapping,
newpage, page);
else
rc = fallback_migrate_page(mapping, newpage, page);
}
if (rc) { if (rc) {
newpage->mapping = NULL; newpage->mapping = NULL;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment