Commit 11bc82d6 authored by Andrea Arcangeli's avatar Andrea Arcangeli Committed by Linus Torvalds

mm: compaction: Use async migration for __GFP_NO_KSWAPD and enforce no writeback

__GFP_NO_KSWAPD allocations are usually very expensive and not mandatory
to succeed as they have graceful fallback.  Waiting for I/O in those,
tends to be overkill in terms of latencies, so we can reduce their latency
by disabling sync migrate.

Unfortunately, even with async migration it's still possible for the
process to be blocked waiting for a request slot (e.g.  get_request_wait
in the block layer) when ->writepage is called.  To prevent
__GFP_NO_KSWAPD blocking, this patch prevents ->writepage being called on
dirty page cache for asynchronous migration.

Addresses https://bugzilla.kernel.org/show_bug.cgi?id=31142

[mel@csn.ul.ie: Avoid writebacks for NFS, retry locked pages, use bool]
Signed-off-by: default avatarAndrea Arcangeli <aarcange@redhat.com>
Signed-off-by: default avatarMel Gorman <mel@csn.ul.ie>
Cc: Arthur Marsh <arthur.marsh@internode.on.net>
Cc: Clemens Ladisch <cladisch@googlemail.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Reported-by: default avatarAlex Villacis Lasso <avillaci@ceibo.fiec.espol.edu.ec>
Tested-by: default avatarAlex Villacis Lasso <avillaci@ceibo.fiec.espol.edu.ec>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b2eef8c0
...@@ -564,7 +564,7 @@ static int fallback_migrate_page(struct address_space *mapping, ...@@ -564,7 +564,7 @@ static int fallback_migrate_page(struct address_space *mapping,
* == 0 - success * == 0 - success
*/ */
static int move_to_new_page(struct page *newpage, struct page *page, static int move_to_new_page(struct page *newpage, struct page *page,
int remap_swapcache) int remap_swapcache, bool sync)
{ {
struct address_space *mapping; struct address_space *mapping;
int rc; int rc;
...@@ -586,18 +586,28 @@ static int move_to_new_page(struct page *newpage, struct page *page, ...@@ -586,18 +586,28 @@ static int move_to_new_page(struct page *newpage, struct page *page,
mapping = page_mapping(page); mapping = page_mapping(page);
if (!mapping) if (!mapping)
rc = migrate_page(mapping, newpage, page); rc = migrate_page(mapping, newpage, page);
else if (mapping->a_ops->migratepage) else {
/* /*
* Most pages have a mapping and most filesystems * Do not writeback pages if !sync and migratepage is
* should provide a migration function. Anonymous * not pointing to migrate_page() which is nonblocking
* pages are part of swap space which also has its * (swapcache/tmpfs uses migratepage = migrate_page).
* own migration function. This is the most common
* path for page migration.
*/ */
rc = mapping->a_ops->migratepage(mapping, if (PageDirty(page) && !sync &&
newpage, page); mapping->a_ops->migratepage != migrate_page)
else rc = -EBUSY;
rc = fallback_migrate_page(mapping, newpage, page); else if (mapping->a_ops->migratepage)
/*
* Most pages have a mapping and most filesystems
* should provide a migration function. Anonymous
* pages are part of swap space which also has its
* own migration function. This is the most common
* path for page migration.
*/
rc = mapping->a_ops->migratepage(mapping,
newpage, page);
else
rc = fallback_migrate_page(mapping, newpage, page);
}
if (rc) { if (rc) {
newpage->mapping = NULL; newpage->mapping = NULL;
...@@ -641,7 +651,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private, ...@@ -641,7 +651,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
rc = -EAGAIN; rc = -EAGAIN;
if (!trylock_page(page)) { if (!trylock_page(page)) {
if (!force) if (!force || !sync)
goto move_newpage; goto move_newpage;
/* /*
...@@ -686,7 +696,15 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private, ...@@ -686,7 +696,15 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
BUG_ON(charge); BUG_ON(charge);
if (PageWriteback(page)) { if (PageWriteback(page)) {
if (!force || !sync) /*
* For !sync, there is no point retrying as the retry loop
* is expected to be too short for PageWriteback to be cleared
*/
if (!sync) {
rc = -EBUSY;
goto uncharge;
}
if (!force)
goto uncharge; goto uncharge;
wait_on_page_writeback(page); wait_on_page_writeback(page);
} }
...@@ -757,7 +775,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private, ...@@ -757,7 +775,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
skip_unmap: skip_unmap:
if (!page_mapped(page)) if (!page_mapped(page))
rc = move_to_new_page(newpage, page, remap_swapcache); rc = move_to_new_page(newpage, page, remap_swapcache, sync);
if (rc && remap_swapcache) if (rc && remap_swapcache)
remove_migration_ptes(page, page); remove_migration_ptes(page, page);
...@@ -850,7 +868,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, ...@@ -850,7 +868,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
if (!page_mapped(hpage)) if (!page_mapped(hpage))
rc = move_to_new_page(new_hpage, hpage, 1); rc = move_to_new_page(new_hpage, hpage, 1, sync);
if (rc) if (rc)
remove_migration_ptes(hpage, hpage); remove_migration_ptes(hpage, hpage);
......
...@@ -2103,7 +2103,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, ...@@ -2103,7 +2103,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
sync_migration); sync_migration);
if (page) if (page)
goto got_pg; goto got_pg;
sync_migration = true; sync_migration = !(gfp_mask & __GFP_NO_KSWAPD);
/* Try direct reclaim and then allocating */ /* Try direct reclaim and then allocating */
page = __alloc_pages_direct_reclaim(gfp_mask, order, page = __alloc_pages_direct_reclaim(gfp_mask, order,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment