Commit 02aa0cdd authored by Vlastimil Babka's avatar Vlastimil Babka Committed by Linus Torvalds

mm, page_alloc: count movable pages when stealing from pageblock

When stealing pages from pageblock of a different migratetype, we count
how many free pages were stolen, and change the pageblock's migratetype
if more than half of the pageblock was free.  This might be too
conservative, as there might be other pages that are not free, but were
allocated with the same migratetype as our allocation requested.

While we cannot determine the migratetype of allocated pages precisely
(at least without the page_owner functionality enabled), we can count
pages that compaction would try to isolate for migration - those are
either on LRU or __PageMovable().  The rest can be assumed to be
MIGRATE_RECLAIMABLE or MIGRATE_UNMOVABLE, which we cannot easily
distinguish.  This counting can be done as part of free page stealing
with little additional overhead.

The page stealing code is changed so that it considers free pages plus
pages of the "good" migratetype for the decision whether to change
pageblock's migratetype.

The result should be more accurate migratetype of pageblocks wrt the
actual pages in the pageblocks, when stealing from semi-occupied
pageblocks.  This should help the efficiency of page grouping by
mobility.

In testing based on 4.9 kernel with stress-highalloc from mmtests
configured for order-4 GFP_KERNEL allocations, this patch has reduced
the number of unmovable allocations falling back to movable pageblocks
by 47%.  The number of movable allocations falling back to other
pageblocks are increased by 55%, but these events don't cause permanent
fragmentation, so the tradeoff should be positive.  Later patches also
offset the movable fallback increase to some extent.

[akpm@linux-foundation.org: merge fix]
Link: http://lkml.kernel.org/r/20170307131545.28577-5-vbabka@suse.czSigned-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarMel Gorman <mgorman@techsingularity.net>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 3bc48f96
...@@ -33,10 +33,7 @@ bool has_unmovable_pages(struct zone *zone, struct page *page, int count, ...@@ -33,10 +33,7 @@ bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
bool skip_hwpoisoned_pages); bool skip_hwpoisoned_pages);
void set_pageblock_migratetype(struct page *page, int migratetype); void set_pageblock_migratetype(struct page *page, int migratetype);
int move_freepages_block(struct zone *zone, struct page *page, int move_freepages_block(struct zone *zone, struct page *page,
int migratetype); int migratetype, int *num_movable);
int move_freepages(struct zone *zone,
struct page *start_page, struct page *end_page,
int migratetype);
/* /*
* Changes migrate type in [start_pfn, end_pfn) to be MIGRATE_ISOLATE. * Changes migrate type in [start_pfn, end_pfn) to be MIGRATE_ISOLATE.
......
...@@ -1832,9 +1832,9 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone, ...@@ -1832,9 +1832,9 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
* Note that start_page and end_pages are not aligned on a pageblock * Note that start_page and end_pages are not aligned on a pageblock
* boundary. If alignment is required, use move_freepages_block() * boundary. If alignment is required, use move_freepages_block()
*/ */
int move_freepages(struct zone *zone, static int move_freepages(struct zone *zone,
struct page *start_page, struct page *end_page, struct page *start_page, struct page *end_page,
int migratetype) int migratetype, int *num_movable)
{ {
struct page *page; struct page *page;
unsigned int order; unsigned int order;
...@@ -1851,6 +1851,9 @@ int move_freepages(struct zone *zone, ...@@ -1851,6 +1851,9 @@ int move_freepages(struct zone *zone,
VM_BUG_ON(page_zone(start_page) != page_zone(end_page)); VM_BUG_ON(page_zone(start_page) != page_zone(end_page));
#endif #endif
if (num_movable)
*num_movable = 0;
for (page = start_page; page <= end_page;) { for (page = start_page; page <= end_page;) {
if (!pfn_valid_within(page_to_pfn(page))) { if (!pfn_valid_within(page_to_pfn(page))) {
page++; page++;
...@@ -1861,6 +1864,15 @@ int move_freepages(struct zone *zone, ...@@ -1861,6 +1864,15 @@ int move_freepages(struct zone *zone,
VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
if (!PageBuddy(page)) { if (!PageBuddy(page)) {
/*
* We assume that pages that could be isolated for
* migration are movable. But we don't actually try
* isolating, as that would be expensive.
*/
if (num_movable &&
(PageLRU(page) || __PageMovable(page)))
(*num_movable)++;
page++; page++;
continue; continue;
} }
...@@ -1876,7 +1888,7 @@ int move_freepages(struct zone *zone, ...@@ -1876,7 +1888,7 @@ int move_freepages(struct zone *zone,
} }
int move_freepages_block(struct zone *zone, struct page *page, int move_freepages_block(struct zone *zone, struct page *page,
int migratetype) int migratetype, int *num_movable)
{ {
unsigned long start_pfn, end_pfn; unsigned long start_pfn, end_pfn;
struct page *start_page, *end_page; struct page *start_page, *end_page;
...@@ -1893,7 +1905,8 @@ int move_freepages_block(struct zone *zone, struct page *page, ...@@ -1893,7 +1905,8 @@ int move_freepages_block(struct zone *zone, struct page *page,
if (!zone_spans_pfn(zone, end_pfn)) if (!zone_spans_pfn(zone, end_pfn))
return 0; return 0;
return move_freepages(zone, start_page, end_page, migratetype); return move_freepages(zone, start_page, end_page, migratetype,
num_movable);
} }
static void change_pageblock_range(struct page *pageblock_page, static void change_pageblock_range(struct page *pageblock_page,
...@@ -1943,22 +1956,26 @@ static bool can_steal_fallback(unsigned int order, int start_mt) ...@@ -1943,22 +1956,26 @@ static bool can_steal_fallback(unsigned int order, int start_mt)
/* /*
* This function implements actual steal behaviour. If order is large enough, * This function implements actual steal behaviour. If order is large enough,
* we can steal whole pageblock. If not, we first move freepages in this * we can steal whole pageblock. If not, we first move freepages in this
* pageblock and check whether half of pages are moved or not. If half of * pageblock to our migratetype and determine how many already-allocated pages
* pages are moved, we can change migratetype of pageblock and permanently * are there in the pageblock with a compatible migratetype. If at least half
* use it's pages as requested migratetype in the future. * of pages are free or compatible, we can change migratetype of the pageblock
* itself, so pages freed in the future will be put on the correct free list.
*/ */
static void steal_suitable_fallback(struct zone *zone, struct page *page, static void steal_suitable_fallback(struct zone *zone, struct page *page,
int start_type, bool whole_block) int start_type, bool whole_block)
{ {
unsigned int current_order = page_order(page); unsigned int current_order = page_order(page);
struct free_area *area; struct free_area *area;
int pages; int free_pages, movable_pages, alike_pages;
int old_block_type;
old_block_type = get_pageblock_migratetype(page);
/* /*
* This can happen due to races and we want to prevent broken * This can happen due to races and we want to prevent broken
* highatomic accounting. * highatomic accounting.
*/ */
if (is_migrate_highatomic_page(page)) if (is_migrate_highatomic(old_block_type))
goto single_page; goto single_page;
/* Take ownership for orders >= pageblock_order */ /* Take ownership for orders >= pageblock_order */
...@@ -1971,13 +1988,39 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page, ...@@ -1971,13 +1988,39 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page,
if (!whole_block) if (!whole_block)
goto single_page; goto single_page;
pages = move_freepages_block(zone, page, start_type); free_pages = move_freepages_block(zone, page, start_type,
&movable_pages);
/*
* Determine how many pages are compatible with our allocation.
* For movable allocation, it's the number of movable pages which
* we just obtained. For other types it's a bit more tricky.
*/
if (start_type == MIGRATE_MOVABLE) {
alike_pages = movable_pages;
} else {
/*
* If we are falling back a RECLAIMABLE or UNMOVABLE allocation
* to MOVABLE pageblock, consider all non-movable pages as
* compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
* vice versa, be conservative since we can't distinguish the
* exact migratetype of non-movable pages.
*/
if (old_block_type == MIGRATE_MOVABLE)
alike_pages = pageblock_nr_pages
- (free_pages + movable_pages);
else
alike_pages = 0;
}
/* moving whole block can fail due to zone boundary conditions */ /* moving whole block can fail due to zone boundary conditions */
if (!pages) if (!free_pages)
goto single_page; goto single_page;
/* Claim the whole block if over half of it is free */ /*
if (pages >= (1 << (pageblock_order-1)) || * If a sufficient number of pages in the block are either free or of
* comparable migratability as our allocation, claim the whole block.
*/
if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
page_group_by_mobility_disabled) page_group_by_mobility_disabled)
set_pageblock_migratetype(page, start_type); set_pageblock_migratetype(page, start_type);
...@@ -2055,7 +2098,7 @@ static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, ...@@ -2055,7 +2098,7 @@ static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
&& !is_migrate_cma(mt)) { && !is_migrate_cma(mt)) {
zone->nr_reserved_highatomic += pageblock_nr_pages; zone->nr_reserved_highatomic += pageblock_nr_pages;
set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
move_freepages_block(zone, page, MIGRATE_HIGHATOMIC); move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
} }
out_unlock: out_unlock:
...@@ -2132,7 +2175,8 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, ...@@ -2132,7 +2175,8 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
* may increase. * may increase.
*/ */
set_pageblock_migratetype(page, ac->migratetype); set_pageblock_migratetype(page, ac->migratetype);
ret = move_freepages_block(zone, page, ac->migratetype); ret = move_freepages_block(zone, page, ac->migratetype,
NULL);
if (ret) { if (ret) {
spin_unlock_irqrestore(&zone->lock, flags); spin_unlock_irqrestore(&zone->lock, flags);
return ret; return ret;
......
...@@ -66,7 +66,8 @@ static int set_migratetype_isolate(struct page *page, ...@@ -66,7 +66,8 @@ static int set_migratetype_isolate(struct page *page,
set_pageblock_migratetype(page, MIGRATE_ISOLATE); set_pageblock_migratetype(page, MIGRATE_ISOLATE);
zone->nr_isolate_pageblock++; zone->nr_isolate_pageblock++;
nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE); nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE,
NULL);
__mod_zone_freepage_state(zone, -nr_pages, migratetype); __mod_zone_freepage_state(zone, -nr_pages, migratetype);
} }
...@@ -120,7 +121,7 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype) ...@@ -120,7 +121,7 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype)
* pageblock scanning for freepage moving. * pageblock scanning for freepage moving.
*/ */
if (!isolated_page) { if (!isolated_page) {
nr_pages = move_freepages_block(zone, page, migratetype); nr_pages = move_freepages_block(zone, page, migratetype, NULL);
__mod_zone_freepage_state(zone, nr_pages, migratetype); __mod_zone_freepage_state(zone, nr_pages, migratetype);
} }
set_pageblock_migratetype(page, migratetype); set_pageblock_migratetype(page, migratetype);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment