Commit 1a16718c authored by Joonsoo Kim's avatar Joonsoo Kim Committed by Linus Torvalds

mm/compaction: correct to flush migrated pages if pageblock skip happens

We cache isolate_start_pfn before entering isolate_migratepages().  If
pageblock is skipped in isolate_migratepages() due to whatever reason,
cc->migrate_pfn can be far from isolate_start_pfn hence we flush pages
that were freed.  For example, the following scenario can be possible:

- assume order-9 compaction, pageblock order is 9
- start_isolate_pfn is 0x200
- isolate_migratepages()
  - skip a number of pageblocks
  - start to isolate from pfn 0x600
  - cc->migrate_pfn = 0x620
  - return
- last_migrated_pfn is set to 0x200
- check flushing condition
  - current_block_start is set to 0x600
  - last_migrated_pfn < current_block_start then do useless flush

This wrong flush would not help the performance and success rate so this
patch tries to fix it.  One simple way to know the exact position where
we start to isolate migratable pages is that we cache it in
isolate_migratepages() before entering actual isolation.  This patch
implements that and fixes the problem.
Signed-off-by: default avatarJoonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 82c1fc71
...@@ -1115,6 +1115,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, ...@@ -1115,6 +1115,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
struct compact_control *cc) struct compact_control *cc)
{ {
unsigned long low_pfn, end_pfn; unsigned long low_pfn, end_pfn;
unsigned long isolate_start_pfn;
struct page *page; struct page *page;
const isolate_mode_t isolate_mode = const isolate_mode_t isolate_mode =
(sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) | (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
...@@ -1163,6 +1164,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, ...@@ -1163,6 +1164,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
continue; continue;
/* Perform the isolation */ /* Perform the isolation */
isolate_start_pfn = low_pfn;
low_pfn = isolate_migratepages_block(cc, low_pfn, end_pfn, low_pfn = isolate_migratepages_block(cc, low_pfn, end_pfn,
isolate_mode); isolate_mode);
...@@ -1171,6 +1173,15 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, ...@@ -1171,6 +1173,15 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
return ISOLATE_ABORT; return ISOLATE_ABORT;
} }
/*
* Record where we could have freed pages by migration and not
* yet flushed them to buddy allocator.
* - this is the lowest page that could have been isolated and
* then freed by migration.
*/
if (cc->nr_migratepages && !cc->last_migrated_pfn)
cc->last_migrated_pfn = isolate_start_pfn;
/* /*
* Either we isolated something and proceed with migration. Or * Either we isolated something and proceed with migration. Or
* we failed and compact_zone should decide if we should * we failed and compact_zone should decide if we should
...@@ -1342,7 +1353,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) ...@@ -1342,7 +1353,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
unsigned long end_pfn = zone_end_pfn(zone); unsigned long end_pfn = zone_end_pfn(zone);
const int migratetype = gfpflags_to_migratetype(cc->gfp_mask); const int migratetype = gfpflags_to_migratetype(cc->gfp_mask);
const bool sync = cc->mode != MIGRATE_ASYNC; const bool sync = cc->mode != MIGRATE_ASYNC;
unsigned long last_migrated_pfn = 0;
ret = compaction_suitable(zone, cc->order, cc->alloc_flags, ret = compaction_suitable(zone, cc->order, cc->alloc_flags,
cc->classzone_idx); cc->classzone_idx);
...@@ -1380,6 +1390,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) ...@@ -1380,6 +1390,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
} }
cc->last_migrated_pfn = 0;
trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
cc->free_pfn, end_pfn, sync); cc->free_pfn, end_pfn, sync);
...@@ -1389,7 +1400,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) ...@@ -1389,7 +1400,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
while ((ret = compact_finished(zone, cc, migratetype)) == while ((ret = compact_finished(zone, cc, migratetype)) ==
COMPACT_CONTINUE) { COMPACT_CONTINUE) {
int err; int err;
unsigned long isolate_start_pfn = cc->migrate_pfn;
switch (isolate_migratepages(zone, cc)) { switch (isolate_migratepages(zone, cc)) {
case ISOLATE_ABORT: case ISOLATE_ABORT:
...@@ -1429,16 +1439,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) ...@@ -1429,16 +1439,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
} }
} }
/*
* Record where we could have freed pages by migration and not
* yet flushed them to buddy allocator. We use the pfn that
* isolate_migratepages() started from in this loop iteration
* - this is the lowest page that could have been isolated and
* then freed by migration.
*/
if (!last_migrated_pfn)
last_migrated_pfn = isolate_start_pfn;
check_drain: check_drain:
/* /*
* Has the migration scanner moved away from the previous * Has the migration scanner moved away from the previous
...@@ -1447,18 +1447,18 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) ...@@ -1447,18 +1447,18 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
* compact_finished() can detect immediately if allocation * compact_finished() can detect immediately if allocation
* would succeed. * would succeed.
*/ */
if (cc->order > 0 && last_migrated_pfn) { if (cc->order > 0 && cc->last_migrated_pfn) {
int cpu; int cpu;
unsigned long current_block_start = unsigned long current_block_start =
cc->migrate_pfn & ~((1UL << cc->order) - 1); cc->migrate_pfn & ~((1UL << cc->order) - 1);
if (last_migrated_pfn < current_block_start) { if (cc->last_migrated_pfn < current_block_start) {
cpu = get_cpu(); cpu = get_cpu();
lru_add_drain_cpu(cpu); lru_add_drain_cpu(cpu);
drain_local_pages(zone); drain_local_pages(zone);
put_cpu(); put_cpu();
/* No more flushing until we migrate again */ /* No more flushing until we migrate again */
last_migrated_pfn = 0; cc->last_migrated_pfn = 0;
} }
} }
......
...@@ -182,6 +182,7 @@ struct compact_control { ...@@ -182,6 +182,7 @@ struct compact_control {
unsigned long nr_migratepages; /* Number of pages to migrate */ unsigned long nr_migratepages; /* Number of pages to migrate */
unsigned long free_pfn; /* isolate_freepages search base */ unsigned long free_pfn; /* isolate_freepages search base */
unsigned long migrate_pfn; /* isolate_migratepages search base */ unsigned long migrate_pfn; /* isolate_migratepages search base */
unsigned long last_migrated_pfn;/* Not yet flushed page being freed */
enum migrate_mode mode; /* Async or sync migration mode */ enum migrate_mode mode; /* Async or sync migration mode */
bool ignore_skip_hint; /* Scan blocks even if marked skip */ bool ignore_skip_hint; /* Scan blocks even if marked skip */
int order; /* order a direct compactor needs */ int order; /* order a direct compactor needs */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment