Commit 93e4a89a authored by KOSAKI Motohiro's avatar KOSAKI Motohiro Committed by Linus Torvalds

mm: restore zone->all_unreclaimable to independence word

commit e815af95 ("change all_unreclaimable zone member to flags") changed
all_unreclaimable member to bit flag.  But it had an undesireble side
effect.  free_one_page() is one of most hot path in linux kernel and
increasing atomic ops in it can reduce kernel performance a bit.

Thus, this patch revert such commit partially. at least
all_unreclaimable shouldn't share memory word with other zone flags.

[akpm@linux-foundation.org: fix patch interaction]
Signed-off-by: default avatarKOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Huang Shijie <shijie8@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent fc91668e
...@@ -306,6 +306,7 @@ struct zone { ...@@ -306,6 +306,7 @@ struct zone {
* free areas of different sizes * free areas of different sizes
*/ */
spinlock_t lock; spinlock_t lock;
int all_unreclaimable; /* All pages pinned */
#ifdef CONFIG_MEMORY_HOTPLUG #ifdef CONFIG_MEMORY_HOTPLUG
/* see spanned/present_pages for more description */ /* see spanned/present_pages for more description */
seqlock_t span_seqlock; seqlock_t span_seqlock;
...@@ -417,7 +418,6 @@ struct zone { ...@@ -417,7 +418,6 @@ struct zone {
} ____cacheline_internodealigned_in_smp; } ____cacheline_internodealigned_in_smp;
typedef enum { typedef enum {
ZONE_ALL_UNRECLAIMABLE, /* all pages pinned */
ZONE_RECLAIM_LOCKED, /* prevents concurrent reclaim */ ZONE_RECLAIM_LOCKED, /* prevents concurrent reclaim */
ZONE_OOM_LOCKED, /* zone is in OOM killer zonelist */ ZONE_OOM_LOCKED, /* zone is in OOM killer zonelist */
} zone_flags_t; } zone_flags_t;
...@@ -437,11 +437,6 @@ static inline void zone_clear_flag(struct zone *zone, zone_flags_t flag) ...@@ -437,11 +437,6 @@ static inline void zone_clear_flag(struct zone *zone, zone_flags_t flag)
clear_bit(flag, &zone->flags); clear_bit(flag, &zone->flags);
} }
static inline int zone_is_all_unreclaimable(const struct zone *zone)
{
return test_bit(ZONE_ALL_UNRECLAIMABLE, &zone->flags);
}
static inline int zone_is_reclaim_locked(const struct zone *zone) static inline int zone_is_reclaim_locked(const struct zone *zone)
{ {
return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags); return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags);
......
...@@ -530,7 +530,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, ...@@ -530,7 +530,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
int batch_free = 0; int batch_free = 0;
spin_lock(&zone->lock); spin_lock(&zone->lock);
zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE); zone->all_unreclaimable = 0;
zone->pages_scanned = 0; zone->pages_scanned = 0;
__mod_zone_page_state(zone, NR_FREE_PAGES, count); __mod_zone_page_state(zone, NR_FREE_PAGES, count);
...@@ -568,7 +568,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order, ...@@ -568,7 +568,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order,
int migratetype) int migratetype)
{ {
spin_lock(&zone->lock); spin_lock(&zone->lock);
zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE); zone->all_unreclaimable = 0;
zone->pages_scanned = 0; zone->pages_scanned = 0;
__mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order); __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
...@@ -2262,7 +2262,7 @@ void show_free_areas(void) ...@@ -2262,7 +2262,7 @@ void show_free_areas(void)
K(zone_page_state(zone, NR_BOUNCE)), K(zone_page_state(zone, NR_BOUNCE)),
K(zone_page_state(zone, NR_WRITEBACK_TEMP)), K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
zone->pages_scanned, zone->pages_scanned,
(zone_is_all_unreclaimable(zone) ? "yes" : "no") (zone->all_unreclaimable ? "yes" : "no")
); );
printk("lowmem_reserve[]:"); printk("lowmem_reserve[]:");
for (i = 0; i < MAX_NR_ZONES; i++) for (i = 0; i < MAX_NR_ZONES; i++)
......
...@@ -1699,8 +1699,7 @@ static void shrink_zones(int priority, struct zonelist *zonelist, ...@@ -1699,8 +1699,7 @@ static void shrink_zones(int priority, struct zonelist *zonelist,
continue; continue;
note_zone_scanning_priority(zone, priority); note_zone_scanning_priority(zone, priority);
if (zone_is_all_unreclaimable(zone) && if (zone->all_unreclaimable && priority != DEF_PRIORITY)
priority != DEF_PRIORITY)
continue; /* Let kswapd poll it */ continue; /* Let kswapd poll it */
sc->all_unreclaimable = 0; sc->all_unreclaimable = 0;
} else { } else {
...@@ -1927,7 +1926,7 @@ static int sleeping_prematurely(pg_data_t *pgdat, int order, long remaining) ...@@ -1927,7 +1926,7 @@ static int sleeping_prematurely(pg_data_t *pgdat, int order, long remaining)
if (!populated_zone(zone)) if (!populated_zone(zone))
continue; continue;
if (zone_is_all_unreclaimable(zone)) if (zone->all_unreclaimable)
continue; continue;
if (!zone_watermark_ok(zone, order, high_wmark_pages(zone), if (!zone_watermark_ok(zone, order, high_wmark_pages(zone),
...@@ -2017,8 +2016,7 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order) ...@@ -2017,8 +2016,7 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
if (!populated_zone(zone)) if (!populated_zone(zone))
continue; continue;
if (zone_is_all_unreclaimable(zone) && if (zone->all_unreclaimable && priority != DEF_PRIORITY)
priority != DEF_PRIORITY)
continue; continue;
/* /*
...@@ -2061,8 +2059,7 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order) ...@@ -2061,8 +2059,7 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
if (!populated_zone(zone)) if (!populated_zone(zone))
continue; continue;
if (zone_is_all_unreclaimable(zone) && if (zone->all_unreclaimable && priority != DEF_PRIORITY)
priority != DEF_PRIORITY)
continue; continue;
temp_priority[i] = priority; temp_priority[i] = priority;
...@@ -2089,12 +2086,11 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order) ...@@ -2089,12 +2086,11 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
lru_pages); lru_pages);
sc.nr_reclaimed += reclaim_state->reclaimed_slab; sc.nr_reclaimed += reclaim_state->reclaimed_slab;
total_scanned += sc.nr_scanned; total_scanned += sc.nr_scanned;
if (zone_is_all_unreclaimable(zone)) if (zone->all_unreclaimable)
continue; continue;
if (nr_slab == 0 && zone->pages_scanned >= if (nr_slab == 0 &&
(zone_reclaimable_pages(zone) * 6)) zone->pages_scanned >= (zone_reclaimable_pages(zone) * 6))
zone_set_flag(zone, zone->all_unreclaimable = 1;
ZONE_ALL_UNRECLAIMABLE);
/* /*
* If we've done a decent amount of scanning and * If we've done a decent amount of scanning and
* the reclaim ratio is low, start doing writepage * the reclaim ratio is low, start doing writepage
...@@ -2624,7 +2620,7 @@ int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) ...@@ -2624,7 +2620,7 @@ int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages) zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)
return ZONE_RECLAIM_FULL; return ZONE_RECLAIM_FULL;
if (zone_is_all_unreclaimable(zone)) if (zone->all_unreclaimable)
return ZONE_RECLAIM_FULL; return ZONE_RECLAIM_FULL;
/* /*
......
...@@ -763,7 +763,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, ...@@ -763,7 +763,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
"\n prev_priority: %i" "\n prev_priority: %i"
"\n start_pfn: %lu" "\n start_pfn: %lu"
"\n inactive_ratio: %u", "\n inactive_ratio: %u",
zone_is_all_unreclaimable(zone), zone->all_unreclaimable,
zone->prev_priority, zone->prev_priority,
zone->zone_start_pfn, zone->zone_start_pfn,
zone->inactive_ratio); zone->inactive_ratio);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment