Commit 499118e9 authored by Vlastimil Babka's avatar Vlastimil Babka Committed by Linus Torvalds

mm: introduce memalloc_noreclaim_{save,restore}

The previous patch ("mm: prevent potential recursive reclaim due to
clearing PF_MEMALLOC") has shown that simply setting and clearing
PF_MEMALLOC in current->flags can result in wrongly clearing a
pre-existing PF_MEMALLOC flag and potentially lead to recursive reclaim.
Let's introduce helpers that support proper nesting by saving the
previous stat of the flag, similar to the existing memalloc_noio_* and
memalloc_nofs_* helpers.  Convert existing setting/clearing of
PF_MEMALLOC within mm to the new helpers.

There are no known issues with the converted code, but the change makes
it more robust.

Link: http://lkml.kernel.org/r/20170405074700.29871-3-vbabka@suse.czSigned-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Suggested-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarHillf Danton <hillf.zj@alibaba-inc.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Boris Brezillon <boris.brezillon@free-electrons.com>
Cc: Chris Leech <cleech@redhat.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Josef Bacik <jbacik@fb.com>
Cc: Lee Duncan <lduncan@suse.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Richard Weinberger <richard@nod.at>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 62be1511
...@@ -191,4 +191,16 @@ static inline void memalloc_nofs_restore(unsigned int flags) ...@@ -191,4 +191,16 @@ static inline void memalloc_nofs_restore(unsigned int flags)
current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags; current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags;
} }
static inline unsigned int memalloc_noreclaim_save(void)
{
unsigned int flags = current->flags & PF_MEMALLOC;
current->flags |= PF_MEMALLOC;
return flags;
}
static inline void memalloc_noreclaim_restore(unsigned int flags)
{
current->flags = (current->flags & ~PF_MEMALLOC) | flags;
}
#endif /* _LINUX_SCHED_MM_H */ #endif /* _LINUX_SCHED_MM_H */
...@@ -3283,15 +3283,15 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, ...@@ -3283,15 +3283,15 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
enum compact_priority prio, enum compact_result *compact_result) enum compact_priority prio, enum compact_result *compact_result)
{ {
struct page *page; struct page *page;
unsigned int noreclaim_flag = current->flags & PF_MEMALLOC; unsigned int noreclaim_flag;
if (!order) if (!order)
return NULL; return NULL;
current->flags |= PF_MEMALLOC; noreclaim_flag = memalloc_noreclaim_save();
*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
prio); prio);
current->flags = (current->flags & ~PF_MEMALLOC) | noreclaim_flag; memalloc_noreclaim_restore(noreclaim_flag);
if (*compact_result <= COMPACT_INACTIVE) if (*compact_result <= COMPACT_INACTIVE)
return NULL; return NULL;
...@@ -3438,12 +3438,13 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, ...@@ -3438,12 +3438,13 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order,
{ {
struct reclaim_state reclaim_state; struct reclaim_state reclaim_state;
int progress; int progress;
unsigned int noreclaim_flag;
cond_resched(); cond_resched();
/* We now go into synchronous reclaim */ /* We now go into synchronous reclaim */
cpuset_memory_pressure_bump(); cpuset_memory_pressure_bump();
current->flags |= PF_MEMALLOC; noreclaim_flag = memalloc_noreclaim_save();
lockdep_set_current_reclaim_state(gfp_mask); lockdep_set_current_reclaim_state(gfp_mask);
reclaim_state.reclaimed_slab = 0; reclaim_state.reclaimed_slab = 0;
current->reclaim_state = &reclaim_state; current->reclaim_state = &reclaim_state;
...@@ -3453,7 +3454,7 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, ...@@ -3453,7 +3454,7 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order,
current->reclaim_state = NULL; current->reclaim_state = NULL;
lockdep_clear_current_reclaim_state(); lockdep_clear_current_reclaim_state();
current->flags &= ~PF_MEMALLOC; memalloc_noreclaim_restore(noreclaim_flag);
cond_resched(); cond_resched();
......
...@@ -3036,6 +3036,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, ...@@ -3036,6 +3036,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
struct zonelist *zonelist; struct zonelist *zonelist;
unsigned long nr_reclaimed; unsigned long nr_reclaimed;
int nid; int nid;
unsigned int noreclaim_flag;
struct scan_control sc = { struct scan_control sc = {
.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
.gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
...@@ -3062,9 +3063,9 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, ...@@ -3062,9 +3063,9 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
sc.gfp_mask, sc.gfp_mask,
sc.reclaim_idx); sc.reclaim_idx);
current->flags |= PF_MEMALLOC; noreclaim_flag = memalloc_noreclaim_save();
nr_reclaimed = do_try_to_free_pages(zonelist, &sc); nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
current->flags &= ~PF_MEMALLOC; memalloc_noreclaim_restore(noreclaim_flag);
trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
...@@ -3589,8 +3590,9 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) ...@@ -3589,8 +3590,9 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
struct task_struct *p = current; struct task_struct *p = current;
unsigned long nr_reclaimed; unsigned long nr_reclaimed;
unsigned int noreclaim_flag;
p->flags |= PF_MEMALLOC; noreclaim_flag = memalloc_noreclaim_save();
lockdep_set_current_reclaim_state(sc.gfp_mask); lockdep_set_current_reclaim_state(sc.gfp_mask);
reclaim_state.reclaimed_slab = 0; reclaim_state.reclaimed_slab = 0;
p->reclaim_state = &reclaim_state; p->reclaim_state = &reclaim_state;
...@@ -3599,7 +3601,7 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) ...@@ -3599,7 +3601,7 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
p->reclaim_state = NULL; p->reclaim_state = NULL;
lockdep_clear_current_reclaim_state(); lockdep_clear_current_reclaim_state();
p->flags &= ~PF_MEMALLOC; memalloc_noreclaim_restore(noreclaim_flag);
return nr_reclaimed; return nr_reclaimed;
} }
...@@ -3764,6 +3766,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in ...@@ -3764,6 +3766,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
struct task_struct *p = current; struct task_struct *p = current;
struct reclaim_state reclaim_state; struct reclaim_state reclaim_state;
int classzone_idx = gfp_zone(gfp_mask); int classzone_idx = gfp_zone(gfp_mask);
unsigned int noreclaim_flag;
struct scan_control sc = { struct scan_control sc = {
.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
.gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)), .gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)),
...@@ -3781,7 +3784,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in ...@@ -3781,7 +3784,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
* and we also need to be able to write out pages for RECLAIM_WRITE * and we also need to be able to write out pages for RECLAIM_WRITE
* and RECLAIM_UNMAP. * and RECLAIM_UNMAP.
*/ */
p->flags |= PF_MEMALLOC | PF_SWAPWRITE; noreclaim_flag = memalloc_noreclaim_save();
p->flags |= PF_SWAPWRITE;
lockdep_set_current_reclaim_state(gfp_mask); lockdep_set_current_reclaim_state(gfp_mask);
reclaim_state.reclaimed_slab = 0; reclaim_state.reclaimed_slab = 0;
p->reclaim_state = &reclaim_state; p->reclaim_state = &reclaim_state;
...@@ -3797,7 +3801,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in ...@@ -3797,7 +3801,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
} }
p->reclaim_state = NULL; p->reclaim_state = NULL;
current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE); current->flags &= ~PF_SWAPWRITE;
memalloc_noreclaim_restore(noreclaim_flag);
lockdep_clear_current_reclaim_state(); lockdep_clear_current_reclaim_state();
return sc.nr_reclaimed >= nr_pages; return sc.nr_reclaimed >= nr_pages;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment