Commit 49426420 authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds

mm: memcg: handle non-error OOM situations more gracefully

Commit 3812c8c8 ("mm: memcg: do not trap chargers with full
callstack on OOM") assumed that only a few places that can trigger a
memcg OOM situation do not return VM_FAULT_OOM, like optional page cache
readahead.  But there are many more and it's impractical to annotate
them all.

First of all, we don't want to invoke the OOM killer when the failed
allocation is gracefully handled, so defer the actual kill to the end of
the fault handling as well.  This simplifies the code quite a bit for
added bonus.

Second, since a failed allocation might not be the abrupt end of the
fault, the memcg OOM handler needs to be re-entrant until the fault
finishes for subsequent allocation attempts.  If an allocation is
attempted after the task already OOMed, allow it to bypass the limit so
that it can quickly finish the fault and invoke the OOM killer.
Reported-by: default avatarazurIt <azurit@pobox.sk>
Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: <stable@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent c88b05b2
...@@ -137,47 +137,24 @@ extern void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, ...@@ -137,47 +137,24 @@ extern void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
extern void mem_cgroup_replace_page_cache(struct page *oldpage, extern void mem_cgroup_replace_page_cache(struct page *oldpage,
struct page *newpage); struct page *newpage);
/** static inline void mem_cgroup_oom_enable(void)
* mem_cgroup_toggle_oom - toggle the memcg OOM killer for the current task
* @new: true to enable, false to disable
*
* Toggle whether a failed memcg charge should invoke the OOM killer
* or just return -ENOMEM. Returns the previous toggle state.
*
* NOTE: Any path that enables the OOM killer before charging must
* call mem_cgroup_oom_synchronize() afterward to finalize the
* OOM handling and clean up.
*/
static inline bool mem_cgroup_toggle_oom(bool new)
{ {
bool old; WARN_ON(current->memcg_oom.may_oom);
current->memcg_oom.may_oom = 1;
old = current->memcg_oom.may_oom;
current->memcg_oom.may_oom = new;
return old;
} }
static inline void mem_cgroup_enable_oom(void) static inline void mem_cgroup_oom_disable(void)
{ {
bool old = mem_cgroup_toggle_oom(true); WARN_ON(!current->memcg_oom.may_oom);
current->memcg_oom.may_oom = 0;
WARN_ON(old == true);
}
static inline void mem_cgroup_disable_oom(void)
{
bool old = mem_cgroup_toggle_oom(false);
WARN_ON(old == false);
} }
static inline bool task_in_memcg_oom(struct task_struct *p) static inline bool task_in_memcg_oom(struct task_struct *p)
{ {
return p->memcg_oom.in_memcg_oom; return p->memcg_oom.memcg;
} }
bool mem_cgroup_oom_synchronize(void); bool mem_cgroup_oom_synchronize(bool wait);
#ifdef CONFIG_MEMCG_SWAP #ifdef CONFIG_MEMCG_SWAP
extern int do_swap_account; extern int do_swap_account;
...@@ -402,16 +379,11 @@ static inline void mem_cgroup_end_update_page_stat(struct page *page, ...@@ -402,16 +379,11 @@ static inline void mem_cgroup_end_update_page_stat(struct page *page,
{ {
} }
static inline bool mem_cgroup_toggle_oom(bool new) static inline void mem_cgroup_oom_enable(void)
{
return false;
}
static inline void mem_cgroup_enable_oom(void)
{ {
} }
static inline void mem_cgroup_disable_oom(void) static inline void mem_cgroup_oom_disable(void)
{ {
} }
...@@ -420,7 +392,7 @@ static inline bool task_in_memcg_oom(struct task_struct *p) ...@@ -420,7 +392,7 @@ static inline bool task_in_memcg_oom(struct task_struct *p)
return false; return false;
} }
static inline bool mem_cgroup_oom_synchronize(void) static inline bool mem_cgroup_oom_synchronize(bool wait)
{ {
return false; return false;
} }
......
...@@ -1394,11 +1394,10 @@ struct task_struct { ...@@ -1394,11 +1394,10 @@ struct task_struct {
} memcg_batch; } memcg_batch;
unsigned int memcg_kmem_skip_account; unsigned int memcg_kmem_skip_account;
struct memcg_oom_info { struct memcg_oom_info {
struct mem_cgroup *memcg;
gfp_t gfp_mask;
int order;
unsigned int may_oom:1; unsigned int may_oom:1;
unsigned int in_memcg_oom:1;
unsigned int oom_locked:1;
int wakeups;
struct mem_cgroup *wait_on_memcg;
} memcg_oom; } memcg_oom;
#endif #endif
#ifdef CONFIG_UPROBES #ifdef CONFIG_UPROBES
......
...@@ -1616,7 +1616,6 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) ...@@ -1616,7 +1616,6 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
struct inode *inode = mapping->host; struct inode *inode = mapping->host;
pgoff_t offset = vmf->pgoff; pgoff_t offset = vmf->pgoff;
struct page *page; struct page *page;
bool memcg_oom;
pgoff_t size; pgoff_t size;
int ret = 0; int ret = 0;
...@@ -1625,11 +1624,7 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) ...@@ -1625,11 +1624,7 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
return VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
/* /*
* Do we have something in the page cache already? Either * Do we have something in the page cache already?
* way, try readahead, but disable the memcg OOM killer for it
* as readahead is optional and no errors are propagated up
* the fault stack. The OOM killer is enabled while trying to
* instantiate the faulting page individually below.
*/ */
page = find_get_page(mapping, offset); page = find_get_page(mapping, offset);
if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) { if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
...@@ -1637,14 +1632,10 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) ...@@ -1637,14 +1632,10 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
* We found the page, so try async readahead before * We found the page, so try async readahead before
* waiting for the lock. * waiting for the lock.
*/ */
memcg_oom = mem_cgroup_toggle_oom(false);
do_async_mmap_readahead(vma, ra, file, page, offset); do_async_mmap_readahead(vma, ra, file, page, offset);
mem_cgroup_toggle_oom(memcg_oom);
} else if (!page) { } else if (!page) {
/* No page in the page cache at all */ /* No page in the page cache at all */
memcg_oom = mem_cgroup_toggle_oom(false);
do_sync_mmap_readahead(vma, ra, file, offset); do_sync_mmap_readahead(vma, ra, file, offset);
mem_cgroup_toggle_oom(memcg_oom);
count_vm_event(PGMAJFAULT); count_vm_event(PGMAJFAULT);
mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
ret = VM_FAULT_MAJOR; ret = VM_FAULT_MAJOR;
......
...@@ -2161,110 +2161,59 @@ static void memcg_oom_recover(struct mem_cgroup *memcg) ...@@ -2161,110 +2161,59 @@ static void memcg_oom_recover(struct mem_cgroup *memcg)
memcg_wakeup_oom(memcg); memcg_wakeup_oom(memcg);
} }
/*
* try to call OOM killer
*/
static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
{ {
bool locked;
int wakeups;
if (!current->memcg_oom.may_oom) if (!current->memcg_oom.may_oom)
return; return;
current->memcg_oom.in_memcg_oom = 1;
/* /*
* As with any blocking lock, a contender needs to start * We are in the middle of the charge context here, so we
* listening for wakeups before attempting the trylock, * don't want to block when potentially sitting on a callstack
* otherwise it can miss the wakeup from the unlock and sleep * that holds all kinds of filesystem and mm locks.
* indefinitely. This is just open-coded because our locking *
* is so particular to memcg hierarchies. * Also, the caller may handle a failed allocation gracefully
* (like optional page cache readahead) and so an OOM killer
* invocation might not even be necessary.
*
* That's why we don't do anything here except remember the
* OOM context and then deal with it at the end of the page
* fault when the stack is unwound, the locks are released,
* and when we know whether the fault was overall successful.
*/ */
wakeups = atomic_read(&memcg->oom_wakeups); css_get(&memcg->css);
mem_cgroup_mark_under_oom(memcg); current->memcg_oom.memcg = memcg;
current->memcg_oom.gfp_mask = mask;
locked = mem_cgroup_oom_trylock(memcg); current->memcg_oom.order = order;
if (locked)
mem_cgroup_oom_notify(memcg);
if (locked && !memcg->oom_kill_disable) {
mem_cgroup_unmark_under_oom(memcg);
mem_cgroup_out_of_memory(memcg, mask, order);
mem_cgroup_oom_unlock(memcg);
/*
* There is no guarantee that an OOM-lock contender
* sees the wakeups triggered by the OOM kill
* uncharges. Wake any sleepers explicitely.
*/
memcg_oom_recover(memcg);
} else {
/*
* A system call can just return -ENOMEM, but if this
* is a page fault and somebody else is handling the
* OOM already, we need to sleep on the OOM waitqueue
* for this memcg until the situation is resolved.
* Which can take some time because it might be
* handled by a userspace task.
*
* However, this is the charge context, which means
* that we may sit on a large call stack and hold
* various filesystem locks, the mmap_sem etc. and we
* don't want the OOM handler to deadlock on them
* while we sit here and wait. Store the current OOM
* context in the task_struct, then return -ENOMEM.
* At the end of the page fault handler, with the
* stack unwound, pagefault_out_of_memory() will check
* back with us by calling
* mem_cgroup_oom_synchronize(), possibly putting the
* task to sleep.
*/
current->memcg_oom.oom_locked = locked;
current->memcg_oom.wakeups = wakeups;
css_get(&memcg->css);
current->memcg_oom.wait_on_memcg = memcg;
}
} }
/** /**
* mem_cgroup_oom_synchronize - complete memcg OOM handling * mem_cgroup_oom_synchronize - complete memcg OOM handling
* @handle: actually kill/wait or just clean up the OOM state
* *
* This has to be called at the end of a page fault if the the memcg * This has to be called at the end of a page fault if the memcg OOM
* OOM handler was enabled and the fault is returning %VM_FAULT_OOM. * handler was enabled.
* *
* Memcg supports userspace OOM handling, so failed allocations must * Memcg supports userspace OOM handling where failed allocations must
* sleep on a waitqueue until the userspace task resolves the * sleep on a waitqueue until the userspace task resolves the
* situation. Sleeping directly in the charge context with all kinds * situation. Sleeping directly in the charge context with all kinds
* of locks held is not a good idea, instead we remember an OOM state * of locks held is not a good idea, instead we remember an OOM state
* in the task and mem_cgroup_oom_synchronize() has to be called at * in the task and mem_cgroup_oom_synchronize() has to be called at
* the end of the page fault to put the task to sleep and clean up the * the end of the page fault to complete the OOM handling.
* OOM state.
* *
* Returns %true if an ongoing memcg OOM situation was detected and * Returns %true if an ongoing memcg OOM situation was detected and
* finalized, %false otherwise. * completed, %false otherwise.
*/ */
bool mem_cgroup_oom_synchronize(void) bool mem_cgroup_oom_synchronize(bool handle)
{ {
struct mem_cgroup *memcg = current->memcg_oom.memcg;
struct oom_wait_info owait; struct oom_wait_info owait;
struct mem_cgroup *memcg; bool locked;
/* OOM is global, do not handle */ /* OOM is global, do not handle */
if (!current->memcg_oom.in_memcg_oom)
return false;
/*
* We invoked the OOM killer but there is a chance that a kill
* did not free up any charges. Everybody else might already
* be sleeping, so restart the fault and keep the rampage
* going until some charges are released.
*/
memcg = current->memcg_oom.wait_on_memcg;
if (!memcg) if (!memcg)
goto out; return false;
if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current)) if (!handle)
goto out_memcg; goto cleanup;
owait.memcg = memcg; owait.memcg = memcg;
owait.wait.flags = 0; owait.wait.flags = 0;
...@@ -2273,13 +2222,25 @@ bool mem_cgroup_oom_synchronize(void) ...@@ -2273,13 +2222,25 @@ bool mem_cgroup_oom_synchronize(void)
INIT_LIST_HEAD(&owait.wait.task_list); INIT_LIST_HEAD(&owait.wait.task_list);
prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
/* Only sleep if we didn't miss any wakeups since OOM */ mem_cgroup_mark_under_oom(memcg);
if (atomic_read(&memcg->oom_wakeups) == current->memcg_oom.wakeups)
locked = mem_cgroup_oom_trylock(memcg);
if (locked)
mem_cgroup_oom_notify(memcg);
if (locked && !memcg->oom_kill_disable) {
mem_cgroup_unmark_under_oom(memcg);
finish_wait(&memcg_oom_waitq, &owait.wait);
mem_cgroup_out_of_memory(memcg, current->memcg_oom.gfp_mask,
current->memcg_oom.order);
} else {
schedule(); schedule();
finish_wait(&memcg_oom_waitq, &owait.wait); mem_cgroup_unmark_under_oom(memcg);
out_memcg: finish_wait(&memcg_oom_waitq, &owait.wait);
mem_cgroup_unmark_under_oom(memcg); }
if (current->memcg_oom.oom_locked) {
if (locked) {
mem_cgroup_oom_unlock(memcg); mem_cgroup_oom_unlock(memcg);
/* /*
* There is no guarantee that an OOM-lock contender * There is no guarantee that an OOM-lock contender
...@@ -2288,10 +2249,9 @@ bool mem_cgroup_oom_synchronize(void) ...@@ -2288,10 +2249,9 @@ bool mem_cgroup_oom_synchronize(void)
*/ */
memcg_oom_recover(memcg); memcg_oom_recover(memcg);
} }
cleanup:
current->memcg_oom.memcg = NULL;
css_put(&memcg->css); css_put(&memcg->css);
current->memcg_oom.wait_on_memcg = NULL;
out:
current->memcg_oom.in_memcg_oom = 0;
return true; return true;
} }
...@@ -2705,6 +2665,9 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm, ...@@ -2705,6 +2665,9 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
|| fatal_signal_pending(current))) || fatal_signal_pending(current)))
goto bypass; goto bypass;
if (unlikely(task_in_memcg_oom(current)))
goto bypass;
/* /*
* We always charge the cgroup the mm_struct belongs to. * We always charge the cgroup the mm_struct belongs to.
* The mm_struct's mem_cgroup changes on task migration if the * The mm_struct's mem_cgroup changes on task migration if the
......
...@@ -3865,15 +3865,21 @@ int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3865,15 +3865,21 @@ int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* space. Kernel faults are handled more gracefully. * space. Kernel faults are handled more gracefully.
*/ */
if (flags & FAULT_FLAG_USER) if (flags & FAULT_FLAG_USER)
mem_cgroup_enable_oom(); mem_cgroup_oom_enable();
ret = __handle_mm_fault(mm, vma, address, flags); ret = __handle_mm_fault(mm, vma, address, flags);
if (flags & FAULT_FLAG_USER) if (flags & FAULT_FLAG_USER) {
mem_cgroup_disable_oom(); mem_cgroup_oom_disable();
/*
if (WARN_ON(task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))) * The task may have entered a memcg OOM situation but
mem_cgroup_oom_synchronize(); * if the allocation error was handled gracefully (no
* VM_FAULT_OOM), there is no need to kill anything.
* Just clean up the OOM state peacefully.
*/
if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
mem_cgroup_oom_synchronize(false);
}
return ret; return ret;
} }
......
...@@ -680,7 +680,7 @@ void pagefault_out_of_memory(void) ...@@ -680,7 +680,7 @@ void pagefault_out_of_memory(void)
{ {
struct zonelist *zonelist; struct zonelist *zonelist;
if (mem_cgroup_oom_synchronize()) if (mem_cgroup_oom_synchronize(true))
return; return;
zonelist = node_zonelist(first_online_node, GFP_KERNEL); zonelist = node_zonelist(first_online_node, GFP_KERNEL);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment