Commit 9376130c authored by Michal Hocko's avatar Michal Hocko Committed by Linus Torvalds

mm/vmalloc: add support for __GFP_NOFAIL

Dave Chinner has mentioned that some of the xfs code would benefit from
kvmalloc support for __GFP_NOFAIL because they have allocations that
cannot fail and they do not fit into a single page.

The large part of the vmalloc implementation already complies with the
given gfp flags so there is no work for those to be done.  The area and
page table allocations are an exception to that.  Implement a retry loop
for those.

Add a short sleep before retrying.  1 jiffy is a completely random
timeout.  Ideally the retry would wait for an explicit event - e.g.  a
change to the vmalloc space change if the failure was caused by the
space fragmentation or depletion.  But there are multiple different
reasons to retry and this could become much more complex.  Keep the
retry simple for now and just sleep to prevent from hogging CPUs.

Link: https://lkml.kernel.org/r/20211122153233.9924-3-mhocko@kernel.orgSigned-off-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Ilya Dryomov <idryomov@gmail.com>
Cc: Jeff Layton <jlayton@kernel.org>
Cc: Neil Brown <neilb@suse.de>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 451769eb
...@@ -2847,6 +2847,8 @@ vm_area_alloc_pages(gfp_t gfp, int nid, ...@@ -2847,6 +2847,8 @@ vm_area_alloc_pages(gfp_t gfp, int nid,
* more permissive. * more permissive.
*/ */
if (!order) { if (!order) {
gfp_t bulk_gfp = gfp & ~__GFP_NOFAIL;
while (nr_allocated < nr_pages) { while (nr_allocated < nr_pages) {
unsigned int nr, nr_pages_request; unsigned int nr, nr_pages_request;
...@@ -2864,12 +2866,12 @@ vm_area_alloc_pages(gfp_t gfp, int nid, ...@@ -2864,12 +2866,12 @@ vm_area_alloc_pages(gfp_t gfp, int nid,
* but mempolcy want to alloc memory by interleaving. * but mempolcy want to alloc memory by interleaving.
*/ */
if (IS_ENABLED(CONFIG_NUMA) && nid == NUMA_NO_NODE) if (IS_ENABLED(CONFIG_NUMA) && nid == NUMA_NO_NODE)
nr = alloc_pages_bulk_array_mempolicy(gfp, nr = alloc_pages_bulk_array_mempolicy(bulk_gfp,
nr_pages_request, nr_pages_request,
pages + nr_allocated); pages + nr_allocated);
else else
nr = alloc_pages_bulk_array_node(gfp, nid, nr = alloc_pages_bulk_array_node(bulk_gfp, nid,
nr_pages_request, nr_pages_request,
pages + nr_allocated); pages + nr_allocated);
...@@ -2924,6 +2926,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, ...@@ -2924,6 +2926,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
{ {
const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
const gfp_t orig_gfp_mask = gfp_mask; const gfp_t orig_gfp_mask = gfp_mask;
bool nofail = gfp_mask & __GFP_NOFAIL;
unsigned long addr = (unsigned long)area->addr; unsigned long addr = (unsigned long)area->addr;
unsigned long size = get_vm_area_size(area); unsigned long size = get_vm_area_size(area);
unsigned long array_size; unsigned long array_size;
...@@ -2988,8 +2991,12 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, ...@@ -2988,8 +2991,12 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
flags = memalloc_noio_save(); flags = memalloc_noio_save();
ret = vmap_pages_range(addr, addr + size, prot, area->pages, do {
ret = vmap_pages_range(addr, addr + size, prot, area->pages,
page_shift); page_shift);
if (nofail && (ret < 0))
schedule_timeout_uninterruptible(1);
} while (nofail && (ret < 0));
if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
memalloc_nofs_restore(flags); memalloc_nofs_restore(flags);
...@@ -3084,9 +3091,14 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, ...@@ -3084,9 +3091,14 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align,
VM_UNINITIALIZED | vm_flags, start, end, node, VM_UNINITIALIZED | vm_flags, start, end, node,
gfp_mask, caller); gfp_mask, caller);
if (!area) { if (!area) {
bool nofail = gfp_mask & __GFP_NOFAIL;
warn_alloc(gfp_mask, NULL, warn_alloc(gfp_mask, NULL,
"vmalloc error: size %lu, vm_struct allocation failed", "vmalloc error: size %lu, vm_struct allocation failed%s",
real_size); real_size, (nofail) ? ". Retrying." : "");
if (nofail) {
schedule_timeout_uninterruptible(1);
goto again;
}
goto fail; goto fail;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment