Commit 5c075c5b authored by Fabio M. De Francesco's avatar Fabio M. De Francesco Committed by Christian Brauner

fs/aio: Stop allocating aio rings from HIGHMEM

There is no need to allocate aio rings from HIGHMEM because of very
little memory needed here.

Therefore, use GFP_USER flag in find_or_create_page() and get rid of
kmap*() mappings.

Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Ira Weiny <ira.weiny@intel.com>
Suggested-by: default avatarMatthew Wilcox <willy@infradead.org>
Signed-off-by: default avatarFabio M. De Francesco <fmdefrancesco@gmail.com>
Reviewed-by: default avatarIra Weiny <ira.weiny@intel.com>
Message-Id: <20230609145937.17610-1-fmdefrancesco@gmail.com>
Signed-off-by: default avatarChristian Brauner <brauner@kernel.org>
parent b6334e2c
...@@ -530,7 +530,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) ...@@ -530,7 +530,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
for (i = 0; i < nr_pages; i++) { for (i = 0; i < nr_pages; i++) {
struct page *page; struct page *page;
page = find_or_create_page(file->f_mapping, page = find_or_create_page(file->f_mapping,
i, GFP_HIGHUSER | __GFP_ZERO); i, GFP_USER | __GFP_ZERO);
if (!page) if (!page)
break; break;
pr_debug("pid(%d) page[%d]->count=%d\n", pr_debug("pid(%d) page[%d]->count=%d\n",
...@@ -571,7 +571,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) ...@@ -571,7 +571,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
ctx->user_id = ctx->mmap_base; ctx->user_id = ctx->mmap_base;
ctx->nr_events = nr_events; /* trusted copy */ ctx->nr_events = nr_events; /* trusted copy */
ring = kmap_atomic(ctx->ring_pages[0]); ring = page_address(ctx->ring_pages[0]);
ring->nr = nr_events; /* user copy */ ring->nr = nr_events; /* user copy */
ring->id = ~0U; ring->id = ~0U;
ring->head = ring->tail = 0; ring->head = ring->tail = 0;
...@@ -579,7 +579,6 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) ...@@ -579,7 +579,6 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
ring->compat_features = AIO_RING_COMPAT_FEATURES; ring->compat_features = AIO_RING_COMPAT_FEATURES;
ring->incompat_features = AIO_RING_INCOMPAT_FEATURES; ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
ring->header_length = sizeof(struct aio_ring); ring->header_length = sizeof(struct aio_ring);
kunmap_atomic(ring);
flush_dcache_page(ctx->ring_pages[0]); flush_dcache_page(ctx->ring_pages[0]);
return 0; return 0;
...@@ -682,9 +681,8 @@ static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) ...@@ -682,9 +681,8 @@ static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
* we are protected from page migration * we are protected from page migration
* changes ring_pages by ->ring_lock. * changes ring_pages by ->ring_lock.
*/ */
ring = kmap_atomic(ctx->ring_pages[0]); ring = page_address(ctx->ring_pages[0]);
ring->id = ctx->id; ring->id = ctx->id;
kunmap_atomic(ring);
return 0; return 0;
} }
...@@ -1025,9 +1023,8 @@ static void user_refill_reqs_available(struct kioctx *ctx) ...@@ -1025,9 +1023,8 @@ static void user_refill_reqs_available(struct kioctx *ctx)
* against ctx->completed_events below will make sure we do the * against ctx->completed_events below will make sure we do the
* safe/right thing. * safe/right thing.
*/ */
ring = kmap_atomic(ctx->ring_pages[0]); ring = page_address(ctx->ring_pages[0]);
head = ring->head; head = ring->head;
kunmap_atomic(ring);
refill_reqs_available(ctx, head, ctx->tail); refill_reqs_available(ctx, head, ctx->tail);
} }
...@@ -1133,12 +1130,11 @@ static void aio_complete(struct aio_kiocb *iocb) ...@@ -1133,12 +1130,11 @@ static void aio_complete(struct aio_kiocb *iocb)
if (++tail >= ctx->nr_events) if (++tail >= ctx->nr_events)
tail = 0; tail = 0;
ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); ev_page = page_address(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
event = ev_page + pos % AIO_EVENTS_PER_PAGE; event = ev_page + pos % AIO_EVENTS_PER_PAGE;
*event = iocb->ki_res; *event = iocb->ki_res;
kunmap_atomic(ev_page);
flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]);
pr_debug("%p[%u]: %p: %p %Lx %Lx %Lx\n", ctx, tail, iocb, pr_debug("%p[%u]: %p: %p %Lx %Lx %Lx\n", ctx, tail, iocb,
...@@ -1152,10 +1148,9 @@ static void aio_complete(struct aio_kiocb *iocb) ...@@ -1152,10 +1148,9 @@ static void aio_complete(struct aio_kiocb *iocb)
ctx->tail = tail; ctx->tail = tail;
ring = kmap_atomic(ctx->ring_pages[0]); ring = page_address(ctx->ring_pages[0]);
head = ring->head; head = ring->head;
ring->tail = tail; ring->tail = tail;
kunmap_atomic(ring);
flush_dcache_page(ctx->ring_pages[0]); flush_dcache_page(ctx->ring_pages[0]);
ctx->completed_events++; ctx->completed_events++;
...@@ -1215,10 +1210,9 @@ static long aio_read_events_ring(struct kioctx *ctx, ...@@ -1215,10 +1210,9 @@ static long aio_read_events_ring(struct kioctx *ctx,
mutex_lock(&ctx->ring_lock); mutex_lock(&ctx->ring_lock);
/* Access to ->ring_pages here is protected by ctx->ring_lock. */ /* Access to ->ring_pages here is protected by ctx->ring_lock. */
ring = kmap_atomic(ctx->ring_pages[0]); ring = page_address(ctx->ring_pages[0]);
head = ring->head; head = ring->head;
tail = ring->tail; tail = ring->tail;
kunmap_atomic(ring);
/* /*
* Ensure that once we've read the current tail pointer, that * Ensure that once we've read the current tail pointer, that
...@@ -1250,10 +1244,9 @@ static long aio_read_events_ring(struct kioctx *ctx, ...@@ -1250,10 +1244,9 @@ static long aio_read_events_ring(struct kioctx *ctx,
avail = min(avail, nr - ret); avail = min(avail, nr - ret);
avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos);
ev = kmap(page); ev = page_address(page);
copy_ret = copy_to_user(event + ret, ev + pos, copy_ret = copy_to_user(event + ret, ev + pos,
sizeof(*ev) * avail); sizeof(*ev) * avail);
kunmap(page);
if (unlikely(copy_ret)) { if (unlikely(copy_ret)) {
ret = -EFAULT; ret = -EFAULT;
...@@ -1265,9 +1258,8 @@ static long aio_read_events_ring(struct kioctx *ctx, ...@@ -1265,9 +1258,8 @@ static long aio_read_events_ring(struct kioctx *ctx,
head %= ctx->nr_events; head %= ctx->nr_events;
} }
ring = kmap_atomic(ctx->ring_pages[0]); ring = page_address(ctx->ring_pages[0]);
ring->head = head; ring->head = head;
kunmap_atomic(ring);
flush_dcache_page(ctx->ring_pages[0]); flush_dcache_page(ctx->ring_pages[0]);
pr_debug("%li h%u t%u\n", ret, head, tail); pr_debug("%li h%u t%u\n", ret, head, tail);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment