Commit d3acb15a authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs

Pull iov_iter updates from Al Viro:
 "iov_iter cleanups and fixes.

  There are followups, but this is what had sat in -next this cycle. IMO
  the macro forest in there became much thinner and easier to follow..."

* 'work.iov_iter' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs: (37 commits)
  csum_and_copy_to_pipe_iter(): leave handling of csum_state to caller
  clean up copy_mc_pipe_to_iter()
  pipe_zero(): we don't need no stinkin' kmap_atomic()...
  iov_iter: clean csum_and_copy_...() primitives up a bit
  copy_page_from_iter(): don't need kmap_atomic() for kvec/bvec cases
  copy_page_to_iter(): don't bother with kmap_atomic() for bvec/kvec cases
  iterate_xarray(): only of the first iteration we might get offset != 0
  pull handling of ->iov_offset into iterate_{iovec,bvec,xarray}
  iov_iter: make iterator callbacks use base and len instead of iovec
  iov_iter: make the amount already copied available to iterator callbacks
  iov_iter: get rid of separate bvec and xarray callbacks
  iov_iter: teach iterate_{bvec,xarray}() about possible short copies
  iterate_bvec(): expand bvec.h macro forest, massage a bit
  iov_iter: unify iterate_iovec and iterate_kvec
  iov_iter: massage iterate_iovec and iterate_kvec to logics similar to iterate_bvec
  iterate_and_advance(): get rid of magic in case when n is 0
  csum_and_copy_to_iter(): massage into form closer to csum_and_copy_from_iter()
  iov_iter: replace iov_iter_copy_from_user_atomic() with iterator-advancing variant
  [xarray] iov_iter_npages(): just use DIV_ROUND_UP()
  iov_iter_npages(): don't bother with iterate_all_kinds()
  ...
parents f92a322a 6852df12
...@@ -890,3 +890,12 @@ been called or returned with non -EIOCBQUEUED code. ...@@ -890,3 +890,12 @@ been called or returned with non -EIOCBQUEUED code.
mnt_want_write_file() can now only be paired with mnt_drop_write_file(), mnt_want_write_file() can now only be paired with mnt_drop_write_file(),
whereas previously it could be paired with mnt_drop_write() as well. whereas previously it could be paired with mnt_drop_write() as well.
---
**mandatory**
iov_iter_copy_from_user_atomic() is gone; use copy_page_from_iter_atomic().
The difference is copy_page_from_iter_atomic() advances the iterator and
you don't need iov_iter_advance() after it. However, if you decide to use
only a part of obtained data, you should do iov_iter_revert().
...@@ -399,7 +399,7 @@ static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes, ...@@ -399,7 +399,7 @@ static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
/* /*
* Copy data from userspace to the current page * Copy data from userspace to the current page
*/ */
copied = iov_iter_copy_from_user_atomic(page, i, offset, count); copied = copy_page_from_iter_atomic(page, offset, count, i);
/* Flush processor's dcache for this page */ /* Flush processor's dcache for this page */
flush_dcache_page(page); flush_dcache_page(page);
...@@ -413,20 +413,19 @@ static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes, ...@@ -413,20 +413,19 @@ static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
* The rest of the btrfs_file_write code will fall * The rest of the btrfs_file_write code will fall
* back to page at a time copies after we return 0. * back to page at a time copies after we return 0.
*/ */
if (!PageUptodate(page) && copied < count) if (unlikely(copied < count)) {
copied = 0; if (!PageUptodate(page)) {
iov_iter_revert(i, copied);
copied = 0;
}
if (!copied)
break;
}
iov_iter_advance(i, copied);
write_bytes -= copied; write_bytes -= copied;
total_copied += copied; total_copied += copied;
offset += copied;
/* Return to btrfs_file_write_iter to fault page */ if (offset == PAGE_SIZE) {
if (unlikely(copied == 0))
break;
if (copied < PAGE_SIZE - offset) {
offset += copied;
} else {
pg++; pg++;
offset = 0; offset = 0;
} }
......
...@@ -1171,14 +1171,12 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, ...@@ -1171,14 +1171,12 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
if (mapping_writably_mapped(mapping)) if (mapping_writably_mapped(mapping))
flush_dcache_page(page); flush_dcache_page(page);
tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes); tmp = copy_page_from_iter_atomic(page, offset, bytes, ii);
flush_dcache_page(page); flush_dcache_page(page);
iov_iter_advance(ii, tmp);
if (!tmp) { if (!tmp) {
unlock_page(page); unlock_page(page);
put_page(page); put_page(page);
bytes = min(bytes, iov_iter_single_seg_count(ii));
goto again; goto again;
} }
......
...@@ -746,10 +746,6 @@ iomap_write_actor(struct inode *inode, loff_t pos, loff_t length, void *data, ...@@ -746,10 +746,6 @@ iomap_write_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
* Otherwise there's a nasty deadlock on copying from the * Otherwise there's a nasty deadlock on copying from the
* same page as we're writing to, without it being marked * same page as we're writing to, without it being marked
* up-to-date. * up-to-date.
*
* Not only is this an optimisation, but it is also required
* to check that the address is actually valid, when atomic
* usercopies are used, below.
*/ */
if (unlikely(iov_iter_fault_in_readable(i, bytes))) { if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
status = -EFAULT; status = -EFAULT;
...@@ -764,30 +760,29 @@ iomap_write_actor(struct inode *inode, loff_t pos, loff_t length, void *data, ...@@ -764,30 +760,29 @@ iomap_write_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
if (mapping_writably_mapped(inode->i_mapping)) if (mapping_writably_mapped(inode->i_mapping))
flush_dcache_page(page); flush_dcache_page(page);
copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes); copied = copy_page_from_iter_atomic(page, offset, bytes, i);
copied = iomap_write_end(inode, pos, bytes, copied, page, iomap, status = iomap_write_end(inode, pos, bytes, copied, page, iomap,
srcmap); srcmap);
cond_resched(); if (unlikely(copied != status))
iov_iter_revert(i, copied - status);
iov_iter_advance(i, copied); cond_resched();
if (unlikely(copied == 0)) { if (unlikely(status == 0)) {
/* /*
* If we were unable to copy any data at all, we must * A short copy made iomap_write_end() reject the
* fall back to a single segment length write. * thing entirely. Might be memory poisoning
* * halfway through, might be a race with munmap,
* If we didn't fallback here, we could livelock * might be severe memory pressure.
* because not all segments in the iov can be copied at
* once without a pagefault.
*/ */
bytes = min_t(unsigned long, PAGE_SIZE - offset, if (copied)
iov_iter_single_seg_count(i)); bytes = copied;
goto again; goto again;
} }
pos += copied; pos += status;
written += copied; written += status;
length -= copied; length -= status;
balance_dirty_pages_ratelimited(inode->i_mapping); balance_dirty_pages_ratelimited(inode->i_mapping);
} while (iov_iter_count(i) && length); } while (iov_iter_count(i) && length);
......
...@@ -1684,20 +1684,17 @@ static size_t ntfs_copy_from_user_iter(struct page **pages, unsigned nr_pages, ...@@ -1684,20 +1684,17 @@ static size_t ntfs_copy_from_user_iter(struct page **pages, unsigned nr_pages,
{ {
struct page **last_page = pages + nr_pages; struct page **last_page = pages + nr_pages;
size_t total = 0; size_t total = 0;
struct iov_iter data = *i;
unsigned len, copied; unsigned len, copied;
do { do {
len = PAGE_SIZE - ofs; len = PAGE_SIZE - ofs;
if (len > bytes) if (len > bytes)
len = bytes; len = bytes;
copied = iov_iter_copy_from_user_atomic(*pages, &data, ofs, copied = copy_page_from_iter_atomic(*pages, ofs, len, i);
len);
total += copied; total += copied;
bytes -= copied; bytes -= copied;
if (!bytes) if (!bytes)
break; break;
iov_iter_advance(&data, copied);
if (copied < len) if (copied < len)
goto err; goto err;
ofs = 0; ofs = 0;
...@@ -1866,34 +1863,24 @@ static ssize_t ntfs_perform_write(struct file *file, struct iov_iter *i, ...@@ -1866,34 +1863,24 @@ static ssize_t ntfs_perform_write(struct file *file, struct iov_iter *i,
if (likely(copied == bytes)) { if (likely(copied == bytes)) {
status = ntfs_commit_pages_after_write(pages, do_pages, status = ntfs_commit_pages_after_write(pages, do_pages,
pos, bytes); pos, bytes);
if (!status)
status = bytes;
} }
do { do {
unlock_page(pages[--do_pages]); unlock_page(pages[--do_pages]);
put_page(pages[do_pages]); put_page(pages[do_pages]);
} while (do_pages); } while (do_pages);
if (unlikely(status < 0)) if (unlikely(status < 0)) {
iov_iter_revert(i, copied);
break; break;
copied = status; }
cond_resched(); cond_resched();
if (unlikely(!copied)) { if (unlikely(copied < bytes)) {
size_t sc; iov_iter_revert(i, copied);
if (copied)
/* bytes = copied;
* We failed to copy anything. Fall back to single else if (bytes > PAGE_SIZE - ofs)
* segment length write. bytes = PAGE_SIZE - ofs;
*
* This is needed to avoid possible livelock in the
* case that all segments in the iov cannot be copied
* at once without a pagefault.
*/
sc = iov_iter_single_seg_count(i);
if (bytes > sc)
bytes = sc;
goto again; goto again;
} }
iov_iter_advance(i, copied);
pos += copied; pos += copied;
written += copied; written += copied;
balance_dirty_pages_ratelimited(mapping); balance_dirty_pages_ratelimited(mapping);
......
...@@ -19,21 +19,17 @@ struct kvec { ...@@ -19,21 +19,17 @@ struct kvec {
enum iter_type { enum iter_type {
/* iter types */ /* iter types */
ITER_IOVEC = 4, ITER_IOVEC,
ITER_KVEC = 8, ITER_KVEC,
ITER_BVEC = 16, ITER_BVEC,
ITER_PIPE = 32, ITER_PIPE,
ITER_DISCARD = 64, ITER_XARRAY,
ITER_XARRAY = 128, ITER_DISCARD,
}; };
struct iov_iter { struct iov_iter {
/* u8 iter_type;
* Bit 0 is the read/write bit, set if we're writing. bool data_source;
* Bit 1 is the BVEC_FLAG_NO_REF bit, set if type is a bvec and
* the caller isn't expecting to drop a page reference when done.
*/
unsigned int type;
size_t iov_offset; size_t iov_offset;
size_t count; size_t count;
union { union {
...@@ -55,7 +51,7 @@ struct iov_iter { ...@@ -55,7 +51,7 @@ struct iov_iter {
static inline enum iter_type iov_iter_type(const struct iov_iter *i) static inline enum iter_type iov_iter_type(const struct iov_iter *i)
{ {
return i->type & ~(READ | WRITE); return i->iter_type;
} }
static inline bool iter_is_iovec(const struct iov_iter *i) static inline bool iter_is_iovec(const struct iov_iter *i)
...@@ -90,7 +86,7 @@ static inline bool iov_iter_is_xarray(const struct iov_iter *i) ...@@ -90,7 +86,7 @@ static inline bool iov_iter_is_xarray(const struct iov_iter *i)
static inline unsigned char iov_iter_rw(const struct iov_iter *i) static inline unsigned char iov_iter_rw(const struct iov_iter *i)
{ {
return i->type & (READ | WRITE); return i->data_source ? WRITE : READ;
} }
/* /*
...@@ -119,11 +115,11 @@ static inline struct iovec iov_iter_iovec(const struct iov_iter *iter) ...@@ -119,11 +115,11 @@ static inline struct iovec iov_iter_iovec(const struct iov_iter *iter)
}; };
} }
size_t iov_iter_copy_from_user_atomic(struct page *page, size_t copy_page_from_iter_atomic(struct page *page, unsigned offset,
struct iov_iter *i, unsigned long offset, size_t bytes); size_t bytes, struct iov_iter *i);
void iov_iter_advance(struct iov_iter *i, size_t bytes); void iov_iter_advance(struct iov_iter *i, size_t bytes);
void iov_iter_revert(struct iov_iter *i, size_t bytes); void iov_iter_revert(struct iov_iter *i, size_t bytes);
int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes); int iov_iter_fault_in_readable(const struct iov_iter *i, size_t bytes);
size_t iov_iter_single_seg_count(const struct iov_iter *i); size_t iov_iter_single_seg_count(const struct iov_iter *i);
size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i); struct iov_iter *i);
...@@ -132,9 +128,7 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, ...@@ -132,9 +128,7 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i); size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i); size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);
bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i);
size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i); size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);
bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i);
static __always_inline __must_check static __always_inline __must_check
size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
...@@ -157,10 +151,11 @@ size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) ...@@ -157,10 +151,11 @@ size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
static __always_inline __must_check static __always_inline __must_check
bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
{ {
if (unlikely(!check_copy_size(addr, bytes, false))) size_t copied = copy_from_iter(addr, bytes, i);
return false; if (likely(copied == bytes))
else return true;
return _copy_from_iter_full(addr, bytes, i); iov_iter_revert(i, copied);
return false;
} }
static __always_inline __must_check static __always_inline __must_check
...@@ -175,10 +170,11 @@ size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) ...@@ -175,10 +170,11 @@ size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
static __always_inline __must_check static __always_inline __must_check
bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
{ {
if (unlikely(!check_copy_size(addr, bytes, false))) size_t copied = copy_from_iter_nocache(addr, bytes, i);
return false; if (likely(copied == bytes))
else return true;
return _copy_from_iter_full_nocache(addr, bytes, i); iov_iter_revert(i, copied);
return false;
} }
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
...@@ -278,7 +274,17 @@ struct csum_state { ...@@ -278,7 +274,17 @@ struct csum_state {
size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csstate, struct iov_iter *i); size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csstate, struct iov_iter *i);
size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i); size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
static __always_inline __must_check
bool csum_and_copy_from_iter_full(void *addr, size_t bytes,
__wsum *csum, struct iov_iter *i)
{
size_t copied = csum_and_copy_from_iter(addr, bytes, csum, i);
if (likely(copied == bytes))
return true;
iov_iter_revert(i, copied);
return false;
}
size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
struct iov_iter *i); struct iov_iter *i);
...@@ -294,8 +300,4 @@ ssize_t __import_iovec(int type, const struct iovec __user *uvec, ...@@ -294,8 +300,4 @@ ssize_t __import_iovec(int type, const struct iovec __user *uvec,
int import_single_range(int type, void __user *buf, size_t len, int import_single_range(int type, void __user *buf, size_t len,
struct iovec *iov, struct iov_iter *i); struct iovec *iov, struct iov_iter *i);
int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,
int (*f)(struct kvec *vec, void *context),
void *context);
#endif #endif
...@@ -80,16 +80,18 @@ static inline __sum16 csum16_sub(__sum16 csum, __be16 addend) ...@@ -80,16 +80,18 @@ static inline __sum16 csum16_sub(__sum16 csum, __be16 addend)
return csum16_add(csum, ~addend); return csum16_add(csum, ~addend);
} }
static inline __wsum static inline __wsum csum_shift(__wsum sum, int offset)
csum_block_add(__wsum csum, __wsum csum2, int offset)
{ {
u32 sum = (__force u32)csum2;
/* rotate sum to align it with a 16b boundary */ /* rotate sum to align it with a 16b boundary */
if (offset & 1) if (offset & 1)
sum = ror32(sum, 8); return (__force __wsum)ror32((__force u32)sum, 8);
return sum;
}
return csum_add(csum, (__force __wsum)sum); static inline __wsum
csum_block_add(__wsum csum, __wsum csum2, int offset)
{
return csum_add(csum, csum_shift(csum2, offset));
} }
static inline __wsum static inline __wsum
......
This diff is collapsed.
...@@ -3642,10 +3642,6 @@ ssize_t generic_perform_write(struct file *file, ...@@ -3642,10 +3642,6 @@ ssize_t generic_perform_write(struct file *file,
* Otherwise there's a nasty deadlock on copying from the * Otherwise there's a nasty deadlock on copying from the
* same page as we're writing to, without it being marked * same page as we're writing to, without it being marked
* up-to-date. * up-to-date.
*
* Not only is this an optimisation, but it is also required
* to check that the address is actually valid, when atomic
* usercopies are used, below.
*/ */
if (unlikely(iov_iter_fault_in_readable(i, bytes))) { if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
status = -EFAULT; status = -EFAULT;
...@@ -3665,33 +3661,31 @@ ssize_t generic_perform_write(struct file *file, ...@@ -3665,33 +3661,31 @@ ssize_t generic_perform_write(struct file *file,
if (mapping_writably_mapped(mapping)) if (mapping_writably_mapped(mapping))
flush_dcache_page(page); flush_dcache_page(page);
copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes); copied = copy_page_from_iter_atomic(page, offset, bytes, i);
flush_dcache_page(page); flush_dcache_page(page);
status = a_ops->write_end(file, mapping, pos, bytes, copied, status = a_ops->write_end(file, mapping, pos, bytes, copied,
page, fsdata); page, fsdata);
if (unlikely(status < 0)) if (unlikely(status != copied)) {
break; iov_iter_revert(i, copied - max(status, 0L));
copied = status; if (unlikely(status < 0))
break;
}
cond_resched(); cond_resched();
iov_iter_advance(i, copied); if (unlikely(status == 0)) {
if (unlikely(copied == 0)) {
/* /*
* If we were unable to copy any data at all, we must * A short copy made ->write_end() reject the
* fall back to a single segment length write. * thing entirely. Might be memory poisoning
* * halfway through, might be a race with munmap,
* If we didn't fallback here, we could livelock * might be severe memory pressure.
* because not all segments in the iov can be copied at
* once without a pagefault.
*/ */
bytes = min_t(unsigned long, PAGE_SIZE - offset, if (copied)
iov_iter_single_seg_count(i)); bytes = copied;
goto again; goto again;
} }
pos += copied; pos += status;
written += copied; written += status;
balance_dirty_pages_ratelimited(mapping); balance_dirty_pages_ratelimited(mapping);
} while (iov_iter_count(i)); } while (iov_iter_count(i));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment