Skip to content

L
linux
Commit 30bff594 authored by Kent Overstreet's avatar Kent Overstreet
Browse files
Options

bcachefs: Initial folio conversion 

This converts fs-io.c to pass folios, not pages. We're not handling
large folios yet, there's no functional changes in this patch - just a
lot of churn doing the initial type conversions.
Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
parent 3342ac13
Hide whitespace changes
Inline Side-by-side
Showing with 317 additions and 296 deletions
fs/bcachefs/fs-io.c
View file @ 30bff594
...@@ -35,6 +35,26 @@ ...@@ -35,6 +35,26 @@
#include <trace/events/writeback.h> #include <trace/events/writeback.h>
static inline loff_t folio_end_pos(struct folio *folio)
{
return folio_pos(folio) + folio_size(folio);
}
static inline size_t folio_sectors(struct folio *folio)
{
return PAGE_SECTORS << folio_order(folio);
}
static inline loff_t folio_sector(struct folio *folio)
{
return folio_pos(folio) >> 9;
}
static inline loff_t folio_end_sector(struct folio *folio)
{
return folio_end_pos(folio) >> 9;
}
struct nocow_flush { struct nocow_flush {
struct closure *cl; struct closure *cl;
struct bch_dev *ca; struct bch_dev *ca;
...@@ -364,35 +384,34 @@ struct bch_folio { ...@@ -364,35 +384,34 @@ struct bch_folio {
struct bch_folio_sector s[PAGE_SECTORS]; struct bch_folio_sector s[PAGE_SECTORS];
}; };
static inline struct bch_folio *__bch2_folio(struct page *page) static inline struct bch_folio *__bch2_folio(struct folio *folio)
{ {
return page_has_private(page) return folio_has_private(folio)
? (struct bch_folio *) page_private(page) ? (struct bch_folio *) folio_get_private(folio)
: NULL; : NULL;
} }
static inline struct bch_folio *bch2_folio(struct page *page) static inline struct bch_folio *bch2_folio(struct folio *folio)
{ {
EBUG_ON(!PageLocked(page)); EBUG_ON(!folio_test_locked(folio));
return __bch2_folio(page); return __bch2_folio(folio);
} }
/* for newly allocated pages: */ /* for newly allocated folios: */
static void __bch2_folio_release(struct page *page) static void __bch2_folio_release(struct folio *folio)
{ {
kfree(detach_page_private(page)); kfree(folio_detach_private(folio));
} }
static void bch2_folio_release(struct page *page) static void bch2_folio_release(struct folio *folio)
{ {
EBUG_ON(!PageLocked(page)); EBUG_ON(!folio_test_locked(folio));
__bch2_folio_release(page); __bch2_folio_release(folio);
} }
/* for newly allocated pages: */ /* for newly allocated folios: */
static struct bch_folio *__bch2_folio_create(struct page *page, static struct bch_folio *__bch2_folio_create(struct folio *folio, gfp_t gfp)
gfp_t gfp)
{ {
struct bch_folio *s; struct bch_folio *s;
...@@ -401,14 +420,13 @@ static struct bch_folio *__bch2_folio_create(struct page *page, ...@@ -401,14 +420,13 @@ static struct bch_folio *__bch2_folio_create(struct page *page,
return NULL; return NULL;
spin_lock_init(&s->lock); spin_lock_init(&s->lock);
attach_page_private(page, s); folio_attach_private(folio, s);
return s; return s;
} }
static struct bch_folio *bch2_folio_create(struct page *page, static struct bch_folio *bch2_folio_create(struct folio *folio, gfp_t gfp)
gfp_t gfp)
{ {
return bch2_folio(page) ?: __bch2_folio_create(page, gfp); return bch2_folio(folio) ?: __bch2_folio_create(folio, gfp);
} }
static unsigned bkey_to_sector_state(struct bkey_s_c k) static unsigned bkey_to_sector_state(struct bkey_s_c k)
...@@ -420,11 +438,11 @@ static unsigned bkey_to_sector_state(struct bkey_s_c k) ...@@ -420,11 +438,11 @@ static unsigned bkey_to_sector_state(struct bkey_s_c k)
return SECTOR_UNALLOCATED; return SECTOR_UNALLOCATED;
} }
static void __bch2_folio_set(struct page *page, static void __bch2_folio_set(struct folio *folio,
unsigned pg_offset, unsigned pg_len, unsigned pg_offset, unsigned pg_len,
unsigned nr_ptrs, unsigned state) unsigned nr_ptrs, unsigned state)
{ {
struct bch_folio *s = bch2_folio_create(page, __GFP_NOFAIL); struct bch_folio *s = bch2_folio_create(folio, __GFP_NOFAIL);
unsigned i; unsigned i;
BUG_ON(pg_offset >= PAGE_SECTORS); BUG_ON(pg_offset >= PAGE_SECTORS);
...@@ -448,13 +466,13 @@ static void __bch2_folio_set(struct page *page, ...@@ -448,13 +466,13 @@ static void __bch2_folio_set(struct page *page,
* extents btree: * extents btree:
*/ */
static int bch2_folio_set(struct bch_fs *c, subvol_inum inum, static int bch2_folio_set(struct bch_fs *c, subvol_inum inum,
struct page **pages, unsigned nr_pages) struct folio **folios, unsigned nr_folios)
{ {
struct btree_trans trans; struct btree_trans trans;
struct btree_iter iter; struct btree_iter iter;
struct bkey_s_c k; struct bkey_s_c k;
u64 offset = pages[0]->index << PAGE_SECTORS_SHIFT; u64 offset = folio_sector(folios[0]);
unsigned pg_idx = 0; unsigned folio_idx = 0;
u32 snapshot; u32 snapshot;
int ret; int ret;
...@@ -472,25 +490,25 @@ static int bch2_folio_set(struct bch_fs *c, subvol_inum inum, ...@@ -472,25 +490,25 @@ static int bch2_folio_set(struct bch_fs *c, subvol_inum inum,
unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k); unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k);
unsigned state = bkey_to_sector_state(k); unsigned state = bkey_to_sector_state(k);
while (pg_idx < nr_pages) { while (folio_idx < nr_folios) {
struct page *page = pages[pg_idx]; struct folio *folio = folios[folio_idx];
u64 pg_start = page->index << PAGE_SECTORS_SHIFT; u64 folio_start = folio_sector(folio);
u64 pg_end = (page->index + 1) << PAGE_SECTORS_SHIFT; u64 folio_end = folio_end_sector(folio);
unsigned pg_offset = max(bkey_start_offset(k.k), pg_start) - pg_start; unsigned folio_offset = max(bkey_start_offset(k.k), folio_start) - folio_start;
unsigned pg_len = min(k.k->p.offset, pg_end) - pg_offset - pg_start; unsigned folio_len = min(k.k->p.offset, folio_end) - folio_offset - folio_start;
BUG_ON(k.k->p.offset < pg_start); BUG_ON(k.k->p.offset < folio_start);
BUG_ON(bkey_start_offset(k.k) > pg_end); BUG_ON(bkey_start_offset(k.k) > folio_end);
if (!bch2_folio_create(page, __GFP_NOFAIL)->uptodate) if (!bch2_folio_create(folio, __GFP_NOFAIL)->uptodate)
__bch2_folio_set(page, pg_offset, pg_len, nr_ptrs, state); __bch2_folio_set(folio, folio_offset, folio_len, nr_ptrs, state);
if (k.k->p.offset < pg_end) if (k.k->p.offset < folio_end)
break; break;
pg_idx++; folio_idx++;
} }
if (pg_idx == nr_pages) if (folio_idx == nr_folios)
break; break;
} }
...@@ -513,8 +531,8 @@ static void bch2_bio_page_state_set(struct bio *bio, struct bkey_s_c k) ...@@ -513,8 +531,8 @@ static void bch2_bio_page_state_set(struct bio *bio, struct bkey_s_c k)
unsigned state = bkey_to_sector_state(k); unsigned state = bkey_to_sector_state(k);
bio_for_each_segment(bv, bio, iter) bio_for_each_segment(bv, bio, iter)
__bch2_folio_set(bv.bv_page, bv.bv_offset >> 9, __bch2_folio_set(page_folio(bv.bv_page), bv.bv_offset >> 9,
bv.bv_len >> 9, nr_ptrs, state); bv.bv_len >> 9, nr_ptrs, state);
} }
static void mark_pagecache_unallocated(struct bch_inode_info *inode, static void mark_pagecache_unallocated(struct bch_inode_info *inode,
...@@ -534,22 +552,22 @@ static void mark_pagecache_unallocated(struct bch_inode_info *inode, ...@@ -534,22 +552,22 @@ static void mark_pagecache_unallocated(struct bch_inode_info *inode,
&index, end_index, &fbatch)) { &index, end_index, &fbatch)) {
for (i = 0; i < folio_batch_count(&fbatch); i++) { for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i]; struct folio *folio = fbatch.folios[i];
u64 pg_start = folio->index << PAGE_SECTORS_SHIFT; u64 folio_start = folio->index << PAGE_SECTORS_SHIFT;
u64 pg_end = (folio->index + 1) << PAGE_SECTORS_SHIFT; u64 folio_end = (folio->index + 1) << PAGE_SECTORS_SHIFT;
unsigned pg_offset = max(start, pg_start) - pg_start; unsigned folio_offset = max(start, folio_start) - folio_start;
unsigned pg_len = min(end, pg_end) - pg_offset - pg_start; unsigned folio_len = min(end, folio_end) - folio_offset - folio_start;
struct bch_folio *s; struct bch_folio *s;
BUG_ON(end <= pg_start); BUG_ON(end <= folio_start);
BUG_ON(pg_offset >= PAGE_SECTORS); BUG_ON(folio_offset >= PAGE_SECTORS);
BUG_ON(pg_offset + pg_len > PAGE_SECTORS); BUG_ON(folio_offset + folio_len > PAGE_SECTORS);
folio_lock(folio); folio_lock(folio);
s = bch2_folio(&folio->page); s = bch2_folio(folio);
if (s) { if (s) {
spin_lock(&s->lock); spin_lock(&s->lock);
for (j = pg_offset; j < pg_offset + pg_len; j++) for (j = folio_offset; j < folio_offset + folio_len; j++)
s->s[j].nr_replicas = 0; s->s[j].nr_replicas = 0;
spin_unlock(&s->lock); spin_unlock(&s->lock);
} }
...@@ -580,22 +598,22 @@ static void mark_pagecache_reserved(struct bch_inode_info *inode, ...@@ -580,22 +598,22 @@ static void mark_pagecache_reserved(struct bch_inode_info *inode,
&index, end_index, &fbatch)) { &index, end_index, &fbatch)) {
for (i = 0; i < folio_batch_count(&fbatch); i++) { for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i]; struct folio *folio = fbatch.folios[i];
u64 pg_start = folio->index << PAGE_SECTORS_SHIFT; u64 folio_start = folio->index << PAGE_SECTORS_SHIFT;
u64 pg_end = (folio->index + 1) << PAGE_SECTORS_SHIFT; u64 folio_end = (folio->index + 1) << PAGE_SECTORS_SHIFT;
unsigned pg_offset = max(start, pg_start) - pg_start; unsigned folio_offset = max(start, folio_start) - folio_start;
unsigned pg_len = min(end, pg_end) - pg_offset - pg_start; unsigned folio_len = min(end, folio_end) - folio_offset - folio_start;
struct bch_folio *s; struct bch_folio *s;
BUG_ON(end <= pg_start); BUG_ON(end <= folio_start);
BUG_ON(pg_offset >= PAGE_SECTORS); BUG_ON(folio_offset >= PAGE_SECTORS);
BUG_ON(pg_offset + pg_len > PAGE_SECTORS); BUG_ON(folio_offset + folio_len > PAGE_SECTORS);
folio_lock(folio); folio_lock(folio);
s = bch2_folio(&folio->page); s = bch2_folio(folio);
if (s) { if (s) {
spin_lock(&s->lock); spin_lock(&s->lock);
for (j = pg_offset; j < pg_offset + pg_len; j++) for (j = folio_offset; j < folio_offset + folio_len; j++)
switch (s->s[j].state) { switch (s->s[j].state) {
case SECTOR_UNALLOCATED: case SECTOR_UNALLOCATED:
s->s[j].state = SECTOR_RESERVED; s->s[j].state = SECTOR_RESERVED;
...@@ -635,11 +653,11 @@ static inline unsigned sectors_to_reserve(struct bch_folio_sector *s, ...@@ -635,11 +653,11 @@ static inline unsigned sectors_to_reserve(struct bch_folio_sector *s,
s->replicas_reserved); s->replicas_reserved);
} }
static int bch2_get_page_disk_reservation(struct bch_fs *c, static int bch2_get_folio_disk_reservation(struct bch_fs *c,
struct bch_inode_info *inode, struct bch_inode_info *inode,
struct page *page, bool check_enospc) struct folio *folio, bool check_enospc)
{ {
struct bch_folio *s = bch2_folio_create(page, 0); struct bch_folio *s = bch2_folio_create(folio, 0);
unsigned nr_replicas = inode_nr_replicas(c, inode); unsigned nr_replicas = inode_nr_replicas(c, inode);
struct disk_reservation disk_res = { 0 }; struct disk_reservation disk_res = { 0 };
unsigned i, disk_res_sectors = 0; unsigned i, disk_res_sectors = 0;
...@@ -669,34 +687,35 @@ static int bch2_get_page_disk_reservation(struct bch_fs *c, ...@@ -669,34 +687,35 @@ static int bch2_get_page_disk_reservation(struct bch_fs *c,
return 0; return 0;
} }
struct bch2_page_reservation { struct bch2_folio_reservation {
struct disk_reservation disk; struct disk_reservation disk;
struct quota_res quota; struct quota_res quota;
}; };
static void bch2_page_reservation_init(struct bch_fs *c, static void bch2_folio_reservation_init(struct bch_fs *c,
struct bch_inode_info *inode, struct bch_inode_info *inode,
struct bch2_page_reservation *res) struct bch2_folio_reservation *res)
{ {
memset(res, 0, sizeof(*res)); memset(res, 0, sizeof(*res));
res->disk.nr_replicas = inode_nr_replicas(c, inode); res->disk.nr_replicas = inode_nr_replicas(c, inode);
} }
static void bch2_page_reservation_put(struct bch_fs *c, static void bch2_folio_reservation_put(struct bch_fs *c,
struct bch_inode_info *inode, struct bch_inode_info *inode,
struct bch2_page_reservation *res) struct bch2_folio_reservation *res)
{ {
bch2_disk_reservation_put(c, &res->disk); bch2_disk_reservation_put(c, &res->disk);
bch2_quota_reservation_put(c, inode, &res->quota); bch2_quota_reservation_put(c, inode, &res->quota);
} }
static int bch2_page_reservation_get(struct bch_fs *c, static int bch2_folio_reservation_get(struct bch_fs *c,
struct bch_inode_info *inode, struct page *page, struct bch_inode_info *inode,
struct bch2_page_reservation *res, struct folio *folio,
struct bch2_folio_reservation *res,
unsigned offset, unsigned len) unsigned offset, unsigned len)
{ {
struct bch_folio *s = bch2_folio_create(page, 0); struct bch_folio *s = bch2_folio_create(folio, 0);
unsigned i, disk_sectors = 0, quota_sectors = 0; unsigned i, disk_sectors = 0, quota_sectors = 0;
int ret; int ret;
...@@ -736,19 +755,19 @@ static int bch2_page_reservation_get(struct bch_fs *c, ...@@ -736,19 +755,19 @@ static int bch2_page_reservation_get(struct bch_fs *c,
return 0; return 0;
} }
static void bch2_clear_page_bits(struct page *page) static void bch2_clear_folio_bits(struct folio *folio)
{ {
struct bch_inode_info *inode = to_bch_ei(page->mapping->host); struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct bch_folio *s = bch2_folio(page); struct bch_folio *s = bch2_folio(folio);
struct disk_reservation disk_res = { 0 }; struct disk_reservation disk_res = { 0 };
int i, dirty_sectors = 0; int i, dirty_sectors = 0;
if (!s) if (!s)
return; return;
EBUG_ON(!PageLocked(page)); EBUG_ON(!folio_test_locked(folio));
EBUG_ON(PageWriteback(page)); EBUG_ON(folio_test_writeback(folio));
for (i = 0; i < ARRAY_SIZE(s->s); i++) { for (i = 0; i < ARRAY_SIZE(s->s); i++) {
disk_res.sectors += s->s[i].replicas_reserved; disk_res.sectors += s->s[i].replicas_reserved;
...@@ -771,18 +790,19 @@ static void bch2_clear_page_bits(struct page *page) ...@@ -771,18 +790,19 @@ static void bch2_clear_page_bits(struct page *page)
i_sectors_acct(c, inode, NULL, dirty_sectors); i_sectors_acct(c, inode, NULL, dirty_sectors);
bch2_folio_release(page); bch2_folio_release(folio);
} }
static void bch2_set_page_dirty(struct bch_fs *c, static void bch2_set_folio_dirty(struct bch_fs *c,
struct bch_inode_info *inode, struct page *page, struct bch_inode_info *inode,
struct bch2_page_reservation *res, struct folio *folio,
struct bch2_folio_reservation *res,
unsigned offset, unsigned len) unsigned offset, unsigned len)
{ {
struct bch_folio *s = bch2_folio(page); struct bch_folio *s = bch2_folio(folio);
unsigned i, dirty_sectors = 0; unsigned i, dirty_sectors = 0;
WARN_ON((u64) page_offset(page) + offset + len > WARN_ON((u64) folio_pos(folio) + offset + len >
round_up((u64) i_size_read(&inode->v), block_bytes(c))); round_up((u64) i_size_read(&inode->v), block_bytes(c)));
spin_lock(&s->lock); spin_lock(&s->lock);
...@@ -819,8 +839,8 @@ static void bch2_set_page_dirty(struct bch_fs *c, ...@@ -819,8 +839,8 @@ static void bch2_set_page_dirty(struct bch_fs *c,
i_sectors_acct(c, inode, &res->quota, dirty_sectors); i_sectors_acct(c, inode, &res->quota, dirty_sectors);
if (!PageDirty(page)) if (!folio_test_dirty(folio))
filemap_dirty_folio(inode->v.i_mapping, page_folio(page)); filemap_dirty_folio(inode->v.i_mapping, folio);
} }
vm_fault_t bch2_page_fault(struct vm_fault *vmf) vm_fault_t bch2_page_fault(struct vm_fault *vmf)
...@@ -863,17 +883,17 @@ vm_fault_t bch2_page_fault(struct vm_fault *vmf) ...@@ -863,17 +883,17 @@ vm_fault_t bch2_page_fault(struct vm_fault *vmf)
vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf) vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
{ {
struct page *page = vmf->page; struct folio *folio = page_folio(vmf->page);
struct file *file = vmf->vma->vm_file; struct file *file = vmf->vma->vm_file;
struct bch_inode_info *inode = file_bch_inode(file); struct bch_inode_info *inode = file_bch_inode(file);
struct address_space *mapping = file->f_mapping; struct address_space *mapping = file->f_mapping;
struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct bch2_page_reservation res; struct bch2_folio_reservation res;
unsigned len; unsigned len;
loff_t isize; loff_t isize;
int ret; int ret;
bch2_page_reservation_init(c, inode, &res); bch2_folio_reservation_init(c, inode, &res);
sb_start_pagefault(inode->v.i_sb); sb_start_pagefault(inode->v.i_sb);
file_update_time(file); file_update_time(file);
...@@ -886,35 +906,35 @@ vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf) ...@@ -886,35 +906,35 @@ vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
*/ */
bch2_pagecache_add_get(inode); bch2_pagecache_add_get(inode);
lock_page(page); folio_lock(folio);
isize = i_size_read(&inode->v); isize = i_size_read(&inode->v);
if (page->mapping != mapping || page_offset(page) >= isize) { if (folio->mapping != mapping || folio_pos(folio) >= isize) {
unlock_page(page); folio_unlock(folio);
ret = VM_FAULT_NOPAGE; ret = VM_FAULT_NOPAGE;
goto out; goto out;
} }
len = min_t(loff_t, PAGE_SIZE, isize - page_offset(page)); len = min_t(loff_t, PAGE_SIZE, isize - folio_pos(folio));
if (!bch2_folio_create(page, __GFP_NOFAIL)->uptodate) { if (!bch2_folio_create(folio, __GFP_NOFAIL)->uptodate) {
if (bch2_folio_set(c, inode_inum(inode), &page, 1)) { if (bch2_folio_set(c, inode_inum(inode), &folio, 1)) {
unlock_page(page); folio_unlock(folio);
ret = VM_FAULT_SIGBUS; ret = VM_FAULT_SIGBUS;
goto out; goto out;
} }
} }
if (bch2_page_reservation_get(c, inode, page, &res, 0, len)) { if (bch2_folio_reservation_get(c, inode, folio, &res, 0, len)) {
unlock_page(page); folio_unlock(folio);
ret = VM_FAULT_SIGBUS; ret = VM_FAULT_SIGBUS;
goto out; goto out;
} }
bch2_set_page_dirty(c, inode, page, &res, 0, len); bch2_set_folio_dirty(c, inode, folio, &res, 0, len);
bch2_page_reservation_put(c, inode, &res); bch2_folio_reservation_put(c, inode, &res);
wait_for_stable_page(page); folio_wait_stable(folio);
ret = VM_FAULT_LOCKED; ret = VM_FAULT_LOCKED;
out: out:
bch2_pagecache_add_put(inode); bch2_pagecache_add_put(inode);
...@@ -928,7 +948,7 @@ void bch2_invalidate_folio(struct folio *folio, size_t offset, size_t length) ...@@ -928,7 +948,7 @@ void bch2_invalidate_folio(struct folio *folio, size_t offset, size_t length)
if (offset || length < folio_size(folio)) if (offset || length < folio_size(folio))
return; return;
bch2_clear_page_bits(&folio->page); bch2_clear_folio_bits(folio);
} }
bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask) bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask)
...@@ -936,7 +956,7 @@ bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask) ...@@ -936,7 +956,7 @@ bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask)
if (folio_test_dirty(folio) || folio_test_writeback(folio)) if (folio_test_dirty(folio) || folio_test_writeback(folio))
return false; return false;
bch2_clear_page_bits(&folio->page); bch2_clear_folio_bits(folio);
return true; return true;
} }
...@@ -944,19 +964,16 @@ bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask) ...@@ -944,19 +964,16 @@ bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask)
static void bch2_readpages_end_io(struct bio *bio) static void bch2_readpages_end_io(struct bio *bio)
{ {
struct bvec_iter_all iter; struct folio_iter fi;
struct bio_vec *bv;
bio_for_each_segment_all(bv, bio, iter) {
struct page *page = bv->bv_page;
bio_for_each_folio_all(fi, bio) {
if (!bio->bi_status) { if (!bio->bi_status) {
SetPageUptodate(page); folio_mark_uptodate(fi.folio);
} else { } else {
ClearPageUptodate(page); folio_clear_uptodate(fi.folio);
SetPageError(page); folio_set_error(fi.folio);
} }
unlock_page(page); folio_unlock(fi.folio);
} }
bio_put(bio); bio_put(bio);
...@@ -987,21 +1004,21 @@ static int readpages_iter_init(struct readpages_iter *iter, ...@@ -987,21 +1004,21 @@ static int readpages_iter_init(struct readpages_iter *iter,
nr_pages = __readahead_batch(ractl, iter->pages, nr_pages); nr_pages = __readahead_batch(ractl, iter->pages, nr_pages);
for (i = 0; i < nr_pages; i++) { for (i = 0; i < nr_pages; i++) {
__bch2_folio_create(iter->pages[i], __GFP_NOFAIL); __bch2_folio_create(page_folio(iter->pages[i]), __GFP_NOFAIL);
put_page(iter->pages[i]); put_page(iter->pages[i]);
} }
return 0; return 0;
} }
static inline struct page *readpage_iter_next(struct readpages_iter *iter) static inline struct folio *readpage_iter_next(struct readpages_iter *iter)
{ {
if (iter->idx >= iter->nr_pages) if (iter->idx >= iter->nr_pages)
return NULL; return NULL;
EBUG_ON(iter->pages[iter->idx]->index != iter->offset + iter->idx); EBUG_ON(iter->pages[iter->idx]->index != iter->offset + iter->idx);
return iter->pages[iter->idx]; return page_folio(iter->pages[iter->idx]);
} }
static bool extent_partial_reads_expensive(struct bkey_s_c k) static bool extent_partial_reads_expensive(struct bkey_s_c k)
...@@ -1023,12 +1040,12 @@ static void readpage_bio_extend(struct readpages_iter *iter, ...@@ -1023,12 +1040,12 @@ static void readpage_bio_extend(struct readpages_iter *iter,
{ {
while (bio_sectors(bio) < sectors_this_extent && while (bio_sectors(bio) < sectors_this_extent &&
bio->bi_vcnt < bio->bi_max_vecs) { bio->bi_vcnt < bio->bi_max_vecs) {
pgoff_t page_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT; pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
struct page *page = readpage_iter_next(iter); struct folio *folio = readpage_iter_next(iter);
int ret; int ret;
if (page) { if (folio) {
if (iter->offset + iter->idx != page_offset) if (iter->offset + iter->idx != folio_offset)
break; break;
iter->idx++; iter->idx++;
...@@ -1036,31 +1053,30 @@ static void readpage_bio_extend(struct readpages_iter *iter, ...@@ -1036,31 +1053,30 @@ static void readpage_bio_extend(struct readpages_iter *iter,
if (!get_more) if (!get_more)
break; break;
page = xa_load(&iter->mapping->i_pages, page_offset); folio = xa_load(&iter->mapping->i_pages, folio_offset);
if (page && !xa_is_value(page)) if (folio && !xa_is_value(folio))
break; break;
page = __page_cache_alloc(readahead_gfp_mask(iter->mapping)); folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);
if (!page) if (!folio)
break; break;
if (!__bch2_folio_create(page, 0)) { if (!__bch2_folio_create(folio, 0)) {
put_page(page); folio_put(folio);
break; break;
} }
ret = add_to_page_cache_lru(page, iter->mapping, ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_NOFS);
page_offset, GFP_NOFS);
if (ret) { if (ret) {
__bch2_folio_release(page); __bch2_folio_release(folio);
put_page(page); folio_put(folio);
break; break;
} }
put_page(page); folio_put(folio);
} }
BUG_ON(!bio_add_page(bio, page, PAGE_SIZE, 0)); BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));
} }
} }
...@@ -1178,7 +1194,7 @@ void bch2_readahead(struct readahead_control *ractl) ...@@ -1178,7 +1194,7 @@ void bch2_readahead(struct readahead_control *ractl)
struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct bch_io_opts opts; struct bch_io_opts opts;
struct btree_trans trans; struct btree_trans trans;
struct page *page; struct folio *folio;
struct readpages_iter readpages_iter; struct readpages_iter readpages_iter;
int ret; int ret;
...@@ -1191,7 +1207,7 @@ void bch2_readahead(struct readahead_control *ractl) ...@@ -1191,7 +1207,7 @@ void bch2_readahead(struct readahead_control *ractl)
bch2_pagecache_add_get(inode); bch2_pagecache_add_get(inode);
while ((page = readpage_iter_next(&readpages_iter))) { while ((folio = readpage_iter_next(&readpages_iter))) {
pgoff_t index = readpages_iter.offset + readpages_iter.idx; pgoff_t index = readpages_iter.offset + readpages_iter.idx;
unsigned n = min_t(unsigned, unsigned n = min_t(unsigned,
readpages_iter.nr_pages - readpages_iter.nr_pages -
...@@ -1206,7 +1222,7 @@ void bch2_readahead(struct readahead_control *ractl) ...@@ -1206,7 +1222,7 @@ void bch2_readahead(struct readahead_control *ractl)
rbio->bio.bi_iter.bi_sector = (sector_t) index << PAGE_SECTORS_SHIFT; rbio->bio.bi_iter.bi_sector = (sector_t) index << PAGE_SECTORS_SHIFT;
rbio->bio.bi_end_io = bch2_readpages_end_io; rbio->bio.bi_end_io = bch2_readpages_end_io;
BUG_ON(!bio_add_page(&rbio->bio, page, PAGE_SIZE, 0)); BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
bchfs_read(&trans, rbio, inode_inum(inode), bchfs_read(&trans, rbio, inode_inum(inode),
&readpages_iter); &readpages_iter);
...@@ -1218,30 +1234,29 @@ void bch2_readahead(struct readahead_control *ractl) ...@@ -1218,30 +1234,29 @@ void bch2_readahead(struct readahead_control *ractl)
kfree(readpages_iter.pages); kfree(readpages_iter.pages);
} }
static void __bchfs_readpage(struct bch_fs *c, struct bch_read_bio *rbio, static void __bchfs_readfolio(struct bch_fs *c, struct bch_read_bio *rbio,
subvol_inum inum, struct page *page) subvol_inum inum, struct folio *folio)
{ {
struct btree_trans trans; struct btree_trans trans;
bch2_folio_create(page, __GFP_NOFAIL); bch2_folio_create(folio, __GFP_NOFAIL);
rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC; rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;
rbio->bio.bi_iter.bi_sector = rbio->bio.bi_iter.bi_sector = folio_sector(folio);
(sector_t) page->index << PAGE_SECTORS_SHIFT; BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
BUG_ON(!bio_add_page(&rbio->bio, page, PAGE_SIZE, 0));
bch2_trans_init(&trans, c, 0, 0); bch2_trans_init(&trans, c, 0, 0);
bchfs_read(&trans, rbio, inum, NULL); bchfs_read(&trans, rbio, inum, NULL);
bch2_trans_exit(&trans); bch2_trans_exit(&trans);
} }
static void bch2_read_single_page_end_io(struct bio *bio) static void bch2_read_single_folio_end_io(struct bio *bio)
{ {
complete(bio->bi_private); complete(bio->bi_private);
} }
static int bch2_read_single_page(struct page *page, static int bch2_read_single_folio(struct folio *folio,
struct address_space *mapping) struct address_space *mapping)
{ {
struct bch_inode_info *inode = to_bch_ei(mapping->host); struct bch_inode_info *inode = to_bch_ei(mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bch_fs *c = inode->v.i_sb->s_fs_info;
...@@ -1255,9 +1270,9 @@ static int bch2_read_single_page(struct page *page, ...@@ -1255,9 +1270,9 @@ static int bch2_read_single_page(struct page *page,
rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_NOFS, &c->bio_read), rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_NOFS, &c->bio_read),
opts); opts);
rbio->bio.bi_private = &done; rbio->bio.bi_private = &done;
rbio->bio.bi_end_io = bch2_read_single_page_end_io; rbio->bio.bi_end_io = bch2_read_single_folio_end_io;
__bchfs_readpage(c, rbio, inode_inum(inode), page); __bchfs_readfolio(c, rbio, inode_inum(inode), folio);
wait_for_completion(&done); wait_for_completion(&done);
ret = blk_status_to_errno(rbio->bio.bi_status); ret = blk_status_to_errno(rbio->bio.bi_status);
...@@ -1266,16 +1281,15 @@ static int bch2_read_single_page(struct page *page, ...@@ -1266,16 +1281,15 @@ static int bch2_read_single_page(struct page *page,
if (ret < 0) if (ret < 0)
return ret; return ret;
SetPageUptodate(page); folio_mark_uptodate(folio);
return 0; return 0;
} }
int bch2_read_folio(struct file *file, struct folio *folio) int bch2_read_folio(struct file *file, struct folio *folio)
{ {
struct page *page = &folio->page;
int ret; int ret;
ret = bch2_read_single_page(page, page->mapping); ret = bch2_read_single_folio(folio, folio->mapping);
folio_unlock(folio); folio_unlock(folio);
return bch2_err_class(ret); return bch2_err_class(ret);
} }
...@@ -1315,7 +1329,7 @@ static void bch2_writepage_io_done(struct bch_write_op *op) ...@@ -1315,7 +1329,7 @@ static void bch2_writepage_io_done(struct bch_write_op *op)
SetPageError(bvec->bv_page); SetPageError(bvec->bv_page);
mapping_set_error(bvec->bv_page->mapping, -EIO); mapping_set_error(bvec->bv_page->mapping, -EIO);
s = __bch2_folio(bvec->bv_page); s = __bch2_folio(page_folio(bvec->bv_page));
spin_lock(&s->lock); spin_lock(&s->lock);
for (i = 0; i < PAGE_SECTORS; i++) for (i = 0; i < PAGE_SECTORS; i++)
s->s[i].nr_replicas = 0; s->s[i].nr_replicas = 0;
...@@ -1327,7 +1341,7 @@ static void bch2_writepage_io_done(struct bch_write_op *op) ...@@ -1327,7 +1341,7 @@ static void bch2_writepage_io_done(struct bch_write_op *op)
bio_for_each_segment_all(bvec, bio, iter) { bio_for_each_segment_all(bvec, bio, iter) {
struct bch_folio *s; struct bch_folio *s;
s = __bch2_folio(bvec->bv_page); s = __bch2_folio(page_folio(bvec->bv_page));
spin_lock(&s->lock); spin_lock(&s->lock);
for (i = 0; i < PAGE_SECTORS; i++) for (i = 0; i < PAGE_SECTORS; i++)
s->s[i].nr_replicas = 0; s->s[i].nr_replicas = 0;
...@@ -1355,10 +1369,11 @@ static void bch2_writepage_io_done(struct bch_write_op *op) ...@@ -1355,10 +1369,11 @@ static void bch2_writepage_io_done(struct bch_write_op *op)
i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta); i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
bio_for_each_segment_all(bvec, bio, iter) { bio_for_each_segment_all(bvec, bio, iter) {
struct bch_folio *s = __bch2_folio(bvec->bv_page); struct folio *folio = page_folio(bvec->bv_page);
struct bch_folio *s = __bch2_folio(folio);
if (atomic_dec_and_test(&s->write_count)) if (atomic_dec_and_test(&s->write_count))
end_page_writeback(bvec->bv_page); folio_end_writeback(folio);
} }
bio_put(&io->op.wbio.bio); bio_put(&io->op.wbio.bio);
...@@ -1410,44 +1425,44 @@ static int __bch2_writepage(struct folio *folio, ...@@ -1410,44 +1425,44 @@ static int __bch2_writepage(struct folio *folio,
struct writeback_control *wbc, struct writeback_control *wbc,
void *data) void *data)
{ {
struct page *page = &folio->page; struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
struct bch_inode_info *inode = to_bch_ei(page->mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct bch_writepage_state *w = data; struct bch_writepage_state *w = data;
struct bch_folio *s, orig; struct bch_folio *s, orig;
unsigned i, offset, nr_replicas_this_write = U32_MAX; unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;
loff_t i_size = i_size_read(&inode->v); loff_t i_size = i_size_read(&inode->v);
pgoff_t end_index = i_size >> PAGE_SHIFT; pgoff_t end_index = i_size >> PAGE_SHIFT;
int ret; int ret;
EBUG_ON(!PageUptodate(page)); EBUG_ON(!folio_test_uptodate(folio));
/* Is the page fully inside i_size? */ /* Is the folio fully inside i_size? */
if (page->index < end_index) if (folio->index < end_index)
goto do_io; goto do_io;
/* Is the page fully outside i_size? (truncate in progress) */ /* Is the folio fully outside i_size? (truncate in progress) */
offset = i_size & (PAGE_SIZE - 1); offset = i_size & (PAGE_SIZE - 1);
if (page->index > end_index || !offset) { if (folio->index > end_index || !offset) {
unlock_page(page); folio_unlock(folio);
return 0; return 0;
} }
/* /*
* The page straddles i_size. It must be zeroed out on each and every * The folio straddles i_size. It must be zeroed out on each and every
* writepage invocation because it may be mmapped. "A file is mapped * writepage invocation because it may be mmapped. "A file is mapped
* in multiples of the page size. For a file that is not a multiple of * in multiples of the folio size. For a file that is not a multiple of
* the page size, the remaining memory is zeroed when mapped, and * the folio size, the remaining memory is zeroed when mapped, and
* writes to that region are not written out to the file." * writes to that region are not written out to the file."
*/ */
zero_user_segment(page, offset, PAGE_SIZE); folio_zero_segment(folio, offset, folio_size(folio));
do_io: do_io:
s = bch2_folio_create(page, __GFP_NOFAIL); f_sectors = folio_sectors(folio);
s = bch2_folio_create(folio, __GFP_NOFAIL);
/* /*
* Things get really hairy with errors during writeback: * Things get really hairy with errors during writeback:
*/ */
ret = bch2_get_page_disk_reservation(c, inode, page, false); ret = bch2_get_folio_disk_reservation(c, inode, folio, false);
BUG_ON(ret); BUG_ON(ret);
/* Before unlocking the page, get copy of reservations: */ /* Before unlocking the page, get copy of reservations: */
...@@ -1455,7 +1470,7 @@ static int __bch2_writepage(struct folio *folio, ...@@ -1455,7 +1470,7 @@ static int __bch2_writepage(struct folio *folio,
orig = *s; orig = *s;
spin_unlock(&s->lock); spin_unlock(&s->lock);
for (i = 0; i < PAGE_SECTORS; i++) { for (i = 0; i < f_sectors; i++) {
if (s->s[i].state < SECTOR_DIRTY) if (s->s[i].state < SECTOR_DIRTY)
continue; continue;
...@@ -1465,7 +1480,7 @@ static int __bch2_writepage(struct folio *folio, ...@@ -1465,7 +1480,7 @@ static int __bch2_writepage(struct folio *folio,
s->s[i].replicas_reserved); s->s[i].replicas_reserved);
} }
for (i = 0; i < PAGE_SECTORS; i++) { for (i = 0; i < f_sectors; i++) {
if (s->s[i].state < SECTOR_DIRTY) if (s->s[i].state < SECTOR_DIRTY)
continue; continue;
...@@ -1479,24 +1494,24 @@ static int __bch2_writepage(struct folio *folio, ...@@ -1479,24 +1494,24 @@ static int __bch2_writepage(struct folio *folio,
BUG_ON(atomic_read(&s->write_count)); BUG_ON(atomic_read(&s->write_count));
atomic_set(&s->write_count, 1); atomic_set(&s->write_count, 1);
BUG_ON(PageWriteback(page)); BUG_ON(folio_test_writeback(folio));
set_page_writeback(page); folio_start_writeback(folio);
unlock_page(page); folio_unlock(folio);
offset = 0; offset = 0;
while (1) { while (1) {
unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0; unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
u64 sector; u64 sector;
while (offset < PAGE_SECTORS && while (offset < f_sectors &&
orig.s[offset].state < SECTOR_DIRTY) orig.s[offset].state < SECTOR_DIRTY)
offset++; offset++;
if (offset == PAGE_SECTORS) if (offset == f_sectors)
break; break;
while (offset + sectors < PAGE_SECTORS && while (offset + sectors < f_sectors &&
orig.s[offset + sectors].state >= SECTOR_DIRTY) { orig.s[offset + sectors].state >= SECTOR_DIRTY) {
reserved_sectors += orig.s[offset + sectors].replicas_reserved; reserved_sectors += orig.s[offset + sectors].replicas_reserved;
dirty_sectors += orig.s[offset + sectors].state == SECTOR_DIRTY; dirty_sectors += orig.s[offset + sectors].state == SECTOR_DIRTY;
...@@ -1504,7 +1519,7 @@ static int __bch2_writepage(struct folio *folio, ...@@ -1504,7 +1519,7 @@ static int __bch2_writepage(struct folio *folio,
} }
BUG_ON(!sectors); BUG_ON(!sectors);
sector = ((u64) page->index << PAGE_SECTORS_SHIFT) + offset; sector = folio_sector(folio) + offset;
if (w->io && if (w->io &&
(w->io->op.res.nr_replicas != nr_replicas_this_write || (w->io->op.res.nr_replicas != nr_replicas_this_write ||
...@@ -1521,7 +1536,7 @@ static int __bch2_writepage(struct folio *folio, ...@@ -1521,7 +1536,7 @@ static int __bch2_writepage(struct folio *folio,
atomic_inc(&s->write_count); atomic_inc(&s->write_count);
BUG_ON(inode != w->io->inode); BUG_ON(inode != w->io->inode);
BUG_ON(!bio_add_page(&w->io->op.wbio.bio, page, BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,
sectors << 9, offset << 9)); sectors << 9, offset << 9));
/* Check for writing past i_size: */ /* Check for writing past i_size: */
...@@ -1541,7 +1556,7 @@ static int __bch2_writepage(struct folio *folio, ...@@ -1541,7 +1556,7 @@ static int __bch2_writepage(struct folio *folio,
} }
if (atomic_dec_and_test(&s->write_count)) if (atomic_dec_and_test(&s->write_count))
end_page_writeback(page); folio_end_writeback(folio);
return 0; return 0;
} }
...@@ -1570,61 +1585,63 @@ int bch2_write_begin(struct file *file, struct address_space *mapping, ...@@ -1570,61 +1585,63 @@ int bch2_write_begin(struct file *file, struct address_space *mapping,
{ {
struct bch_inode_info *inode = to_bch_ei(mapping->host); struct bch_inode_info *inode = to_bch_ei(mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct bch2_page_reservation *res; struct bch2_folio_reservation *res;
pgoff_t index = pos >> PAGE_SHIFT; pgoff_t index = pos >> PAGE_SHIFT;
unsigned offset = pos & (PAGE_SIZE - 1); unsigned offset = pos & (PAGE_SIZE - 1);
struct page *page; struct folio *folio;
int ret = -ENOMEM; int ret = -ENOMEM;
res = kmalloc(sizeof(*res), GFP_KERNEL); res = kmalloc(sizeof(*res), GFP_KERNEL);
if (!res) if (!res)
return -ENOMEM; return -ENOMEM;
bch2_page_reservation_init(c, inode, res); bch2_folio_reservation_init(c, inode, res);
*fsdata = res; *fsdata = res;
bch2_pagecache_add_get(inode); bch2_pagecache_add_get(inode);
page = grab_cache_page_write_begin(mapping, index); folio = __filemap_get_folio(mapping, index,
if (!page) FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
mapping_gfp_mask(mapping));
if (!folio)
goto err_unlock; goto err_unlock;
if (PageUptodate(page)) if (folio_test_uptodate(folio))
goto out; goto out;
/* If we're writing entire page, don't need to read it in first: */ /* If we're writing entire folio, don't need to read it in first: */
if (len == PAGE_SIZE) if (len == folio_size(folio))
goto out; goto out;
if (!offset && pos + len >= inode->v.i_size) { if (!offset && pos + len >= inode->v.i_size) {
zero_user_segment(page, len, PAGE_SIZE); folio_zero_segment(folio, len, folio_size(folio));
flush_dcache_page(page); flush_dcache_folio(folio);
goto out; goto out;
} }
if (index > inode->v.i_size >> PAGE_SHIFT) { if (index > inode->v.i_size >> PAGE_SHIFT) {
zero_user_segments(page, 0, offset, offset + len, PAGE_SIZE); folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));
flush_dcache_page(page); flush_dcache_folio(folio);
goto out; goto out;
} }
readpage: readpage:
ret = bch2_read_single_page(page, mapping); ret = bch2_read_single_folio(folio, mapping);
if (ret) if (ret)
goto err; goto err;
out: out:
if (!bch2_folio_create(page, __GFP_NOFAIL)->uptodate) { if (!bch2_folio_create(folio, __GFP_NOFAIL)->uptodate) {
ret = bch2_folio_set(c, inode_inum(inode), &page, 1); ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
if (ret) if (ret)
goto err; goto err;
} }
ret = bch2_page_reservation_get(c, inode, page, res, offset, len); ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);
if (ret) { if (ret) {
if (!PageUptodate(page)) { if (!folio_test_uptodate(folio)) {
/* /*
* If the page hasn't been read in, we won't know if we * If the folio hasn't been read in, we won't know if we
* actually need a reservation - we don't actually need * actually need a reservation - we don't actually need
* to read here, we just need to check if the page is * to read here, we just need to check if the folio is
* fully backed by uncompressed data: * fully backed by uncompressed data:
*/ */
goto readpage; goto readpage;
...@@ -1633,11 +1650,11 @@ int bch2_write_begin(struct file *file, struct address_space *mapping, ...@@ -1633,11 +1650,11 @@ int bch2_write_begin(struct file *file, struct address_space *mapping,
goto err; goto err;
} }
*pagep = page; *pagep = &folio->page;
return 0; return 0;
err: err:
unlock_page(page); folio_unlock(folio);
put_page(page); folio_put(folio);
*pagep = NULL; *pagep = NULL;
err_unlock: err_unlock:
bch2_pagecache_add_put(inode); bch2_pagecache_add_put(inode);
...@@ -1652,19 +1669,20 @@ int bch2_write_end(struct file *file, struct address_space *mapping, ...@@ -1652,19 +1669,20 @@ int bch2_write_end(struct file *file, struct address_space *mapping,
{ {
struct bch_inode_info *inode = to_bch_ei(mapping->host); struct bch_inode_info *inode = to_bch_ei(mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct bch2_page_reservation *res = fsdata; struct bch2_folio_reservation *res = fsdata;
struct folio *folio = page_folio(page);
unsigned offset = pos & (PAGE_SIZE - 1); unsigned offset = pos & (PAGE_SIZE - 1);
lockdep_assert_held(&inode->v.i_rwsem); lockdep_assert_held(&inode->v.i_rwsem);
if (unlikely(copied < len && !PageUptodate(page))) { if (unlikely(copied < len && !folio_test_uptodate(folio))) {
/* /*
* The page needs to be read in, but that would destroy * The folio needs to be read in, but that would destroy
* our partial write - simplest thing is to just force * our partial write - simplest thing is to just force
* userspace to redo the write: * userspace to redo the write:
*/ */
zero_user(page, 0, PAGE_SIZE); folio_zero_range(folio, 0, folio_size(folio));
flush_dcache_page(page); flush_dcache_folio(folio);
copied = 0; copied = 0;
} }
...@@ -1674,19 +1692,19 @@ int bch2_write_end(struct file *file, struct address_space *mapping, ...@@ -1674,19 +1692,19 @@ int bch2_write_end(struct file *file, struct address_space *mapping,
spin_unlock(&inode->v.i_lock); spin_unlock(&inode->v.i_lock);
if (copied) { if (copied) {
if (!PageUptodate(page)) if (!folio_test_uptodate(folio))
SetPageUptodate(page); folio_mark_uptodate(folio);
bch2_set_page_dirty(c, inode, page, res, offset, copied); bch2_set_folio_dirty(c, inode, folio, res, offset, copied);
inode->ei_last_dirtied = (unsigned long) current; inode->ei_last_dirtied = (unsigned long) current;
} }
unlock_page(page); folio_unlock(folio);
put_page(page); folio_put(folio);
bch2_pagecache_add_put(inode); bch2_pagecache_add_put(inode);
bch2_page_reservation_put(c, inode, res); bch2_folio_reservation_put(c, inode, res);
kfree(res); kfree(res);
return copied; return copied;
...@@ -1700,46 +1718,49 @@ static int __bch2_buffered_write(struct bch_inode_info *inode, ...@@ -1700,46 +1718,49 @@ static int __bch2_buffered_write(struct bch_inode_info *inode,
loff_t pos, unsigned len) loff_t pos, unsigned len)
{ {
struct bch_fs *c = inode->v.i_sb->s_fs_info; struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct page *pages[WRITE_BATCH_PAGES]; struct folio *folios[WRITE_BATCH_PAGES];
struct bch2_page_reservation res; struct bch2_folio_reservation res;
unsigned long index = pos >> PAGE_SHIFT; unsigned long index = pos >> PAGE_SHIFT;
unsigned offset = pos & (PAGE_SIZE - 1); unsigned offset = pos & (PAGE_SIZE - 1);
unsigned nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); unsigned nr_folios = DIV_ROUND_UP(offset + len, PAGE_SIZE);
unsigned i, reserved = 0, set_dirty = 0; unsigned i, reserved = 0, set_dirty = 0;
unsigned copied = 0, nr_pages_copied = 0; unsigned copied = 0, nr_folios_copied = 0;
int ret = 0; int ret = 0;
BUG_ON(!len); BUG_ON(!len);
BUG_ON(nr_pages > ARRAY_SIZE(pages)); BUG_ON(nr_folios > ARRAY_SIZE(folios));
bch2_page_reservation_init(c, inode, &res); bch2_folio_reservation_init(c, inode, &res);
for (i = 0; i < nr_pages; i++) { for (i = 0; i < nr_folios; i++) {
pages[i] = grab_cache_page_write_begin(mapping, index + i); folios[i] = __filemap_get_folio(mapping, index + i,
if (!pages[i]) { FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
nr_pages = i; mapping_gfp_mask(mapping));
if (!folios[i]) {
nr_folios = i;
if (!i) { if (!i) {
ret = -ENOMEM; ret = -ENOMEM;
goto out; goto out;
} }
len = min_t(unsigned, len, len = min_t(unsigned, len,
nr_pages * PAGE_SIZE - offset); nr_folios * PAGE_SIZE - offset);
break; break;
} }
} }
if (offset && !PageUptodate(pages[0])) { if (offset && !folio_test_uptodate(folios[0])) {
ret = bch2_read_single_page(pages[0], mapping); ret = bch2_read_single_folio(folios[0], mapping);
if (ret) if (ret)
goto out; goto out;
} }
if ((pos + len) & (PAGE_SIZE - 1) && if ((pos + len) & (PAGE_SIZE - 1) &&
!PageUptodate(pages[nr_pages - 1])) { !folio_test_uptodate(folios[nr_folios - 1])) {
if ((index + nr_pages - 1) << PAGE_SHIFT >= inode->v.i_size) { if ((index + nr_folios - 1) << PAGE_SHIFT >= inode->v.i_size) {
zero_user(pages[nr_pages - 1], 0, PAGE_SIZE); folio_zero_range(folios[nr_folios - 1], 0,
folio_size(folios[nr_folios - 1]));
} else { } else {
ret = bch2_read_single_page(pages[nr_pages - 1], mapping); ret = bch2_read_single_folio(folios[nr_folios - 1], mapping);
if (ret) if (ret)
goto out; goto out;
} }
...@@ -1747,14 +1768,14 @@ static int __bch2_buffered_write(struct bch_inode_info *inode, ...@@ -1747,14 +1768,14 @@ static int __bch2_buffered_write(struct bch_inode_info *inode,
while (reserved < len) { while (reserved < len) {
unsigned i = (offset + reserved) >> PAGE_SHIFT; unsigned i = (offset + reserved) >> PAGE_SHIFT;
struct page *page = pages[i]; struct folio *folio = folios[i];
unsigned pg_offset = (offset + reserved) & (PAGE_SIZE - 1); unsigned folio_offset = (offset + reserved) & (PAGE_SIZE - 1);
unsigned pg_len = min_t(unsigned, len - reserved, unsigned folio_len = min_t(unsigned, len - reserved,
PAGE_SIZE - pg_offset); PAGE_SIZE - folio_offset);
if (!bch2_folio_create(page, __GFP_NOFAIL)->uptodate) { if (!bch2_folio_create(folio, __GFP_NOFAIL)->uptodate) {
ret = bch2_folio_set(c, inode_inum(inode), ret = bch2_folio_set(c, inode_inum(inode),
pages + i, nr_pages - i); folios + i, nr_folios - i);
if (ret) if (ret)
goto out; goto out;
} }
...@@ -1767,43 +1788,43 @@ static int __bch2_buffered_write(struct bch_inode_info *inode, ...@@ -1767,43 +1788,43 @@ static int __bch2_buffered_write(struct bch_inode_info *inode,
* we aren't completely out of disk space - we don't do that * we aren't completely out of disk space - we don't do that
* yet: * yet:
*/ */
ret = bch2_page_reservation_get(c, inode, page, &res, ret = bch2_folio_reservation_get(c, inode, folio, &res,
pg_offset, pg_len); folio_offset, folio_len);
if (unlikely(ret)) { if (unlikely(ret)) {
if (!reserved) if (!reserved)
goto out; goto out;
break; break;
} }
reserved += pg_len; reserved += folio_len;
} }
if (mapping_writably_mapped(mapping)) if (mapping_writably_mapped(mapping))
for (i = 0; i < nr_pages; i++) for (i = 0; i < nr_folios; i++)
flush_dcache_page(pages[i]); flush_dcache_folio(folios[i]);
while (copied < reserved) { while (copied < reserved) {
struct page *page = pages[(offset + copied) >> PAGE_SHIFT]; struct folio *folio = folios[(offset + copied) >> PAGE_SHIFT];
unsigned pg_offset = (offset + copied) & (PAGE_SIZE - 1); unsigned folio_offset = (offset + copied) & (PAGE_SIZE - 1);
unsigned pg_len = min_t(unsigned, reserved - copied, unsigned folio_len = min_t(unsigned, reserved - copied,
PAGE_SIZE - pg_offset); PAGE_SIZE - folio_offset);
unsigned pg_copied = copy_page_from_iter_atomic(page, unsigned folio_copied = copy_page_from_iter_atomic(&folio->page,
pg_offset, pg_len, iter); folio_offset, folio_len, iter);
if (!pg_copied) if (!folio_copied)
break; break;
if (!PageUptodate(page) && if (!folio_test_uptodate(folio) &&
pg_copied != PAGE_SIZE && folio_copied != PAGE_SIZE &&
pos + copied + pg_copied < inode->v.i_size) { pos + copied + folio_copied < inode->v.i_size) {
zero_user(page, 0, PAGE_SIZE); folio_zero_range(folio, 0, folio_size(folio));
break; break;
} }
flush_dcache_page(page); flush_dcache_folio(folio);
copied += pg_copied; copied += folio_copied;
if (pg_copied != pg_len) if (folio_copied != folio_len)
break; break;
} }
...@@ -1816,30 +1837,30 @@ static int __bch2_buffered_write(struct bch_inode_info *inode, ...@@ -1816,30 +1837,30 @@ static int __bch2_buffered_write(struct bch_inode_info *inode,
spin_unlock(&inode->v.i_lock); spin_unlock(&inode->v.i_lock);
while (set_dirty < copied) { while (set_dirty < copied) {
struct page *page = pages[(offset + set_dirty) >> PAGE_SHIFT]; struct folio *folio = folios[(offset + set_dirty) >> PAGE_SHIFT];
unsigned pg_offset = (offset + set_dirty) & (PAGE_SIZE - 1); unsigned folio_offset = (offset + set_dirty) & (PAGE_SIZE - 1);
unsigned pg_len = min_t(unsigned, copied - set_dirty, unsigned folio_len = min_t(unsigned, copied - set_dirty,
PAGE_SIZE - pg_offset); PAGE_SIZE - folio_offset);
if (!PageUptodate(page)) if (!folio_test_uptodate(folio))
SetPageUptodate(page); folio_mark_uptodate(folio);
bch2_set_page_dirty(c, inode, page, &res, pg_offset, pg_len); bch2_set_folio_dirty(c, inode, folio, &res, folio_offset, folio_len);
unlock_page(page); folio_unlock(folio);
put_page(page); folio_put(folio);
set_dirty += pg_len; set_dirty += folio_len;
} }
nr_pages_copied = DIV_ROUND_UP(offset + copied, PAGE_SIZE); nr_folios_copied = DIV_ROUND_UP(offset + copied, PAGE_SIZE);
inode->ei_last_dirtied = (unsigned long) current; inode->ei_last_dirtied = (unsigned long) current;
out: out:
for (i = nr_pages_copied; i < nr_pages; i++) { for (i = nr_folios_copied; i < nr_folios; i++) {
unlock_page(pages[i]); folio_unlock(folios[i]);
put_page(pages[i]); folio_put(folios[i]);
} }
bch2_page_reservation_put(c, inode, &res); bch2_folio_reservation_put(c, inode, &res);
return copied ?: ret; return copied ?: ret;
} }
...@@ -2646,7 +2667,7 @@ static int __bch2_truncate_page(struct bch_inode_info *inode, ...@@ -2646,7 +2667,7 @@ static int __bch2_truncate_page(struct bch_inode_info *inode,
unsigned start_offset = start & (PAGE_SIZE - 1); unsigned start_offset = start & (PAGE_SIZE - 1);
unsigned end_offset = ((end - 1) & (PAGE_SIZE - 1)) + 1; unsigned end_offset = ((end - 1) & (PAGE_SIZE - 1)) + 1;
unsigned i; unsigned i;
struct page *page; struct folio *folio;
s64 i_sectors_delta = 0; s64 i_sectors_delta = 0;
int ret = 0; int ret = 0;
...@@ -2659,11 +2680,11 @@ static int __bch2_truncate_page(struct bch_inode_info *inode, ...@@ -2659,11 +2680,11 @@ static int __bch2_truncate_page(struct bch_inode_info *inode,
if (index << PAGE_SHIFT >= inode->v.i_size) if (index << PAGE_SHIFT >= inode->v.i_size)
return 0; return 0;
page = find_lock_page(mapping, index); folio = filemap_lock_folio(mapping, index);
if (!page) { if (!folio) {
/* /*
* XXX: we're doing two index lookups when we end up reading the * XXX: we're doing two index lookups when we end up reading the
* page * folio
*/ */
ret = range_has_data(c, inode->ei_subvol, ret = range_has_data(c, inode->ei_subvol,
POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT)), POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT)),
...@@ -2671,21 +2692,22 @@ static int __bch2_truncate_page(struct bch_inode_info *inode, ...@@ -2671,21 +2692,22 @@ static int __bch2_truncate_page(struct bch_inode_info *inode,
if (ret <= 0) if (ret <= 0)
return ret; return ret;
page = find_or_create_page(mapping, index, GFP_KERNEL); folio = __filemap_get_folio(mapping, index,
if (unlikely(!page)) { FGP_LOCK|FGP_CREAT, GFP_KERNEL);
if (unlikely(!folio)) {
ret = -ENOMEM; ret = -ENOMEM;
goto out; goto out;
} }
} }
s = bch2_folio_create(page, 0); s = bch2_folio_create(folio, 0);
if (!s) { if (!s) {
ret = -ENOMEM; ret = -ENOMEM;
goto unlock; goto unlock;
} }
if (!PageUptodate(page)) { if (!folio_test_uptodate(folio)) {
ret = bch2_read_single_page(page, mapping); ret = bch2_read_single_folio(folio, mapping);
if (ret) if (ret)
goto unlock; goto unlock;
} }
...@@ -2709,33 +2731,33 @@ static int __bch2_truncate_page(struct bch_inode_info *inode, ...@@ -2709,33 +2731,33 @@ static int __bch2_truncate_page(struct bch_inode_info *inode,
i_sectors_acct(c, inode, NULL, i_sectors_delta); i_sectors_acct(c, inode, NULL, i_sectors_delta);
/* /*
* Caller needs to know whether this page will be written out by * Caller needs to know whether this folio will be written out by
* writeback - doing an i_size update if necessary - or whether it will * writeback - doing an i_size update if necessary - or whether it will
* be responsible for the i_size update: * be responsible for the i_size update:
*/ */
ret = s->s[(min_t(u64, inode->v.i_size - (index << PAGE_SHIFT), ret = s->s[(min_t(u64, inode->v.i_size - (index << PAGE_SHIFT),
PAGE_SIZE) - 1) >> 9].state >= SECTOR_DIRTY; PAGE_SIZE) - 1) >> 9].state >= SECTOR_DIRTY;
zero_user_segment(page, start_offset, end_offset); folio_zero_segment(folio, start_offset, end_offset);
/* /*
* Bit of a hack - we don't want truncate to fail due to -ENOSPC. * Bit of a hack - we don't want truncate to fail due to -ENOSPC.
* *
* XXX: because we aren't currently tracking whether the page has actual * XXX: because we aren't currently tracking whether the folio has actual
* data in it (vs. just 0s, or only partially written) this wrong. ick. * data in it (vs. just 0s, or only partially written) this wrong. ick.
*/ */
BUG_ON(bch2_get_page_disk_reservation(c, inode, page, false)); BUG_ON(bch2_get_folio_disk_reservation(c, inode, folio, false));
/* /*
* This removes any writeable userspace mappings; we need to force * This removes any writeable userspace mappings; we need to force
* .page_mkwrite to be called again before any mmapped writes, to * .page_mkwrite to be called again before any mmapped writes, to
* redirty the full page: * redirty the full page:
*/ */
page_mkclean(page); folio_mkclean(folio);
filemap_dirty_folio(mapping, page_folio(page)); filemap_dirty_folio(mapping, folio);
unlock: unlock:
unlock_page(page); folio_unlock(folio);
put_page(page); folio_put(folio);
out: out:
return ret; return ret;
} }
...@@ -3467,7 +3489,7 @@ loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src, ...@@ -3467,7 +3489,7 @@ loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src,
static int folio_data_offset(struct folio *folio, unsigned offset) static int folio_data_offset(struct folio *folio, unsigned offset)
{ {
struct bch_folio *s = bch2_folio(&folio->page); struct bch_folio *s = bch2_folio(folio);
unsigned i; unsigned i;
if (s) if (s)
...@@ -3572,9 +3594,9 @@ static loff_t bch2_seek_data(struct file *file, u64 offset) ...@@ -3572,9 +3594,9 @@ static loff_t bch2_seek_data(struct file *file, u64 offset)
return vfs_setpos(file, next_data, MAX_LFS_FILESIZE); return vfs_setpos(file, next_data, MAX_LFS_FILESIZE);
} }
static int __page_hole_offset(struct page *page, unsigned offset) static int __folio_hole_offset(struct folio *folio, unsigned offset)
{ {
struct bch_folio *s = bch2_folio(page); struct bch_folio *s = bch2_folio(folio);
unsigned i; unsigned i;
if (!s) if (!s)
...@@ -3587,22 +3609,21 @@ static int __page_hole_offset(struct page *page, unsigned offset) ...@@ -3587,22 +3609,21 @@ static int __page_hole_offset(struct page *page, unsigned offset)
return -1; return -1;
} }
static loff_t page_hole_offset(struct address_space *mapping, loff_t offset) static loff_t folio_hole_offset(struct address_space *mapping, loff_t offset)
{ {
pgoff_t index = offset >> PAGE_SHIFT; pgoff_t index = offset >> PAGE_SHIFT;
struct page *page; struct folio *folio;
int pg_offset; int folio_offset;
loff_t ret = -1; loff_t ret = -1;
page = find_lock_page(mapping, index); folio = filemap_lock_folio(mapping, index);
if (!page) if (!folio)
return offset; return offset;
pg_offset = __page_hole_offset(page, offset & (PAGE_SIZE - 1)); folio_offset = __folio_hole_offset(folio, offset & (folio_size(folio) - 1));
if (pg_offset >= 0) if (folio_offset >= 0)
ret = ((loff_t) index << PAGE_SHIFT) + pg_offset; ret = folio_pos(folio) + folio_offset;
folio_unlock(folio);
unlock_page(page);
return ret; return ret;
} }
...@@ -3615,7 +3636,7 @@ static loff_t bch2_seek_pagecache_hole(struct inode *vinode, ...@@ -3615,7 +3636,7 @@ static loff_t bch2_seek_pagecache_hole(struct inode *vinode,
loff_t offset = start_offset, hole; loff_t offset = start_offset, hole;
while (offset < end_offset) { while (offset < end_offset) {
hole = page_hole_offset(mapping, offset); hole = folio_hole_offset(mapping, offset);
if (hole >= 0 && hole <= end_offset) if (hole >= 0 && hole <= end_offset)
return max(start_offset, hole); return max(start_offset, hole);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7