bcachefs: Factor out two_state_shared_lock

We have a unique lock used for controlling adding to the pagecache: the
lock has two states, where both states are shared - the lock may be held
multiple times for either state - but not both states at the same time.

This is exactly what we need for nocow mode locking, so this patch pulls
it out of fs.c into its own file.
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/Makefile

@@ -65,6 +65,7 @@ bcachefs-y		:=	\
 	sysfs.o			\
 	tests.o			\
 	trace.o			\
+	two_state_shared_lock.o	\
 	util.o			\
 	varint.o		\
 	xattr.o

fs/bcachefs/fs-io.c

@@ -751,25 +751,25 @@ vm_fault_t bch2_page_fault(struct vm_fault *vmf)
 	if (fdm > mapping) {
 		struct bch_inode_info *fdm_host = to_bch_ei(fdm->host);
 
-		if (bch2_pagecache_add_tryget(&inode->ei_pagecache_lock))
+		if (bch2_pagecache_add_tryget(inode))
 			goto got_lock;
 
-		bch2_pagecache_block_put(&fdm_host->ei_pagecache_lock);
+		bch2_pagecache_block_put(fdm_host);
 
-		bch2_pagecache_add_get(&inode->ei_pagecache_lock);
-		bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+		bch2_pagecache_add_get(inode);
+		bch2_pagecache_add_put(inode);
 
-		bch2_pagecache_block_get(&fdm_host->ei_pagecache_lock);
+		bch2_pagecache_block_get(fdm_host);
 
 		/* Signal that lock has been dropped: */
 		set_fdm_dropped_locks();
 		return VM_FAULT_SIGBUS;
 	}
 
-	bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_get(inode);
 got_lock:
 	ret = filemap_fault(vmf);
-	bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_put(inode);
 
 	return ret;
 }
@@ -797,7 +797,7 @@ vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
 	 * a write_invalidate_inode_pages_range() that works without dropping
 	 * page lock before invalidating page
 	 */
-	bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_get(inode);
 
 	lock_page(page);
 	isize = i_size_read(&inode->v);
@@ -830,7 +830,7 @@ vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
 	wait_for_stable_page(page);
 	ret = VM_FAULT_LOCKED;
 out:
-	bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_put(inode);
 	sb_end_pagefault(inode->v.i_sb);
 
 	return ret;
@@ -1098,7 +1098,7 @@ void bch2_readahead(struct readahead_control *ractl)
 
 	bch2_trans_init(&trans, c, 0, 0);
 
-	bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_get(inode);
 
 	while ((page = readpage_iter_next(&readpages_iter))) {
 		pgoff_t index = readpages_iter.offset + readpages_iter.idx;
@@ -1121,7 +1121,7 @@ void bch2_readahead(struct readahead_control *ractl)
 			   &readpages_iter);
 	}
 
-	bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_put(inode);
 
 	bch2_trans_exit(&trans);
 	kfree(readpages_iter.pages);
@@ -1483,7 +1483,7 @@ int bch2_write_begin(struct file *file, struct address_space *mapping,
 	bch2_page_reservation_init(c, inode, res);
 	*fsdata = res;
 
-	bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_get(inode);
 
 	page = grab_cache_page_write_begin(mapping, index);
 	if (!page)
@@ -1540,7 +1540,7 @@ int bch2_write_begin(struct file *file, struct address_space *mapping,
 	put_page(page);
 	*pagep = NULL;
 err_unlock:
-	bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_put(inode);
 	kfree(res);
 	*fsdata = NULL;
 	return bch2_err_class(ret);
@@ -1584,7 +1584,7 @@ int bch2_write_end(struct file *file, struct address_space *mapping,
 
 	unlock_page(page);
 	put_page(page);
-	bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_put(inode);
 
 	bch2_page_reservation_put(c, inode, res);
 	kfree(res);
@@ -1753,7 +1753,7 @@ static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
 	ssize_t written = 0;
 	int ret = 0;
 
-	bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_get(inode);
 
 	do {
 		unsigned offset = pos & (PAGE_SIZE - 1);
@@ -1811,7 +1811,7 @@ static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
 		balance_dirty_pages_ratelimited(mapping);
 	} while (iov_iter_count(iter));
 
-	bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_add_put(inode);
 
 	return written ? written : ret;
 }
@@ -1991,9 +1991,9 @@ ssize_t bch2_read_iter(struct kiocb *iocb, struct iov_iter *iter)
 		if (ret >= 0)
 			iocb->ki_pos += ret;
 	} else {
-		bch2_pagecache_add_get(&inode->ei_pagecache_lock);
+		bch2_pagecache_add_get(inode);
 		ret = generic_file_read_iter(iocb, iter);
-		bch2_pagecache_add_put(&inode->ei_pagecache_lock);
+		bch2_pagecache_add_put(inode);
 	}
 out:
 	return bch2_err_class(ret);
@@ -2149,7 +2149,7 @@ static __always_inline long bch2_dio_write_done(struct dio_write *dio)
 			return -EIOCBQUEUED;
 	}
 
-	bch2_pagecache_block_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_block_put(inode);
 	bch2_quota_reservation_put(c, inode, &dio->quota_res);
 
 	if (dio->free_iov)
@@ -2357,7 +2357,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 		goto err;
 
 	inode_dio_begin(&inode->v);
-	bch2_pagecache_block_get(&inode->ei_pagecache_lock);
+	bch2_pagecache_block_get(inode);
 
 	extending = req->ki_pos + iter->count > inode->v.i_size;
 	if (!extending) {
@@ -2403,7 +2403,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 		inode_unlock(&inode->v);
 	return ret;
 err_put_bio:
-	bch2_pagecache_block_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_block_put(inode);
 	bch2_quota_reservation_put(c, inode, &dio->quota_res);
 	bio_put(bio);
 	inode_dio_end(&inode->v);
@@ -2704,7 +2704,7 @@ int bch2_truncate(struct mnt_idmap *idmap,
 	}
 
 	inode_dio_wait(&inode->v);
-	bch2_pagecache_block_get(&inode->ei_pagecache_lock);
+	bch2_pagecache_block_get(inode);
 
 	ret = bch2_inode_find_by_inum(c, inode_inum(inode), &inode_u);
 	if (ret)
@@ -2783,7 +2783,7 @@ int bch2_truncate(struct mnt_idmap *idmap,
 
 	ret = bch2_setattr_nonsize(idmap, inode, iattr);
 err:
-	bch2_pagecache_block_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_block_put(inode);
 	return bch2_err_class(ret);
 }
 
@@ -3195,7 +3195,7 @@ long bch2_fallocate_dispatch(struct file *file, int mode,
 
 	inode_lock(&inode->v);
 	inode_dio_wait(&inode->v);
-	bch2_pagecache_block_get(&inode->ei_pagecache_lock);
+	bch2_pagecache_block_get(inode);
 
 	ret = file_modified(file);
 	if (ret)
@@ -3212,7 +3212,7 @@ long bch2_fallocate_dispatch(struct file *file, int mode,
 	else
 		ret = -EOPNOTSUPP;
 err:
-	bch2_pagecache_block_put(&inode->ei_pagecache_lock);
+	bch2_pagecache_block_put(inode);
 	inode_unlock(&inode->v);
 	percpu_ref_put(&c->writes);
 

fs/bcachefs/fs.c

@@ -43,58 +43,6 @@ static void bch2_vfs_inode_init(struct btree_trans *, subvol_inum,
 				struct bch_inode_unpacked *,
 				struct bch_subvolume *);
 
-static void __pagecache_lock_put(struct pagecache_lock *lock, long i)
-{
-	BUG_ON(atomic_long_read(&lock->v) == 0);
-
-	if (atomic_long_sub_return_release(i, &lock->v) == 0)
-		wake_up_all(&lock->wait);
-}
-
-static bool __pagecache_lock_tryget(struct pagecache_lock *lock, long i)
-{
-	long v = atomic_long_read(&lock->v), old;
-
-	do {
-		old = v;
-
-		if (i > 0 ? v < 0 : v > 0)
-			return false;
-	} while ((v = atomic_long_cmpxchg_acquire(&lock->v,
-					old, old + i)) != old);
-	return true;
-}
-
-static void __pagecache_lock_get(struct pagecache_lock *lock, long i)
-{
-	wait_event(lock->wait, __pagecache_lock_tryget(lock, i));
-}
-
-void bch2_pagecache_add_put(struct pagecache_lock *lock)
-{
-	__pagecache_lock_put(lock, 1);
-}
-
-bool bch2_pagecache_add_tryget(struct pagecache_lock *lock)
-{
-	return __pagecache_lock_tryget(lock, 1);
-}
-
-void bch2_pagecache_add_get(struct pagecache_lock *lock)
-{
-	__pagecache_lock_get(lock, 1);
-}
-
-void bch2_pagecache_block_put(struct pagecache_lock *lock)
-{
-	__pagecache_lock_put(lock, -1);
-}
-
-void bch2_pagecache_block_get(struct pagecache_lock *lock)
-{
-	__pagecache_lock_get(lock, -1);
-}
-
 void bch2_inode_update_after_write(struct btree_trans *trans,
 				   struct bch_inode_info *inode,
 				   struct bch_inode_unpacked *bi,
@@ -1410,7 +1358,7 @@ static struct inode *bch2_alloc_inode(struct super_block *sb)
 
 	inode_init_once(&inode->v);
 	mutex_init(&inode->ei_update_lock);
-	pagecache_lock_init(&inode->ei_pagecache_lock);
+	two_state_lock_init(&inode->ei_pagecache_lock);
 	mutex_init(&inode->ei_quota_lock);
 
 	return &inode->v;

fs/bcachefs/fs.h

@@ -6,31 +6,11 @@
 #include "opts.h"
 #include "str_hash.h"
 #include "quota_types.h"
+#include "two_state_shared_lock.h"
 
 #include <linux/seqlock.h>
 #include <linux/stat.h>
 
-/*
- * Two-state lock - can be taken for add or block - both states are shared,
- * like read side of rwsem, but conflict with other state:
- */
-struct pagecache_lock {
-	atomic_long_t		v;
-	wait_queue_head_t	wait;
-};
-
-static inline void pagecache_lock_init(struct pagecache_lock *lock)
-{
-	atomic_long_set(&lock->v, 0);
-	init_waitqueue_head(&lock->wait);
-}
-
-void bch2_pagecache_add_put(struct pagecache_lock *);
-bool bch2_pagecache_add_tryget(struct pagecache_lock *);
-void bch2_pagecache_add_get(struct pagecache_lock *);
-void bch2_pagecache_block_put(struct pagecache_lock *);
-void bch2_pagecache_block_get(struct pagecache_lock *);
-
 struct bch_inode_info {
 	struct inode		v;
 	unsigned long		ei_flags;
@@ -38,7 +18,7 @@ struct bch_inode_info {
 	struct mutex		ei_update_lock;
 	u64			ei_quota_reserved;
 	unsigned long		ei_last_dirtied;
-	struct pagecache_lock	ei_pagecache_lock;
+	two_state_lock_t	ei_pagecache_lock;
 
 	struct mutex		ei_quota_lock;
 	struct bch_qid		ei_qid;
@@ -49,6 +29,13 @@ struct bch_inode_info {
 	struct bch_inode_unpacked ei_inode;
 };
 
+#define bch2_pagecache_add_put(i)	bch2_two_state_unlock(&i->ei_pagecache_lock, 0)
+#define bch2_pagecache_add_tryget(i)	bch2_two_state_trylock(&i->ei_pagecache_lock, 0)
+#define bch2_pagecache_add_get(i)	bch2_two_state_lock(&i->ei_pagecache_lock, 0)
+
+#define bch2_pagecache_block_put(i)	bch2_two_state_unlock(&i->ei_pagecache_lock, 1)
+#define bch2_pagecache_block_get(i)	bch2_two_state_lock(&i->ei_pagecache_lock, 1)
+
 static inline subvol_inum inode_inum(struct bch_inode_info *inode)
 {
 	return (subvol_inum) {
@@ -95,7 +82,7 @@ do {									\
 			if ((_locks) & INODE_LOCK)			\
 				down_write_nested(&a[i]->v.i_rwsem, i);	\
 			if ((_locks) & INODE_PAGECACHE_BLOCK)		\
-				bch2_pagecache_block_get(&a[i]->ei_pagecache_lock);\
+				bch2_pagecache_block_get(a[i]);\
 			if ((_locks) & INODE_UPDATE_LOCK)			\
 				mutex_lock_nested(&a[i]->ei_update_lock, i);\
 		}							\
@@ -113,7 +100,7 @@ do {									\
 			if ((_locks) & INODE_LOCK)			\
 				up_write(&a[i]->v.i_rwsem);		\
 			if ((_locks) & INODE_PAGECACHE_BLOCK)		\
-				bch2_pagecache_block_put(&a[i]->ei_pagecache_lock);\
+				bch2_pagecache_block_put(a[i]);\
 			if ((_locks) & INODE_UPDATE_LOCK)			\
 				mutex_unlock(&a[i]->ei_update_lock);	\
 		}							\

fs/bcachefs/two_state_shared_lock.c

+// SPDX-License-Identifier: GPL-2.0
+
+#include "two_state_shared_lock.h"
+
+void bch2_two_state_unlock(two_state_lock_t *lock, int s)
+{
+	long i = s ? 1 : -1;
+
+	BUG_ON(atomic_long_read(&lock->v) == 0);
+
+	if (atomic_long_sub_return_release(i, &lock->v) == 0)
+		wake_up_all(&lock->wait);
+}
+
+bool bch2_two_state_trylock(two_state_lock_t *lock, int s)
+{
+	long i = s ? 1 : -1;
+	long v = atomic_long_read(&lock->v), old;
+
+	do {
+		old = v;
+
+		if (i > 0 ? v < 0 : v > 0)
+			return false;
+	} while ((v = atomic_long_cmpxchg_acquire(&lock->v,
+					old, old + i)) != old);
+	return true;
+}
+
+void bch2_two_state_lock(two_state_lock_t *lock, int s)
+{
+	wait_event(lock->wait, bch2_two_state_trylock(lock, s));
+}

fs/bcachefs/two_state_shared_lock.h

+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _BCACHEFS_TWO_STATE_LOCK_H
+#define _BCACHEFS_TWO_STATE_LOCK_H
+
+#include <linux/atomic.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+
+/*
+ * Two-state lock - can be taken for add or block - both states are shared,
+ * like read side of rwsem, but conflict with other state:
+ */
+typedef struct {
+	atomic_long_t		v;
+	wait_queue_head_t	wait;
+} two_state_lock_t;
+
+static inline void two_state_lock_init(two_state_lock_t *lock)
+{
+	atomic_long_set(&lock->v, 0);
+	init_waitqueue_head(&lock->wait);
+}
+
+void bch2_two_state_unlock(two_state_lock_t *, int);
+bool bch2_two_state_trylock(two_state_lock_t *, int);
+void bch2_two_state_lock(two_state_lock_t *, int);
+
+#endif /* _BCACHEFS_TWO_STATE_LOCK_H */