bcachefs: Add buffered IO fallback for userspace

In userspace, we want to be able to switch to buffered IO when we're
dealing with an image on a filesystem/device that doesn't support the
blocksize the filesystem was formatted with.

This plumbs through !opts.direct_io -> FMODE_BUFFERED, which will be
supported by the shim version of blkdev_get_by_path() in -tools, and it
adds a fallback to disable direct IO and retry for userspace.
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/super-io.c

@@ -662,7 +662,9 @@ int bch2_read_super(const char *path, struct bch_opts *opts,
 	struct printbuf err = PRINTBUF;
 	__le64 *i;
 	int ret;
-
+#ifndef __KERNEL__
+retry:
+#endif
 	memset(sb, 0, sizeof(*sb));
 	sb->mode	= BLK_OPEN_READ;
 	sb->have_bio	= true;
@@ -670,6 +672,11 @@ int bch2_read_super(const char *path, struct bch_opts *opts,
 	if (!sb->holder)
 		return -ENOMEM;
 
+#ifndef __KERNEL__
+	if (opt_get(*opts, direct_io) == false)
+		sb->mode |= FMODE_BUFFERED;
+#endif
+
 	if (!opt_get(*opts, noexcl))
 		sb->mode |= BLK_OPEN_EXCL;
 
@@ -754,7 +761,13 @@ int bch2_read_super(const char *path, struct bch_opts *opts,
 
 got_super:
 	if (le16_to_cpu(sb->sb->block_size) << 9 <
-	    bdev_logical_block_size(sb->bdev)) {
+	    bdev_logical_block_size(sb->bdev) &&
+	    opt_get(*opts, direct_io)) {
+#ifndef __KERNEL__
+		opt_set(*opts, direct_io, false);
+		bch2_free_super(sb);
+		goto retry;
+#endif
 		prt_printf(&err, "block size (%u) smaller than device block size (%u)",
 		       le16_to_cpu(sb->sb->block_size) << 9,
 		       bdev_logical_block_size(sb->bdev));