VFS: implement sendfile file operation

Currently the sendfile syscalls hardcode assumptions about the
implementation of the read file operations implementation.  Although
it checks for the presence of a readpage address-space operation
filesystems in Linux are free to implement read differently from
the generic version (generic_file_read).  Many filesystems such
as tmpfs, smbfs or xfs chose to implement it differently and need
additional locking, revalidation or checks. 

fs/adfs/file.c

@@ -36,6 +36,7 @@ struct file_operations adfs_file_operations = {
 	mmap:		generic_file_mmap,
 	fsync:		file_fsync,
 	write:		generic_file_write,
+	sendfile:	generic_file_sendfile,
 };
 
 struct inode_operations adfs_file_inode_operations = {

fs/affs/file.c

@@ -50,6 +50,7 @@ struct file_operations affs_file_operations = {
 	open:		affs_file_open,
 	release:	affs_file_release,
 	fsync:		file_fsync,
+	sendfile:	generic_file_sendfile,
 };
 
 struct inode_operations affs_file_inode_operations = {

fs/bfs/file.c

@@ -18,10 +18,11 @@
 #endif
 
 struct file_operations bfs_file_operations = {
-	llseek: generic_file_llseek,
-	read:	generic_file_read,
-	write:	generic_file_write,
-	mmap:	generic_file_mmap,
+	llseek: 	generic_file_llseek,
+	read:		generic_file_read,
+	write:		generic_file_write,
+	mmap:		generic_file_mmap,
+	sendfile:	generic_file_sendfile,
 };
 
 static int bfs_move_block(unsigned long from, unsigned long to, struct super_block *sb)

fs/block_dev.c

@@ -774,6 +774,7 @@ struct file_operations def_blk_fops = {
 	mmap:		generic_file_mmap,
 	fsync:		block_fsync,
 	ioctl:		blkdev_ioctl,
+	sendfile:	generic_file_sendfile,
 };
 
 int ioctl_by_bdev(struct block_device *bdev, unsigned cmd, unsigned long arg)

fs/ext2/file.c

@@ -46,6 +46,7 @@ struct file_operations ext2_file_operations = {
 	open:		generic_file_open,
 	release:	ext2_release_file,
 	fsync:		ext2_sync_file,
+	sendfile:	generic_file_sendfile,
 };
 
 struct inode_operations ext2_file_inode_operations = {

fs/ext3/file.c

@@ -84,6 +84,7 @@ struct file_operations ext3_file_operations = {
 	open:		ext3_open_file,		/* BKL not held.  Don't need */
 	release:	ext3_release_file,	/* BKL not held.  Don't need */
 	fsync:		ext3_sync_file,		/* BKL held */
+	sendfile:	generic_file_sendfile,	/* BKL not held.  Don't need */
 };
 
 struct inode_operations ext3_file_inode_operations = {

fs/freevxfs/vxfs_inode.c

@@ -53,6 +53,7 @@ static struct file_operations vxfs_file_operations = {
 	.llseek =		generic_file_llseek,
 	.read =			generic_file_read,
 	.mmap =			generic_file_mmap,
+	.sendfile =		generic_file_sendfile,
 };
 
 

fs/hpfs/inode.c

@@ -21,6 +21,7 @@ static struct file_operations hpfs_file_ops =
 	open:		hpfs_open,
 	release:	hpfs_file_release,
 	fsync:		hpfs_file_fsync,
+	sendfile:	generic_file_sendfile,
 };
 
 static struct inode_operations hpfs_file_iops =

fs/jffs/inode-v23.c

@@ -1631,13 +1631,14 @@ extern loff_t generic_file_llseek(struct file *, loff_t, int) __attribute__((wea
 
 static struct file_operations jffs_file_operations =
 {
-	open:	generic_file_open,
-	llseek:	generic_file_llseek,
-	read:	generic_file_read,
-	write:	generic_file_write,
-	ioctl:	jffs_ioctl,
-	mmap:	generic_file_mmap,
-	fsync:	jffs_fsync,
+	open:		generic_file_open,
+	llseek:		generic_file_llseek,
+	read:		generic_file_read,
+	write:		generic_file_write,
+	ioctl:		jffs_ioctl,
+	mmap:		generic_file_mmap,
+	fsync:		jffs_fsync,
+	sendfile:	generic_file_sendfile,
 };
 
 

fs/jffs2/file.c

@@ -82,6 +82,7 @@ struct file_operations jffs2_file_operations =
 	ioctl:		jffs2_ioctl,
 	mmap:		generic_file_mmap,
 	fsync:		jffs2_fsync
+	sendfile:	generic_file_sendfile,
 };
 
 /* jffs2_file_inode_operations */

fs/jfs/file.c

@@ -98,5 +98,6 @@ struct file_operations jfs_file_operations = {
 	.write		= generic_file_write,
 	.read		= generic_file_read,
 	.mmap		= generic_file_mmap,
+ 	.sendfile	= generic_file_sendfile,
 	.fsync		= jfs_fsync,
 };

fs/minix/file.c

@@ -21,6 +21,7 @@ struct file_operations minix_file_operations = {
 	write:		generic_file_write,
 	mmap:		generic_file_mmap,
 	fsync:		minix_sync_file,
+	sendfile:	generic_file_sendfile,
 };
 
 struct inode_operations minix_file_inode_operations = {

fs/ntfs/file.c

@@ -52,6 +52,7 @@ struct file_operations ntfs_file_ops = {
 	llseek:		generic_file_llseek,	/* Seek inside file. */
 	read:		generic_file_read,	/* Read from file. */
 	mmap:		generic_file_mmap,	/* Mmap file. */
+ 	sendfile:	generic_file_sendfile,	/* Zero-copy data send */
 	open:		ntfs_file_open,		/* Open file. */
 };
 

fs/qnx4/file.c

@@ -33,6 +33,7 @@ struct file_operations qnx4_file_operations =
 #ifdef CONFIG_QNX4FS_RW
 	fsync:			qnx4_sync_file,
 #endif
+	sendfile:		generic_file_sendfile,
 };
 
 struct inode_operations qnx4_file_inode_operations =

fs/ramfs/inode.c

@@ -276,6 +276,7 @@ static struct file_operations ramfs_file_operations = {
 	write:		generic_file_write,
 	mmap:		generic_file_mmap,
 	fsync:		ramfs_sync_file,
+	sendfile:	generic_file_sendfile,
 };
 
 static struct inode_operations ramfs_dir_inode_operations = {

fs/read_write.c

@@ -19,6 +19,7 @@ struct file_operations generic_ro_fops = {
 	llseek:		generic_file_llseek,
 	read:		generic_file_read,
 	mmap:		generic_file_mmap,
+	sendfile:	generic_file_sendfile,
 };
 
 loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
@@ -436,3 +437,113 @@ asmlinkage ssize_t sys_writev(unsigned long fd, const struct iovec * vector,
 bad_file:
 	return ret;
 }
+
+static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
+			   size_t count, loff_t max)
+{
+	struct file * in_file, * out_file;
+	struct inode * in_inode, * out_inode;
+	loff_t pos;
+	ssize_t retval;
+
+	/*
+	 * Get input file, and verify that it is ok..
+	 */
+	retval = -EBADF;
+	in_file = fget(in_fd);
+	if (!in_file)
+		goto out;
+	if (!(in_file->f_mode & FMODE_READ))
+		goto fput_in;
+	retval = -EINVAL;
+	in_inode = in_file->f_dentry->d_inode;
+	if (!in_inode)
+		goto fput_in;
+	if (!in_file->f_op || !in_file->f_op->sendfile)
+		goto fput_in;
+	retval = locks_verify_area(FLOCK_VERIFY_READ, in_inode, in_file, in_file->f_pos, count);
+	if (retval)
+		goto fput_in;
+
+	/*
+	 * Get output file, and verify that it is ok..
+	 */
+	retval = -EBADF;
+	out_file = fget(out_fd);
+	if (!out_file)
+		goto fput_in;
+	if (!(out_file->f_mode & FMODE_WRITE))
+		goto fput_out;
+	retval = -EINVAL;
+	if (!out_file->f_op || !out_file->f_op->sendpage)
+		goto fput_out;
+	out_inode = out_file->f_dentry->d_inode;
+	retval = locks_verify_area(FLOCK_VERIFY_WRITE, out_inode, out_file, out_file->f_pos, count);
+	if (retval)
+		goto fput_out;
+
+	if (!ppos)
+		ppos = &in_file->f_pos;
+
+	if (!max)
+		max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
+
+	pos = *ppos;
+	retval = -EINVAL;
+	if (unlikely(pos < 0))
+		goto fput_out;
+	if (unlikely(pos + count > max)) {
+		retval = -EOVERFLOW;
+		if (pos >= max)
+			goto fput_out;
+		count = max - pos;
+	}
+
+	retval = in_file->f_op->sendfile(out_file, in_file, ppos, count);
+
+	if (*ppos > max)
+		retval = -EOVERFLOW;
+
+fput_out:
+	fput(out_file);
+fput_in:
+	fput(in_file);
+out:
+	return retval;
+}
+
+asmlinkage ssize_t sys_sendfile(int out_fd, int in_fd, off_t *offset, size_t count)
+{
+	loff_t pos;
+	off_t off;
+	ssize_t ret;
+
+	if (offset) {
+		if (unlikely(get_user(off, offset)))
+			return -EFAULT;
+		pos = off;
+		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
+		if (unlikely(put_user(pos, offset)))
+			return -EFAULT;
+		return ret;
+	}
+
+	return do_sendfile(out_fd, in_fd, NULL, count, MAX_NON_LFS);
+}
+
+asmlinkage ssize_t sys_sendfile64(int out_fd, int in_fd, loff_t *offset, size_t count)
+{
+	loff_t pos;
+	ssize_t ret;
+
+	if (offset) {
+		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
+			return -EFAULT;
+		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
+		if (unlikely(put_user(pos, offset)))
+			return -EFAULT;
+		return ret;
+	}
+
+	return do_sendfile(out_fd, in_fd, NULL, count, 0);
+}

fs/reiserfs/file.c

@@ -147,6 +147,7 @@ struct file_operations reiserfs_file_operations = {
     mmap:	generic_file_mmap,
     release:	reiserfs_file_release,
     fsync:	reiserfs_sync_file,
+    sendfile:	generic_file_sendfile,
 };
 
 

fs/sysv/file.c

@@ -25,6 +25,7 @@ struct file_operations sysv_file_operations = {
 	write:		generic_file_write,
 	mmap:		generic_file_mmap,
 	fsync:		sysv_sync_file,
+	sendfile:	generic_file_sendfile,
 };
 
 struct inode_operations sysv_file_inode_operations = {

fs/udf/file.c

@@ -365,6 +365,7 @@ struct file_operations udf_file_operations = {
 	write:			udf_file_write,
 	release:		udf_release_file,
 	fsync:			udf_fsync_file,
+	sendfile:		generic_file_sendfile,
 };
 
 struct inode_operations udf_file_inode_operations = {

fs/ufs/file.c

@@ -47,6 +47,7 @@ struct file_operations ufs_file_operations = {
 	write:		generic_file_write,
 	mmap:		generic_file_mmap,
 	open:           generic_file_open,
+	sendfile:	generic_file_sendfile,
 };
 
 struct inode_operations ufs_file_inode_operations = {

include/linux/fs.h

@@ -757,6 +757,7 @@ struct file_operations {
 	int (*lock) (struct file *, int, struct file_lock *);
 	ssize_t (*readv) (struct file *, const struct iovec *, unsigned long, loff_t *);
 	ssize_t (*writev) (struct file *, const struct iovec *, unsigned long, loff_t *);
+	ssize_t (*sendfile) (struct file *, struct file *, loff_t *, size_t);
 	ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int);
 	unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
 };
@@ -1234,6 +1235,7 @@ extern int generic_file_mmap(struct file *, struct vm_area_struct *);
 extern int file_read_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size);
 extern ssize_t generic_file_read(struct file *, char *, size_t, loff_t *);
 extern ssize_t generic_file_write(struct file *, const char *, size_t, loff_t *);
+extern ssize_t generic_file_sendfile(struct file *, struct file *, loff_t *, size_t);
 extern void do_generic_file_read(struct file *, loff_t *, read_descriptor_t *, read_actor_t);
 ssize_t generic_file_direct_IO(int rw, struct inode *inode, char *buf,
 				loff_t offset, size_t count);

kernel/ksyms.c

@@ -218,6 +218,7 @@ EXPORT_SYMBOL(generic_commit_write);
 EXPORT_SYMBOL(block_truncate_page);
 EXPORT_SYMBOL(generic_block_bmap);
 EXPORT_SYMBOL(generic_file_read);
+EXPORT_SYMBOL(generic_file_sendfile);
 EXPORT_SYMBOL(do_generic_file_read);
 EXPORT_SYMBOL(generic_file_write);
 EXPORT_SYMBOL(generic_file_mmap);

mm/filemap.c

@@ -1119,127 +1119,23 @@ static int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned
 	return written;
 }
 
-static ssize_t common_sendfile(int out_fd, int in_fd, loff_t *offset, size_t count, loff_t max)
+ssize_t generic_file_sendfile(struct file *in_file, struct file *out_file,
+			      loff_t *ppos, size_t count)
 {
-	ssize_t retval;
-	struct file * in_file, * out_file;
-	struct inode * in_inode, * out_inode;
+	read_descriptor_t desc;
 
-	/*
-	 * Get input file, and verify that it is ok..
-	 */
-	retval = -EBADF;
-	in_file = fget(in_fd);
-	if (!in_file)
-		goto out;
-	if (!(in_file->f_mode & FMODE_READ))
-		goto fput_in;
-	retval = -EINVAL;
-	in_inode = in_file->f_dentry->d_inode;
-	if (!in_inode)
-		goto fput_in;
-	if (!in_inode->i_mapping->a_ops->readpage)
-		goto fput_in;
-	retval = locks_verify_area(FLOCK_VERIFY_READ, in_inode, in_file, in_file->f_pos, count);
-	if (retval)
-		goto fput_in;
-
-	retval = security_ops->file_permission (in_file, MAY_READ);
-	if (retval)
-		goto fput_in;
+	if (!count)
+		return 0;
 
-	/*
-	 * Get output file, and verify that it is ok..
-	 */
-	retval = -EBADF;
-	out_file = fget(out_fd);
-	if (!out_file)
-		goto fput_in;
-	if (!(out_file->f_mode & FMODE_WRITE))
-		goto fput_out;
-	retval = -EINVAL;
-	if (!out_file->f_op || !out_file->f_op->sendpage)
-		goto fput_out;
-	out_inode = out_file->f_dentry->d_inode;
-	retval = locks_verify_area(FLOCK_VERIFY_WRITE, out_inode, out_file, out_file->f_pos, count);
-	if (retval)
-		goto fput_out;
-
-	retval = security_ops->file_permission (out_file, MAY_WRITE);
-	if (retval)
-		goto fput_out;
-
-	retval = 0;
-	if (count) {
-		read_descriptor_t desc;
-		loff_t pos;
+	desc.written = 0;
+	desc.count = count;
+	desc.buf = (char *)out_file;
+	desc.error = 0;
 
-		if (!offset)
-			offset = &in_file->f_pos;
-
-		pos = *offset;
-		retval = -EINVAL;
-		if (unlikely(pos < 0))
-			goto fput_out;
-		if (unlikely(pos + count > max)) {
-			retval = -EOVERFLOW;
-			if (pos >= max)
-				goto fput_out;
-			count = max - pos;
-		}
-
-		desc.written = 0;
-		desc.count = count;
-		desc.buf = (char *) out_file;
-		desc.error = 0;
-		do_generic_file_read(in_file, offset, &desc, file_send_actor);
-
-		retval = desc.written;
-		if (!retval)
-			retval = desc.error;
-		pos = *offset;
-		if (pos > max)
-			retval = -EOVERFLOW;
-	}
-
-fput_out:
-	fput(out_file);
-fput_in:
-	fput(in_file);
-out:
-	return retval;
-}
-
-asmlinkage ssize_t sys_sendfile(int out_fd, int in_fd, off_t *offset, size_t count)
-{
-	loff_t pos, *ppos = NULL;
-	ssize_t ret;
-	if (offset) {
-		off_t off;
-		if (unlikely(get_user(off, offset)))
-			return -EFAULT;
-		pos = off;
-		ppos = &pos;
-	}
-	ret = common_sendfile(out_fd, in_fd, ppos, count, MAX_NON_LFS);
-	if (offset && put_user(pos, offset))
-		ret = -EFAULT;
-	return ret;
-}
-
-asmlinkage ssize_t sys_sendfile64(int out_fd, int in_fd, loff_t *offset, size_t count)
-{
-	loff_t pos, *ppos = NULL;
-	ssize_t ret;
-	if (offset) {
-		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
-			return -EFAULT;
-		ppos = &pos;
-	}
-	ret = common_sendfile(out_fd, in_fd, ppos, count, MAX_LFS_FILESIZE);
-	if (offset && put_user(pos, offset))
-		ret = -EFAULT;
-	return ret;
+	do_generic_file_read(in_file, ppos, &desc, file_send_actor);
+	if (desc.written)
+		return desc.written;
+	return desc.error;
 }
 
 static ssize_t