Commit 6a414e64 authored by Andrew Morton's avatar Andrew Morton Committed by Linus Torvalds

[PATCH] reduced overheads in fget/fput

From: Dipankar Sarma <dipankar@in.ibm.com>


fget() shows up on profiles, especially on SMP.  Dipankar's patch
special-cases the situation wherein there are no sharers of current->files.

In this situation we know that no other process can close this file, so it
is not necessary to increment the file's refcount.

It's ugly as sin, but makes a substantial difference.

The test is

	dd if=/dev/zero of=foo bs=1 count=1M

On 4CPU P3 xeon with 1MB L2 cache and 512MB ram:

	kernel           sys time     std-dev
	------------     --------     -------

	UP - vanilla     2.104        0.028
	UP - file        1.867        0.019

	SMP - vanilla    2.976        0.023
	SMP - file       2.719        0.026
parent 09f95761
......@@ -141,7 +141,7 @@ void close_private_file(struct file *file)
security_file_free(file);
}
void fput(struct file * file)
void fput(struct file *file)
{
if (atomic_dec_and_test(&file->f_count))
__fput(file);
......@@ -190,6 +190,34 @@ struct file *fget(unsigned int fd)
return file;
}
/*
* Lightweight file lookup - no refcnt increment if fd table isn't shared.
* You can use this only if it is guranteed that the current task already
* holds a refcnt to that file. That check has to be done at fget() only
* and a flag is returned to be passed to the corresponding fput_light().
* There must not be a cloning between an fget_light/fput_light pair.
*/
struct file *fget_light(unsigned int fd, int *fput_needed)
{
struct file *file;
struct files_struct *files = current->files;
*fput_needed = 0;
if (likely((atomic_read(&files->count) == 1))) {
file = fcheck(fd);
} else {
spin_lock(&files->file_lock);
file = fcheck(fd);
if (file) {
get_file(file);
*fput_needed = 1;
}
spin_unlock(&files->file_lock);
}
return file;
}
void put_filp(struct file *file)
{
if (atomic_dec_and_test(&file->f_count)) {
......
......@@ -115,9 +115,10 @@ asmlinkage off_t sys_lseek(unsigned int fd, off_t offset, unsigned int origin)
{
off_t retval;
struct file * file;
int fput_needed;
retval = -EBADF;
file = fget(fd);
file = fget_light(fd, &fput_needed);
if (!file)
goto bad;
......@@ -128,7 +129,7 @@ asmlinkage off_t sys_lseek(unsigned int fd, off_t offset, unsigned int origin)
if (res != (loff_t)retval)
retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
}
fput(file);
fput_light(file, fput_needed);
bad:
return retval;
}
......@@ -141,9 +142,10 @@ asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
int retval;
struct file * file;
loff_t offset;
int fput_needed;
retval = -EBADF;
file = fget(fd);
file = fget_light(fd, &fput_needed);
if (!file)
goto bad;
......@@ -161,7 +163,7 @@ asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
retval = 0;
}
out_putf:
fput(file);
fput_light(file, fput_needed);
bad:
return retval;
}
......@@ -251,11 +253,12 @@ asmlinkage ssize_t sys_read(unsigned int fd, char __user * buf, size_t count)
{
struct file *file;
ssize_t ret = -EBADF;
int fput_needed;
file = fget(fd);
file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_read(file, buf, count, &file->f_pos);
fput(file);
fput_light(file, fput_needed);
}
return ret;
......@@ -265,11 +268,12 @@ asmlinkage ssize_t sys_write(unsigned int fd, const char __user * buf, size_t co
{
struct file *file;
ssize_t ret = -EBADF;
int fput_needed;
file = fget(fd);
file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_write(file, buf, count, &file->f_pos);
fput(file);
fput_light(file, fput_needed);
}
return ret;
......@@ -280,14 +284,15 @@ asmlinkage ssize_t sys_pread64(unsigned int fd, char __user *buf,
{
struct file *file;
ssize_t ret = -EBADF;
int fput_needed;
if (pos < 0)
return -EINVAL;
file = fget(fd);
file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_read(file, buf, count, &pos);
fput(file);
fput_light(file, fput_needed);
}
return ret;
......@@ -298,14 +303,15 @@ asmlinkage ssize_t sys_pwrite64(unsigned int fd, const char __user *buf,
{
struct file *file;
ssize_t ret = -EBADF;
int fput_needed;
if (pos < 0)
return -EINVAL;
file = fget(fd);
file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_write(file, buf, count, &pos);
fput(file);
fput_light(file, fput_needed);
}
return ret;
......@@ -479,11 +485,12 @@ sys_readv(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
{
struct file *file;
ssize_t ret = -EBADF;
int fput_needed;
file = fget(fd);
file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_readv(file, vec, vlen, &file->f_pos);
fput(file);
fput_light(file, fput_needed);
}
return ret;
......@@ -494,11 +501,12 @@ sys_writev(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
{
struct file *file;
ssize_t ret = -EBADF;
int fput_needed;
file = fget(fd);
file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_writev(file, vec, vlen, &file->f_pos);
fput(file);
fput_light(file, fput_needed);
}
return ret;
......@@ -511,12 +519,13 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
struct inode * in_inode, * out_inode;
loff_t pos;
ssize_t retval;
int fput_needed_in, fput_needed_out;
/*
* Get input file, and verify that it is ok..
*/
retval = -EBADF;
in_file = fget(in_fd);
in_file = fget_light(in_fd, &fput_needed_in);
if (!in_file)
goto out;
if (!(in_file->f_mode & FMODE_READ))
......@@ -539,7 +548,7 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
* Get output file, and verify that it is ok..
*/
retval = -EBADF;
out_file = fget(out_fd);
out_file = fget_light(out_fd, &fput_needed_out);
if (!out_file)
goto fput_in;
if (!(out_file->f_mode & FMODE_WRITE))
......@@ -579,9 +588,9 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
retval = -EOVERFLOW;
fput_out:
fput(out_file);
fput_light(out_file, fput_needed_out);
fput_in:
fput(in_file);
fput_light(in_file, fput_needed_in);
out:
return retval;
}
......
......@@ -35,7 +35,15 @@ struct files_struct {
extern void FASTCALL(__fput(struct file *));
extern void FASTCALL(fput(struct file *));
static inline void fput_light(struct file *file, int fput_needed)
{
if (unlikely(fput_needed))
fput(file);
}
extern struct file * FASTCALL(fget(unsigned int fd));
extern struct file * FASTCALL(fget_light(unsigned int fd, int *fput_needed));
extern void FASTCALL(set_close_on_exec(unsigned int fd, int flag));
extern void put_filp(struct file *);
extern int get_unused_fd(void);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment