Commit 898683e9 authored by Hanna V. Linder's avatar Hanna V. Linder Committed by Linus Torvalds

[PATCH] FastWalk Dcache

Reduce cacheline bouncing when a dentry is in the cache.

Specifically, the d_count reference counter is not incremented and
decremented for every dentry in a path during path walking if the dentry
is in the dcache.  Execcisve atomic inc/dec's are expensive on SMP
systems due to the cachline bouncing.
parent fe7885a4
...@@ -846,13 +846,22 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) ...@@ -846,13 +846,22 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
struct dentry * d_lookup(struct dentry * parent, struct qstr * name) struct dentry * d_lookup(struct dentry * parent, struct qstr * name)
{ {
struct dentry * dentry;
spin_lock(&dcache_lock);
dentry = __d_lookup(parent,name);
spin_unlock(&dcache_lock);
return dentry;
}
struct dentry * __d_lookup(struct dentry * parent, struct qstr * name)
{
unsigned int len = name->len; unsigned int len = name->len;
unsigned int hash = name->hash; unsigned int hash = name->hash;
const unsigned char *str = name->name; const unsigned char *str = name->name;
struct list_head *head = d_hash(parent,hash); struct list_head *head = d_hash(parent,hash);
struct list_head *tmp; struct list_head *tmp;
spin_lock(&dcache_lock);
tmp = head->next; tmp = head->next;
for (;;) { for (;;) {
struct dentry * dentry = list_entry(tmp, struct dentry, d_hash); struct dentry * dentry = list_entry(tmp, struct dentry, d_hash);
...@@ -874,10 +883,8 @@ struct dentry * d_lookup(struct dentry * parent, struct qstr * name) ...@@ -874,10 +883,8 @@ struct dentry * d_lookup(struct dentry * parent, struct qstr * name)
} }
__dget_locked(dentry); __dget_locked(dentry);
dentry->d_vfs_flags |= DCACHE_REFERENCED; dentry->d_vfs_flags |= DCACHE_REFERENCED;
spin_unlock(&dcache_lock);
return dentry; return dentry;
} }
spin_unlock(&dcache_lock);
return NULL; return NULL;
} }
......
...@@ -268,8 +268,41 @@ void path_release(struct nameidata *nd) ...@@ -268,8 +268,41 @@ void path_release(struct nameidata *nd)
static struct dentry * cached_lookup(struct dentry * parent, struct qstr * name, int flags) static struct dentry * cached_lookup(struct dentry * parent, struct qstr * name, int flags)
{ {
struct dentry * dentry = d_lookup(parent, name); struct dentry * dentry = d_lookup(parent, name);
if (dentry && dentry->d_op && dentry->d_op->d_revalidate) {
if (!dentry->d_op->d_revalidate(dentry, flags) && !d_invalidate(dentry)) {
dput(dentry);
dentry = NULL;
}
}
return dentry;
}
/*for fastwalking*/
static inline void undo_locked(struct nameidata *nd)
{
if(nd->flags & LOOKUP_LOCKED){
dget_locked(nd->dentry);
mntget(nd->mnt);
spin_unlock(&dcache_lock);
nd->flags &= ~LOOKUP_LOCKED;
}
}
/*
* For fast path lookup while holding the dcache_lock.
* SMP-safe
*/
static struct dentry * cached_lookup_nd(struct nameidata * nd, struct qstr * name, int flags)
{
struct dentry * dentry = NULL;
if(!(nd->flags & LOOKUP_LOCKED))
return cached_lookup(nd->dentry, name, flags);
dentry = __d_lookup(nd->dentry, name);
if (dentry && dentry->d_op && dentry->d_op->d_revalidate) { if (dentry && dentry->d_op && dentry->d_op->d_revalidate) {
undo_locked(nd);
if (!dentry->d_op->d_revalidate(dentry, flags) && !d_invalidate(dentry)) { if (!dentry->d_op->d_revalidate(dentry, flags) && !d_invalidate(dentry)) {
dput(dentry); dput(dentry);
dentry = NULL; dentry = NULL;
...@@ -278,6 +311,34 @@ static struct dentry * cached_lookup(struct dentry * parent, struct qstr * name, ...@@ -278,6 +311,34 @@ static struct dentry * cached_lookup(struct dentry * parent, struct qstr * name,
return dentry; return dentry;
} }
/*
* Short-cut version of permission(), for calling by
* path_walk(), when dcache lock is held. Combines parts
* of permission() and vfs_permission(), and tests ONLY for
* MAY_EXEC permission.
*
* If appropriate, check DAC only. If not appropriate, or
* short-cut DAC fails, then call permission() to do more
* complete permission check.
*/
static inline int exec_permission_lite(struct inode *inode)
{
umode_t mode = inode->i_mode;
if ((inode->i_op && inode->i_op->permission))
return -EACCES;
if (current->fsuid == inode->i_uid)
mode >>= 6;
else if (in_group_p(inode->i_gid))
mode >>= 3;
if (mode & MAY_EXEC)
return 0;
return -EACCES;
}
/* /*
* This is called when everything else fails, and we actually have * This is called when everything else fails, and we actually have
* to go to the low-level filesystem to find out what we should do.. * to go to the low-level filesystem to find out what we should do..
...@@ -472,7 +533,11 @@ int link_path_walk(const char * name, struct nameidata *nd) ...@@ -472,7 +533,11 @@ int link_path_walk(const char * name, struct nameidata *nd)
struct qstr this; struct qstr this;
unsigned int c; unsigned int c;
err = permission(inode, MAY_EXEC); err = exec_permission_lite(inode);
if(err){
undo_locked(nd);
err = permission(inode, MAY_EXEC);
}
dentry = ERR_PTR(err); dentry = ERR_PTR(err);
if (err) if (err)
break; break;
...@@ -507,6 +572,7 @@ int link_path_walk(const char * name, struct nameidata *nd) ...@@ -507,6 +572,7 @@ int link_path_walk(const char * name, struct nameidata *nd)
case 2: case 2:
if (this.name[1] != '.') if (this.name[1] != '.')
break; break;
undo_locked(nd);
follow_dotdot(nd); follow_dotdot(nd);
inode = nd->dentry->d_inode; inode = nd->dentry->d_inode;
/* fallthrough */ /* fallthrough */
...@@ -523,16 +589,20 @@ int link_path_walk(const char * name, struct nameidata *nd) ...@@ -523,16 +589,20 @@ int link_path_walk(const char * name, struct nameidata *nd)
break; break;
} }
/* This does the actual lookups.. */ /* This does the actual lookups.. */
dentry = cached_lookup(nd->dentry, &this, LOOKUP_CONTINUE); dentry = cached_lookup_nd(nd, &this, LOOKUP_CONTINUE);
if (!dentry) { if (!dentry) {
undo_locked(nd);
dentry = real_lookup(nd->dentry, &this, LOOKUP_CONTINUE); dentry = real_lookup(nd->dentry, &this, LOOKUP_CONTINUE);
err = PTR_ERR(dentry); err = PTR_ERR(dentry);
if (IS_ERR(dentry)) if (IS_ERR(dentry))
break; break;
} }
/* Check mountpoints.. */ /* Check mountpoints.. */
while (d_mountpoint(dentry) && __follow_down(&nd->mnt, &dentry)) if(d_mountpoint(dentry)){
; undo_locked(nd);
while (d_mountpoint(dentry) && __follow_down(&nd->mnt, &dentry))
;
}
err = -ENOENT; err = -ENOENT;
inode = dentry->d_inode; inode = dentry->d_inode;
...@@ -543,6 +613,7 @@ int link_path_walk(const char * name, struct nameidata *nd) ...@@ -543,6 +613,7 @@ int link_path_walk(const char * name, struct nameidata *nd)
goto out_dput; goto out_dput;
if (inode->i_op->follow_link) { if (inode->i_op->follow_link) {
undo_locked(nd);
err = do_follow_link(dentry, nd); err = do_follow_link(dentry, nd);
dput(dentry); dput(dentry);
if (err) if (err)
...@@ -555,7 +626,8 @@ int link_path_walk(const char * name, struct nameidata *nd) ...@@ -555,7 +626,8 @@ int link_path_walk(const char * name, struct nameidata *nd)
if (!inode->i_op) if (!inode->i_op)
break; break;
} else { } else {
dput(nd->dentry); if (!(nd->flags & LOOKUP_LOCKED))
dput(nd->dentry);
nd->dentry = dentry; nd->dentry = dentry;
} }
err = -ENOTDIR; err = -ENOTDIR;
...@@ -575,6 +647,7 @@ int link_path_walk(const char * name, struct nameidata *nd) ...@@ -575,6 +647,7 @@ int link_path_walk(const char * name, struct nameidata *nd)
case 2: case 2:
if (this.name[1] != '.') if (this.name[1] != '.')
break; break;
undo_locked(nd);
follow_dotdot(nd); follow_dotdot(nd);
inode = nd->dentry->d_inode; inode = nd->dentry->d_inode;
/* fallthrough */ /* fallthrough */
...@@ -586,7 +659,8 @@ int link_path_walk(const char * name, struct nameidata *nd) ...@@ -586,7 +659,8 @@ int link_path_walk(const char * name, struct nameidata *nd)
if (err < 0) if (err < 0)
break; break;
} }
dentry = cached_lookup(nd->dentry, &this, 0); dentry = cached_lookup_nd(nd, &this, 0);
undo_locked(nd);
if (!dentry) { if (!dentry) {
dentry = real_lookup(nd->dentry, &this, 0); dentry = real_lookup(nd->dentry, &this, 0);
err = PTR_ERR(dentry); err = PTR_ERR(dentry);
...@@ -626,11 +700,14 @@ int link_path_walk(const char * name, struct nameidata *nd) ...@@ -626,11 +700,14 @@ int link_path_walk(const char * name, struct nameidata *nd)
else if (this.len == 2 && this.name[1] == '.') else if (this.len == 2 && this.name[1] == '.')
nd->last_type = LAST_DOTDOT; nd->last_type = LAST_DOTDOT;
return_base: return_base:
undo_locked(nd);
return 0; return 0;
out_dput: out_dput:
undo_locked(nd);
dput(dentry); dput(dentry);
break; break;
} }
undo_locked(nd);
path_release(nd); path_release(nd);
return_err: return_err:
return err; return err;
...@@ -736,6 +813,36 @@ int path_init(const char *name, unsigned int flags, struct nameidata *nd) ...@@ -736,6 +813,36 @@ int path_init(const char *name, unsigned int flags, struct nameidata *nd)
return 1; return 1;
} }
int path_lookup(const char *name, unsigned int flags, struct nameidata *nd)
{
nd->last_type = LAST_ROOT; /* if there are only slashes... */
nd->flags = flags;
if (*name=='/'){
read_lock(&current->fs->lock);
if (current->fs->altroot && !(nd->flags & LOOKUP_NOALT)) {
nd->mnt = mntget(current->fs->altrootmnt);
nd->dentry = dget(current->fs->altroot);
read_unlock(&current->fs->lock);
if (__emul_lookup_dentry(name,nd))
return 0;
read_lock(&current->fs->lock);
}
spin_lock(&dcache_lock); /*to avoid cacheline bouncing with d_count*/
nd->mnt = current->fs->rootmnt;
nd->dentry = current->fs->root;
read_unlock(&current->fs->lock);
}
else{
read_lock(&current->fs->lock);
spin_lock(&dcache_lock);
nd->mnt = current->fs->pwdmnt;
nd->dentry = current->fs->pwd;
read_unlock(&current->fs->lock);
}
nd->flags |= LOOKUP_LOCKED;
return (path_walk(name, nd));
}
/* /*
* Restricted form of lookup. Doesn't follow links, single-component only, * Restricted form of lookup. Doesn't follow links, single-component only,
* needs parent already locked. Doesn't follow mounts. * needs parent already locked. Doesn't follow mounts.
......
...@@ -229,6 +229,7 @@ extern void d_move(struct dentry *, struct dentry *); ...@@ -229,6 +229,7 @@ extern void d_move(struct dentry *, struct dentry *);
/* appendix may either be NULL or be used for transname suffixes */ /* appendix may either be NULL or be used for transname suffixes */
extern struct dentry * d_lookup(struct dentry *, struct qstr *); extern struct dentry * d_lookup(struct dentry *, struct qstr *);
extern struct dentry * __d_lookup(struct dentry *, struct qstr *);
/* validate "insecure" dentry pointer */ /* validate "insecure" dentry pointer */
extern int d_validate(struct dentry *, struct dentry *); extern int d_validate(struct dentry *, struct dentry *);
......
...@@ -1386,12 +1386,15 @@ extern ino_t find_inode_number(struct dentry *, struct qstr *); ...@@ -1386,12 +1386,15 @@ extern ino_t find_inode_number(struct dentry *, struct qstr *);
* - require a directory * - require a directory
* - ending slashes ok even for nonexistent files * - ending slashes ok even for nonexistent files
* - internal "there are more path compnents" flag * - internal "there are more path compnents" flag
* - locked when lookup done with dcache_lock held
*/ */
#define LOOKUP_FOLLOW (1) #define LOOKUP_FOLLOW (1)
#define LOOKUP_DIRECTORY (2) #define LOOKUP_DIRECTORY (2)
#define LOOKUP_CONTINUE (4) #define LOOKUP_CONTINUE (4)
#define LOOKUP_PARENT (16) #define LOOKUP_PARENT (16)
#define LOOKUP_NOALT (32) #define LOOKUP_NOALT (32)
#define LOOKUP_LOCKED (64)
/* /*
* Type of the last component on LOOKUP_PARENT * Type of the last component on LOOKUP_PARENT
*/ */
...@@ -1421,14 +1424,8 @@ extern loff_t default_llseek(struct file *file, loff_t offset, int origin); ...@@ -1421,14 +1424,8 @@ extern loff_t default_llseek(struct file *file, loff_t offset, int origin);
extern int FASTCALL(__user_walk(const char *, unsigned, struct nameidata *)); extern int FASTCALL(__user_walk(const char *, unsigned, struct nameidata *));
extern int FASTCALL(path_init(const char *, unsigned, struct nameidata *)); extern int FASTCALL(path_init(const char *, unsigned, struct nameidata *));
extern int FASTCALL(path_walk(const char *, struct nameidata *)); extern int FASTCALL(path_walk(const char *, struct nameidata *));
extern int FASTCALL(path_lookup(const char *, unsigned, struct nameidata *));
extern int FASTCALL(link_path_walk(const char *, struct nameidata *)); extern int FASTCALL(link_path_walk(const char *, struct nameidata *));
static inline int path_lookup(const char *path, unsigned flags, struct nameidata *nd)
{
int error = 0;
if (path_init(path, flags, nd))
error = path_walk(path, nd);
return error;
}
extern void path_release(struct nameidata *); extern void path_release(struct nameidata *);
extern int follow_down(struct vfsmount **, struct dentry **); extern int follow_down(struct vfsmount **, struct dentry **);
extern int follow_up(struct vfsmount **, struct dentry **); extern int follow_up(struct vfsmount **, struct dentry **);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment