Commit 8d85b484 authored by Al Viro's avatar Al Viro

Allow sharing external names after __d_move()

* external dentry names get a small structure prepended to them
(struct external_name).
* it contains an atomic refcount, matching the number of struct dentry
instances that have ->d_name.name pointing to that external name.  The
first thing free_dentry() does is decrementing refcount of external name,
so the instances that are between the call of free_dentry() and
RCU-delayed actual freeing do not contribute.
* __d_move(x, y, false) makes the name of x equal to the name of y,
external or not.  If y has an external name, extra reference is grabbed
and put into x->d_name.name.  If x used to have an external name, the
reference to the old name is dropped and, should it reach zero, freeing
is scheduled via kfree_rcu().
* free_dentry() in dentry with external name decrements the refcount of
that name and, should it reach zero, does RCU-delayed call that will
free both the dentry and external name.  Otherwise it does what it
used to do, except that __d_free() doesn't even look at ->d_name.name;
it simply frees the dentry.

All non-RCU accesses to dentry external name are safe wrt freeing since they
all should happen before free_dentry() is called.  RCU accesses might run
into a dentry seen by free_dentry() or into an old name that got already
dropped by __d_move(); however, in both cases dentry must have been
alive and refer to that name at some point after we'd done rcu_read_lock(),
which means that any freeing must be still pending.
Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
parent 6d13f694
...@@ -235,18 +235,44 @@ static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *c ...@@ -235,18 +235,44 @@ static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *c
return dentry_string_cmp(cs, ct, tcount); return dentry_string_cmp(cs, ct, tcount);
} }
struct external_name {
union {
atomic_t count;
struct rcu_head head;
} u;
unsigned char name[];
};
static inline struct external_name *external_name(struct dentry *dentry)
{
return container_of(dentry->d_name.name, struct external_name, name[0]);
}
static void __d_free(struct rcu_head *head) static void __d_free(struct rcu_head *head)
{ {
struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
WARN_ON(!hlist_unhashed(&dentry->d_alias)); WARN_ON(!hlist_unhashed(&dentry->d_alias));
if (dname_external(dentry)) kmem_cache_free(dentry_cache, dentry);
kfree(dentry->d_name.name); }
static void __d_free_external(struct rcu_head *head)
{
struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
WARN_ON(!hlist_unhashed(&dentry->d_alias));
kfree(external_name(dentry));
kmem_cache_free(dentry_cache, dentry); kmem_cache_free(dentry_cache, dentry);
} }
static void dentry_free(struct dentry *dentry) static void dentry_free(struct dentry *dentry)
{ {
if (unlikely(dname_external(dentry))) {
struct external_name *p = external_name(dentry);
if (likely(atomic_dec_and_test(&p->u.count))) {
call_rcu(&dentry->d_u.d_rcu, __d_free_external);
return;
}
}
/* if dentry was never visible to RCU, immediate free is OK */ /* if dentry was never visible to RCU, immediate free is OK */
if (!(dentry->d_flags & DCACHE_RCUACCESS)) if (!(dentry->d_flags & DCACHE_RCUACCESS))
__d_free(&dentry->d_u.d_rcu); __d_free(&dentry->d_u.d_rcu);
...@@ -1438,11 +1464,14 @@ struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) ...@@ -1438,11 +1464,14 @@ struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
*/ */
dentry->d_iname[DNAME_INLINE_LEN-1] = 0; dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
if (name->len > DNAME_INLINE_LEN-1) { if (name->len > DNAME_INLINE_LEN-1) {
dname = kmalloc(name->len + 1, GFP_KERNEL); size_t size = offsetof(struct external_name, name[1]);
if (!dname) { struct external_name *p = kmalloc(size + name->len, GFP_KERNEL);
if (!p) {
kmem_cache_free(dentry_cache, dentry); kmem_cache_free(dentry_cache, dentry);
return NULL; return NULL;
} }
atomic_set(&p->u.count, 1);
dname = p->name;
} else { } else {
dname = dentry->d_iname; dname = dentry->d_iname;
} }
...@@ -2372,11 +2401,10 @@ void dentry_update_name_case(struct dentry *dentry, struct qstr *name) ...@@ -2372,11 +2401,10 @@ void dentry_update_name_case(struct dentry *dentry, struct qstr *name)
} }
EXPORT_SYMBOL(dentry_update_name_case); EXPORT_SYMBOL(dentry_update_name_case);
static void switch_names(struct dentry *dentry, struct dentry *target, static void swap_names(struct dentry *dentry, struct dentry *target)
bool exchange)
{ {
if (dname_external(target)) { if (unlikely(dname_external(target))) {
if (dname_external(dentry)) { if (unlikely(dname_external(dentry))) {
/* /*
* Both external: swap the pointers * Both external: swap the pointers
*/ */
...@@ -2392,7 +2420,7 @@ static void switch_names(struct dentry *dentry, struct dentry *target, ...@@ -2392,7 +2420,7 @@ static void switch_names(struct dentry *dentry, struct dentry *target,
target->d_name.name = target->d_iname; target->d_name.name = target->d_iname;
} }
} else { } else {
if (dname_external(dentry)) { if (unlikely(dname_external(dentry))) {
/* /*
* dentry:external, target:internal. Give dentry's * dentry:external, target:internal. Give dentry's
* storage to target and make dentry internal * storage to target and make dentry internal
...@@ -2407,12 +2435,6 @@ static void switch_names(struct dentry *dentry, struct dentry *target, ...@@ -2407,12 +2435,6 @@ static void switch_names(struct dentry *dentry, struct dentry *target,
*/ */
unsigned int i; unsigned int i;
BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long))); BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
if (!exchange) {
memcpy(dentry->d_iname, target->d_name.name,
target->d_name.len + 1);
dentry->d_name.hash_len = target->d_name.hash_len;
return;
}
for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) { for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
swap(((long *) &dentry->d_iname)[i], swap(((long *) &dentry->d_iname)[i],
((long *) &target->d_iname)[i]); ((long *) &target->d_iname)[i]);
...@@ -2422,6 +2444,24 @@ static void switch_names(struct dentry *dentry, struct dentry *target, ...@@ -2422,6 +2444,24 @@ static void switch_names(struct dentry *dentry, struct dentry *target,
swap(dentry->d_name.hash_len, target->d_name.hash_len); swap(dentry->d_name.hash_len, target->d_name.hash_len);
} }
static void copy_name(struct dentry *dentry, struct dentry *target)
{
struct external_name *old_name = NULL;
if (unlikely(dname_external(dentry)))
old_name = external_name(dentry);
if (unlikely(dname_external(target))) {
atomic_inc(&external_name(target)->u.count);
dentry->d_name = target->d_name;
} else {
memcpy(dentry->d_iname, target->d_name.name,
target->d_name.len + 1);
dentry->d_name.name = dentry->d_iname;
dentry->d_name.hash_len = target->d_name.hash_len;
}
if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
kfree_rcu(old_name, u.head);
}
static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target) static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target)
{ {
/* /*
...@@ -2518,7 +2558,10 @@ static void __d_move(struct dentry *dentry, struct dentry *target, ...@@ -2518,7 +2558,10 @@ static void __d_move(struct dentry *dentry, struct dentry *target,
} }
/* Switch the names.. */ /* Switch the names.. */
switch_names(dentry, target, exchange); if (exchange)
swap_names(dentry, target);
else
copy_name(dentry, target);
/* ... and switch them in the tree */ /* ... and switch them in the tree */
if (IS_ROOT(dentry)) { if (IS_ROOT(dentry)) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment