Commit b24d18aa authored by Herbert Xu's avatar Herbert Xu Committed by Linus Torvalds

[PATCH] list: add missing rcu_dereference on first element

It seems that all the list_*_rcu primitives are missing a memory barrier
on the very first dereference.  For example,

#define list_for_each_rcu(pos, head) \
	for (pos = (head)->next; prefetch(pos->next), pos != (head); \
		pos = rcu_dereference(pos->next))

It will go something like:

	pos = (head)->next

	prefetch(pos->next)

	pos != (head)

	do stuff

We're missing a barrier here.

	pos = rcu_dereference(pos->next)

		fetch pos->next

		barrier given by rcu_dereference(pos->next)

		store pos

Without the missing barrier, the pos->next value may turn out to be stale.
In fact, if "do stuff" were also dereferencing pos and relying on
list_for_each_rcu to provide the barrier then it may also break.

So here is a patch to make sure that we have a barrier for the first
element in the list.
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
Acked-by: default avatar"Paul E. McKenney" <paulmck@us.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 3d80636a
...@@ -442,12 +442,14 @@ static inline void list_splice_init(struct list_head *list, ...@@ -442,12 +442,14 @@ static inline void list_splice_init(struct list_head *list,
* as long as the traversal is guarded by rcu_read_lock(). * as long as the traversal is guarded by rcu_read_lock().
*/ */
#define list_for_each_rcu(pos, head) \ #define list_for_each_rcu(pos, head) \
for (pos = (head)->next; prefetch(pos->next), pos != (head); \ for (pos = (head)->next; \
pos = rcu_dereference(pos->next)) prefetch(rcu_dereference(pos)->next), pos != (head); \
pos = pos->next)
#define __list_for_each_rcu(pos, head) \ #define __list_for_each_rcu(pos, head) \
for (pos = (head)->next; pos != (head); \ for (pos = (head)->next; \
pos = rcu_dereference(pos->next)) rcu_dereference(pos) != (head); \
pos = pos->next)
/** /**
* list_for_each_safe_rcu - iterate over an rcu-protected list safe * list_for_each_safe_rcu - iterate over an rcu-protected list safe
...@@ -461,8 +463,9 @@ static inline void list_splice_init(struct list_head *list, ...@@ -461,8 +463,9 @@ static inline void list_splice_init(struct list_head *list,
* as long as the traversal is guarded by rcu_read_lock(). * as long as the traversal is guarded by rcu_read_lock().
*/ */
#define list_for_each_safe_rcu(pos, n, head) \ #define list_for_each_safe_rcu(pos, n, head) \
for (pos = (head)->next, n = pos->next; pos != (head); \ for (pos = (head)->next; \
pos = rcu_dereference(n), n = pos->next) n = rcu_dereference(pos)->next, pos != (head); \
pos = n)
/** /**
* list_for_each_entry_rcu - iterate over rcu list of given type * list_for_each_entry_rcu - iterate over rcu list of given type
...@@ -476,9 +479,9 @@ static inline void list_splice_init(struct list_head *list, ...@@ -476,9 +479,9 @@ static inline void list_splice_init(struct list_head *list,
*/ */
#define list_for_each_entry_rcu(pos, head, member) \ #define list_for_each_entry_rcu(pos, head, member) \
for (pos = list_entry((head)->next, typeof(*pos), member); \ for (pos = list_entry((head)->next, typeof(*pos), member); \
prefetch(pos->member.next), &pos->member != (head); \ prefetch(rcu_dereference(pos)->member.next), \
pos = rcu_dereference(list_entry(pos->member.next, \ &pos->member != (head); \
typeof(*pos), member))) pos = list_entry(pos->member.next, typeof(*pos), member))
/** /**
...@@ -492,8 +495,9 @@ static inline void list_splice_init(struct list_head *list, ...@@ -492,8 +495,9 @@ static inline void list_splice_init(struct list_head *list,
* as long as the traversal is guarded by rcu_read_lock(). * as long as the traversal is guarded by rcu_read_lock().
*/ */
#define list_for_each_continue_rcu(pos, head) \ #define list_for_each_continue_rcu(pos, head) \
for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \ for ((pos) = (pos)->next; \
(pos) = rcu_dereference((pos)->next)) prefetch(rcu_dereference((pos))->next), (pos) != (head); \
(pos) = (pos)->next)
/* /*
* Double linked lists with a single pointer list head. * Double linked lists with a single pointer list head.
...@@ -696,8 +700,9 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, ...@@ -696,8 +700,9 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
pos = n) pos = n)
#define hlist_for_each_rcu(pos, head) \ #define hlist_for_each_rcu(pos, head) \
for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \ for ((pos) = (head)->first; \
(pos) = rcu_dereference((pos)->next)) rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \
(pos) = (pos)->next)
/** /**
* hlist_for_each_entry - iterate over list of given type * hlist_for_each_entry - iterate over list of given type
...@@ -762,9 +767,9 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, ...@@ -762,9 +767,9 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
*/ */
#define hlist_for_each_entry_rcu(tpos, pos, head, member) \ #define hlist_for_each_entry_rcu(tpos, pos, head, member) \
for (pos = (head)->first; \ for (pos = (head)->first; \
pos && ({ prefetch(pos->next); 1;}) && \ rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) && \
({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
pos = rcu_dereference(pos->next)) pos = pos->next)
#else #else
#warning "don't include kernel headers in userspace" #warning "don't include kernel headers in userspace"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment