Commit 69f02461 authored by Julian Anastasov's avatar Julian Anastasov Committed by Greg Kroah-Hartman

net: call rcu_read_lock early in process_backlog

[ Upstream commit 2c17d27c ]

Incoming packet should be either in backlog queue or
in RCU read-side section. Otherwise, the final sequence of
flush_backlog() and synchronize_net() may miss packets
that can run without device reference:

CPU 1                  CPU 2
                       skb->dev: no reference
                       process_backlog:__skb_dequeue
                       process_backlog:local_irq_enable

on_each_cpu for
flush_backlog =>       IPI(hardirq): flush_backlog
                       - packet not found in backlog

                       CPU delayed ...
synchronize_net
- no ongoing RCU
read-side sections

netdev_run_todo,
rcu_barrier: no
ongoing callbacks
                       __netif_receive_skb_core:rcu_read_lock
                       - too late
free dev
                       process packet for freed dev

Fixes: 6e583ce5 ("net: eliminate refcounting in backlog queue")
Cc: Eric W. Biederman <ebiederm@xmission.com>
Cc: Stephen Hemminger <stephen@networkplumber.org>
Signed-off-by: default avatarJulian Anastasov <ja@ssi.bg>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 2095ab24
...@@ -3554,8 +3554,6 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) ...@@ -3554,8 +3554,6 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
pt_prev = NULL; pt_prev = NULL;
rcu_read_lock();
another_round: another_round:
skb->skb_iif = skb->dev->ifindex; skb->skb_iif = skb->dev->ifindex;
...@@ -3565,7 +3563,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) ...@@ -3565,7 +3563,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
skb->protocol == cpu_to_be16(ETH_P_8021AD)) { skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
skb = skb_vlan_untag(skb); skb = skb_vlan_untag(skb);
if (unlikely(!skb)) if (unlikely(!skb))
goto unlock; goto out;
} }
#ifdef CONFIG_NET_CLS_ACT #ifdef CONFIG_NET_CLS_ACT
...@@ -3590,7 +3588,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) ...@@ -3590,7 +3588,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
#ifdef CONFIG_NET_CLS_ACT #ifdef CONFIG_NET_CLS_ACT
skb = handle_ing(skb, &pt_prev, &ret, orig_dev); skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
if (!skb) if (!skb)
goto unlock; goto out;
ncls: ncls:
#endif #endif
...@@ -3605,7 +3603,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) ...@@ -3605,7 +3603,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
if (vlan_do_receive(&skb)) if (vlan_do_receive(&skb))
goto another_round; goto another_round;
else if (unlikely(!skb)) else if (unlikely(!skb))
goto unlock; goto out;
} }
rx_handler = rcu_dereference(skb->dev->rx_handler); rx_handler = rcu_dereference(skb->dev->rx_handler);
...@@ -3617,7 +3615,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) ...@@ -3617,7 +3615,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
switch (rx_handler(&skb)) { switch (rx_handler(&skb)) {
case RX_HANDLER_CONSUMED: case RX_HANDLER_CONSUMED:
ret = NET_RX_SUCCESS; ret = NET_RX_SUCCESS;
goto unlock; goto out;
case RX_HANDLER_ANOTHER: case RX_HANDLER_ANOTHER:
goto another_round; goto another_round;
case RX_HANDLER_EXACT: case RX_HANDLER_EXACT:
...@@ -3669,8 +3667,6 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) ...@@ -3669,8 +3667,6 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
ret = NET_RX_DROP; ret = NET_RX_DROP;
} }
unlock:
rcu_read_unlock();
out: out:
return ret; return ret;
} }
...@@ -3702,29 +3698,30 @@ static int __netif_receive_skb(struct sk_buff *skb) ...@@ -3702,29 +3698,30 @@ static int __netif_receive_skb(struct sk_buff *skb)
static int netif_receive_skb_internal(struct sk_buff *skb) static int netif_receive_skb_internal(struct sk_buff *skb)
{ {
int ret;
net_timestamp_check(netdev_tstamp_prequeue, skb); net_timestamp_check(netdev_tstamp_prequeue, skb);
if (skb_defer_rx_timestamp(skb)) if (skb_defer_rx_timestamp(skb))
return NET_RX_SUCCESS; return NET_RX_SUCCESS;
rcu_read_lock();
#ifdef CONFIG_RPS #ifdef CONFIG_RPS
if (static_key_false(&rps_needed)) { if (static_key_false(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow; struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret; int cpu = get_rps_cpu(skb->dev, skb, &rflow);
rcu_read_lock();
cpu = get_rps_cpu(skb->dev, skb, &rflow);
if (cpu >= 0) { if (cpu >= 0) {
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock(); rcu_read_unlock();
return ret; return ret;
} }
rcu_read_unlock();
} }
#endif #endif
return __netif_receive_skb(skb); ret = __netif_receive_skb(skb);
rcu_read_unlock();
return ret;
} }
/** /**
...@@ -4185,8 +4182,10 @@ static int process_backlog(struct napi_struct *napi, int quota) ...@@ -4185,8 +4182,10 @@ static int process_backlog(struct napi_struct *napi, int quota)
unsigned int qlen; unsigned int qlen;
while ((skb = __skb_dequeue(&sd->process_queue))) { while ((skb = __skb_dequeue(&sd->process_queue))) {
rcu_read_lock();
local_irq_enable(); local_irq_enable();
__netif_receive_skb(skb); __netif_receive_skb(skb);
rcu_read_unlock();
local_irq_disable(); local_irq_disable();
input_queue_head_incr(sd); input_queue_head_incr(sd);
if (++work >= quota) { if (++work >= quota) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment