Commit fb1843ee authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

net/mlx4_en: Avoid false sharing in mlx4_en_en_process_tx_cq()

mlx4_en_process_tx_cq() carefully fetches and writes ring->last_nr_txbb
and ring->cons only one time to avoid false sharing
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarAmir Vadai <amirv@mellanox.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 29d40c90
...@@ -387,6 +387,8 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev, ...@@ -387,6 +387,8 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev,
u64 timestamp = 0; u64 timestamp = 0;
int done = 0; int done = 0;
int budget = priv->tx_work_limit; int budget = priv->tx_work_limit;
u32 last_nr_txbb;
u32 ring_cons;
if (!priv->port_up) if (!priv->port_up)
return true; return true;
...@@ -394,7 +396,9 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev, ...@@ -394,7 +396,9 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev,
prefetchw(&ring->tx_queue->dql.limit); prefetchw(&ring->tx_queue->dql.limit);
index = cons_index & size_mask; index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor; cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
ring_index = ring->cons & size_mask; last_nr_txbb = ACCESS_ONCE(ring->last_nr_txbb);
ring_cons = ACCESS_ONCE(ring->cons);
ring_index = ring_cons & size_mask;
stamp_index = ring_index; stamp_index = ring_index;
/* Process all completed CQEs */ /* Process all completed CQEs */
...@@ -419,19 +423,19 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev, ...@@ -419,19 +423,19 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev,
new_index = be16_to_cpu(cqe->wqe_index) & size_mask; new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do { do {
txbbs_skipped += ring->last_nr_txbb; txbbs_skipped += last_nr_txbb;
ring_index = (ring_index + ring->last_nr_txbb) & size_mask; ring_index = (ring_index + last_nr_txbb) & size_mask;
if (ring->tx_info[ring_index].ts_requested) if (ring->tx_info[ring_index].ts_requested)
timestamp = mlx4_en_get_cqe_ts(cqe); timestamp = mlx4_en_get_cqe_ts(cqe);
/* free next descriptor */ /* free next descriptor */
ring->last_nr_txbb = mlx4_en_free_tx_desc( last_nr_txbb = mlx4_en_free_tx_desc(
priv, ring, ring_index, priv, ring, ring_index,
!!((ring->cons + txbbs_skipped) & !!((ring_cons + txbbs_skipped) &
ring->size), timestamp); ring->size), timestamp);
mlx4_en_stamp_wqe(priv, ring, stamp_index, mlx4_en_stamp_wqe(priv, ring, stamp_index,
!!((ring->cons + txbbs_stamp) & !!((ring_cons + txbbs_stamp) &
ring->size)); ring->size));
stamp_index = ring_index; stamp_index = ring_index;
txbbs_stamp = txbbs_skipped; txbbs_stamp = txbbs_skipped;
...@@ -452,7 +456,11 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev, ...@@ -452,7 +456,11 @@ static bool mlx4_en_process_tx_cq(struct net_device *dev,
mcq->cons_index = cons_index; mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq); mlx4_cq_set_ci(mcq);
wmb(); wmb();
ring->cons += txbbs_skipped;
/* we want to dirty this cache line once */
ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb;
ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped;
netdev_tx_completed_queue(ring->tx_queue, packets, bytes); netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment