Commit 2eacc23c authored by Yuval Atias's avatar Yuval Atias Committed by David S. Miller

net/mlx4_core: Enforce irq affinity changes immediatly

During heavy traffic, napi is constatntly polling the complition queue
and no interrupt is fired. Because of that, changes to irq affinity are
ignored until traffic is stopped and resumed.

By registering to the irq notifier mechanism, and forcing interrupt when
affinity is changed, irq affinity changes will be immediatly enforced.
Signed-off-by: default avatarYuval Atias <yuvala@mellanox.com>
Signed-off-by: default avatarAmir Vadai <amirv@mellanox.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 3763e7ef
...@@ -293,6 +293,9 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, ...@@ -293,6 +293,9 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
atomic_set(&cq->refcount, 1); atomic_set(&cq->refcount, 1);
init_completion(&cq->free); init_completion(&cq->free);
cq->irq = priv->eq_table.eq[cq->vector].irq;
cq->irq_affinity_change = false;
return 0; return 0;
err_radix: err_radix:
......
...@@ -895,10 +895,17 @@ int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget) ...@@ -895,10 +895,17 @@ int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
mlx4_en_cq_unlock_napi(cq); mlx4_en_cq_unlock_napi(cq);
/* If we used up all the quota - we're probably not done yet... */ /* If we used up all the quota - we're probably not done yet... */
if (done == budget) if (done == budget) {
INC_PERF_COUNTER(priv->pstats.napi_quota); INC_PERF_COUNTER(priv->pstats.napi_quota);
else { if (unlikely(cq->mcq.irq_affinity_change)) {
cq->mcq.irq_affinity_change = false;
napi_complete(napi);
mlx4_en_arm_cq(priv, cq);
return 0;
}
} else {
/* Done for now */ /* Done for now */
cq->mcq.irq_affinity_change = false;
napi_complete(napi); napi_complete(napi);
mlx4_en_arm_cq(priv, cq); mlx4_en_arm_cq(priv, cq);
} }
......
...@@ -474,9 +474,15 @@ int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget) ...@@ -474,9 +474,15 @@ int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget)
/* If we used up all the quota - we're probably not done yet... */ /* If we used up all the quota - we're probably not done yet... */
if (done < budget) { if (done < budget) {
/* Done for now */ /* Done for now */
cq->mcq.irq_affinity_change = false;
napi_complete(napi); napi_complete(napi);
mlx4_en_arm_cq(priv, cq); mlx4_en_arm_cq(priv, cq);
return done; return done;
} else if (unlikely(cq->mcq.irq_affinity_change)) {
cq->mcq.irq_affinity_change = false;
napi_complete(napi);
mlx4_en_arm_cq(priv, cq);
return 0;
} }
return budget; return budget;
} }
......
...@@ -53,6 +53,11 @@ enum { ...@@ -53,6 +53,11 @@ enum {
MLX4_EQ_ENTRY_SIZE = 0x20 MLX4_EQ_ENTRY_SIZE = 0x20
}; };
struct mlx4_irq_notify {
void *arg;
struct irq_affinity_notify notify;
};
#define MLX4_EQ_STATUS_OK ( 0 << 28) #define MLX4_EQ_STATUS_OK ( 0 << 28)
#define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28) #define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28)
#define MLX4_EQ_OWNER_SW ( 0 << 24) #define MLX4_EQ_OWNER_SW ( 0 << 24)
...@@ -1083,6 +1088,57 @@ static void mlx4_unmap_clr_int(struct mlx4_dev *dev) ...@@ -1083,6 +1088,57 @@ static void mlx4_unmap_clr_int(struct mlx4_dev *dev)
iounmap(priv->clr_base); iounmap(priv->clr_base);
} }
static void mlx4_irq_notifier_notify(struct irq_affinity_notify *notify,
const cpumask_t *mask)
{
struct mlx4_irq_notify *n = container_of(notify,
struct mlx4_irq_notify,
notify);
struct mlx4_priv *priv = (struct mlx4_priv *)n->arg;
struct radix_tree_iter iter;
void **slot;
radix_tree_for_each_slot(slot, &priv->cq_table.tree, &iter, 0) {
struct mlx4_cq *cq = (struct mlx4_cq *)(*slot);
if (cq->irq == notify->irq)
cq->irq_affinity_change = true;
}
}
static void mlx4_release_irq_notifier(struct kref *ref)
{
struct mlx4_irq_notify *n = container_of(ref, struct mlx4_irq_notify,
notify.kref);
kfree(n);
}
static void mlx4_assign_irq_notifier(struct mlx4_priv *priv,
struct mlx4_dev *dev, int irq)
{
struct mlx4_irq_notify *irq_notifier = NULL;
int err = 0;
irq_notifier = kzalloc(sizeof(*irq_notifier), GFP_KERNEL);
if (!irq_notifier) {
mlx4_warn(dev, "Failed to allocate irq notifier. irq %d\n",
irq);
return;
}
irq_notifier->notify.irq = irq;
irq_notifier->notify.notify = mlx4_irq_notifier_notify;
irq_notifier->notify.release = mlx4_release_irq_notifier;
irq_notifier->arg = priv;
err = irq_set_affinity_notifier(irq, &irq_notifier->notify);
if (err) {
kfree(irq_notifier);
irq_notifier = NULL;
mlx4_warn(dev, "Failed to set irq notifier. irq %d\n", irq);
}
}
int mlx4_alloc_eq_table(struct mlx4_dev *dev) int mlx4_alloc_eq_table(struct mlx4_dev *dev)
{ {
struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_priv *priv = mlx4_priv(dev);
...@@ -1353,6 +1409,9 @@ int mlx4_assign_eq(struct mlx4_dev *dev, char *name, struct cpu_rmap *rmap, ...@@ -1353,6 +1409,9 @@ int mlx4_assign_eq(struct mlx4_dev *dev, char *name, struct cpu_rmap *rmap,
continue; continue;
/*we dont want to break here*/ /*we dont want to break here*/
} }
mlx4_assign_irq_notifier(priv, dev,
priv->eq_table.eq[vec].irq);
eq_set_ci(&priv->eq_table.eq[vec], 1); eq_set_ci(&priv->eq_table.eq[vec], 1);
} }
} }
...@@ -1379,6 +1438,9 @@ void mlx4_release_eq(struct mlx4_dev *dev, int vec) ...@@ -1379,6 +1438,9 @@ void mlx4_release_eq(struct mlx4_dev *dev, int vec)
Belonging to a legacy EQ*/ Belonging to a legacy EQ*/
mutex_lock(&priv->msix_ctl.pool_lock); mutex_lock(&priv->msix_ctl.pool_lock);
if (priv->msix_ctl.pool_bm & 1ULL << i) { if (priv->msix_ctl.pool_bm & 1ULL << i) {
irq_set_affinity_notifier(
priv->eq_table.eq[vec].irq,
NULL);
free_irq(priv->eq_table.eq[vec].irq, free_irq(priv->eq_table.eq[vec].irq,
&priv->eq_table.eq[vec]); &priv->eq_table.eq[vec]);
priv->msix_ctl.pool_bm &= ~(1ULL << i); priv->msix_ctl.pool_bm &= ~(1ULL << i);
......
...@@ -577,6 +577,9 @@ struct mlx4_cq { ...@@ -577,6 +577,9 @@ struct mlx4_cq {
u32 cons_index; u32 cons_index;
u16 irq;
bool irq_affinity_change;
__be32 *set_ci_db; __be32 *set_ci_db;
__be32 *arm_db; __be32 *arm_db;
int arm_sn; int arm_sn;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment