Commit 05ac2c0b authored by Mohamad Haj Yahia's avatar Mohamad Haj Yahia Committed by David S. Miller

net/mlx5: Fix race between PCI error handlers and health work

Currently there is a race between the health care work and the kernel
pci error handlers because both of them detect the error, the first one
to be called will do the error handling.
There is a chance that health care will disable the pci after resuming
pci slot.
Also create a separate WQ because now we will have two types of health
works, one for the error detection and one for the recovery.

Fixes: 89d44f0a ('net/mlx5_core: Add pci error handlers to mlx5_core driver')
Signed-off-by: default avatarMohamad Haj Yahia <mohamad@mellanox.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 2241007b
...@@ -64,6 +64,10 @@ enum { ...@@ -64,6 +64,10 @@ enum {
MLX5_NIC_IFC_NO_DRAM_NIC = 2 MLX5_NIC_IFC_NO_DRAM_NIC = 2
}; };
enum {
MLX5_DROP_NEW_HEALTH_WORK,
};
static u8 get_nic_interface(struct mlx5_core_dev *dev) static u8 get_nic_interface(struct mlx5_core_dev *dev)
{ {
return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3; return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;
...@@ -272,7 +276,13 @@ static void poll_health(unsigned long data) ...@@ -272,7 +276,13 @@ static void poll_health(unsigned long data)
if (in_fatal(dev) && !health->sick) { if (in_fatal(dev) && !health->sick) {
health->sick = true; health->sick = true;
print_health_info(dev); print_health_info(dev);
schedule_work(&health->work); spin_lock(&health->wq_lock);
if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
queue_work(health->wq, &health->work);
else
dev_err(&dev->pdev->dev,
"new health works are not permitted at this stage\n");
spin_unlock(&health->wq_lock);
} }
} }
...@@ -282,6 +292,7 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev) ...@@ -282,6 +292,7 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev)
init_timer(&health->timer); init_timer(&health->timer);
health->sick = 0; health->sick = 0;
clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
health->health = &dev->iseg->health; health->health = &dev->iseg->health;
health->health_counter = &dev->iseg->health_counter; health->health_counter = &dev->iseg->health_counter;
...@@ -298,11 +309,21 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev) ...@@ -298,11 +309,21 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev)
del_timer_sync(&health->timer); del_timer_sync(&health->timer);
} }
void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
{
struct mlx5_core_health *health = &dev->priv.health;
spin_lock(&health->wq_lock);
set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
spin_unlock(&health->wq_lock);
cancel_work_sync(&health->work);
}
void mlx5_health_cleanup(struct mlx5_core_dev *dev) void mlx5_health_cleanup(struct mlx5_core_dev *dev)
{ {
struct mlx5_core_health *health = &dev->priv.health; struct mlx5_core_health *health = &dev->priv.health;
flush_work(&health->work); destroy_workqueue(health->wq);
} }
int mlx5_health_init(struct mlx5_core_dev *dev) int mlx5_health_init(struct mlx5_core_dev *dev)
...@@ -317,8 +338,11 @@ int mlx5_health_init(struct mlx5_core_dev *dev) ...@@ -317,8 +338,11 @@ int mlx5_health_init(struct mlx5_core_dev *dev)
strcpy(name, "mlx5_health"); strcpy(name, "mlx5_health");
strcat(name, dev_name(&dev->pdev->dev)); strcat(name, dev_name(&dev->pdev->dev));
health->wq = create_singlethread_workqueue(name);
kfree(name); kfree(name);
if (!health->wq)
return -ENOMEM;
spin_lock_init(&health->wq_lock);
INIT_WORK(&health->work, health_care); INIT_WORK(&health->work, health_care);
return 0; return 0;
......
...@@ -1315,8 +1315,13 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, ...@@ -1315,8 +1315,13 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
dev_info(&pdev->dev, "%s was called\n", __func__); dev_info(&pdev->dev, "%s was called\n", __func__);
mlx5_enter_error_state(dev); mlx5_enter_error_state(dev);
mlx5_unload_one(dev, priv, false); mlx5_unload_one(dev, priv, false);
/* In case of kernel call save the pci state and drain health wq */
if (state) {
pci_save_state(pdev); pci_save_state(pdev);
mlx5_drain_health_wq(dev);
mlx5_pci_disable_device(dev); mlx5_pci_disable_device(dev);
}
return state == pci_channel_io_perm_failure ? return state == pci_channel_io_perm_failure ?
PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
} }
......
...@@ -418,7 +418,10 @@ struct mlx5_core_health { ...@@ -418,7 +418,10 @@ struct mlx5_core_health {
u32 prev; u32 prev;
int miss_counter; int miss_counter;
bool sick; bool sick;
/* wq spinlock to synchronize draining */
spinlock_t wq_lock;
struct workqueue_struct *wq; struct workqueue_struct *wq;
unsigned long flags;
struct work_struct work; struct work_struct work;
}; };
...@@ -778,6 +781,7 @@ void mlx5_health_cleanup(struct mlx5_core_dev *dev); ...@@ -778,6 +781,7 @@ void mlx5_health_cleanup(struct mlx5_core_dev *dev);
int mlx5_health_init(struct mlx5_core_dev *dev); int mlx5_health_init(struct mlx5_core_dev *dev);
void mlx5_start_health_poll(struct mlx5_core_dev *dev); void mlx5_start_health_poll(struct mlx5_core_dev *dev);
void mlx5_stop_health_poll(struct mlx5_core_dev *dev); void mlx5_stop_health_poll(struct mlx5_core_dev *dev);
void mlx5_drain_health_wq(struct mlx5_core_dev *dev);
int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,
struct mlx5_buf *buf, int node); struct mlx5_buf *buf, int node);
int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, struct mlx5_buf *buf); int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, struct mlx5_buf *buf);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment