Commit 57de5a0a authored by Sagi Grimberg's avatar Sagi Grimberg

nvme-rdma: Fix device removal handling

Device removal sequence may have crashed because the
controller (and admin queue space) was freed before
we destroyed the admin queue resources. Thus we
want to destroy the admin queue and only then queue
controller deletion and wait for it to complete.

More specifically we:
1. own the controller deletion (make sure we are not
   competing with another deletion).
2. get rid of inflight reconnects if exists (which
   also destroy and create queues).
3. destroy the queue.
4. safely queue controller deletion (and wait for it
   to complete).
Reported-by: default avatarSteve Wise <swise@opengridcomputing.com>
Signed-off-by: default avatarSagi Grimberg <sagi@grimberg.me>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
parent 5f372eb3
...@@ -169,7 +169,6 @@ MODULE_PARM_DESC(register_always, ...@@ -169,7 +169,6 @@ MODULE_PARM_DESC(register_always,
static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id, static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
struct rdma_cm_event *event); struct rdma_cm_event *event);
static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc); static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
static int __nvme_rdma_del_ctrl(struct nvme_rdma_ctrl *ctrl);
/* XXX: really should move to a generic header sooner or later.. */ /* XXX: really should move to a generic header sooner or later.. */
static inline void put_unaligned_le24(u32 val, u8 *p) static inline void put_unaligned_le24(u32 val, u8 *p)
...@@ -1320,37 +1319,39 @@ static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue) ...@@ -1320,37 +1319,39 @@ static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
* that caught the event. Since we hold the callout until the controller * that caught the event. Since we hold the callout until the controller
* deletion is completed, we'll deadlock if the controller deletion will * deletion is completed, we'll deadlock if the controller deletion will
* call rdma_destroy_id on this queue's cm_id. Thus, we claim ownership * call rdma_destroy_id on this queue's cm_id. Thus, we claim ownership
* of destroying this queue before-hand, destroy the queue resources * of destroying this queue before-hand, destroy the queue resources,
* after the controller deletion completed with the exception of destroying * then queue the controller deletion which won't destroy this queue and
* the cm_id implicitely by returning a non-zero rc to the callout. * we destroy the cm_id implicitely by returning a non-zero rc to the callout.
*/ */
static int nvme_rdma_device_unplug(struct nvme_rdma_queue *queue) static int nvme_rdma_device_unplug(struct nvme_rdma_queue *queue)
{ {
struct nvme_rdma_ctrl *ctrl = queue->ctrl; struct nvme_rdma_ctrl *ctrl = queue->ctrl;
int ret, ctrl_deleted = 0; int ret;
/* First disable the queue so ctrl delete won't free it */ /* Own the controller deletion */
if (!test_and_clear_bit(NVME_RDMA_Q_CONNECTED, &queue->flags)) if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING))
goto out; return 0;
/* delete the controller */ dev_warn(ctrl->ctrl.device,
ret = __nvme_rdma_del_ctrl(ctrl); "Got rdma device removal event, deleting ctrl\n");
if (!ret) {
dev_warn(ctrl->ctrl.device,
"Got rdma device removal event, deleting ctrl\n");
flush_work(&ctrl->delete_work);
/* Return non-zero so the cm_id will destroy implicitly */ /* Get rid of reconnect work if its running */
ctrl_deleted = 1; cancel_delayed_work_sync(&ctrl->reconnect_work);
/* Disable the queue so ctrl delete won't free it */
if (test_and_clear_bit(NVME_RDMA_Q_CONNECTED, &queue->flags)) {
/* Free this queue ourselves */ /* Free this queue ourselves */
rdma_disconnect(queue->cm_id); nvme_rdma_stop_queue(queue);
ib_drain_qp(queue->qp);
nvme_rdma_destroy_queue_ib(queue); nvme_rdma_destroy_queue_ib(queue);
/* Return non-zero so the cm_id will destroy implicitly */
ret = 1;
} }
out: /* Queue controller deletion */
return ctrl_deleted; queue_work(nvme_rdma_wq, &ctrl->delete_work);
flush_work(&ctrl->delete_work);
return ret;
} }
static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id, static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment