Skip to content

L
linux
Commit 5fd11085 authored by James Smart's avatar James Smart Committed by Martin K. Petersen
Browse files
Options

scsi: lpfc: Streamline NVME Initiator WQE setup 

To reduce latency when initializing WQE content, create templates for the
most common wqes. This reduces the number of operations taken to set the
content. It's not a lot of speed up, but every bit helps.

This patch updates the NVME initiator path.

[mkp: fixed typo]
Signed-off-by: default avatarDick Kennedy <dick.kennedy@broadcom.com>
Signed-off-by: default avatarJames Smart <james.smart@broadcom.com>
Signed-off-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
parent 205e8240
Show whitespace changes
Inline Side-by-side
Showing with 203 additions and 127 deletions
drivers/scsi/lpfc/lpfc_crtn.h
View file @ 5fd11085
......@@ -565,6 +565,7 @@ void lpfc_nvme_mod_param_dep(struct lpfc_hba *phba);
void lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba,
struct lpfc_iocbq *cmdiocb,
struct lpfc_wcqe_complete *abts_cmpl);
void lpfc_nvme_cmd_template(void);
extern int lpfc_enable_nvmet_cnt;
extern unsigned long long lpfc_enable_nvmet[];
extern int lpfc_no_hba_reset_cnt;
......
drivers/scsi/lpfc/lpfc_hw4.h
View file @ 5fd11085
......@@ -4183,6 +4183,7 @@ struct wqe_common {
#define wqe_iod_SHIFT 13
#define wqe_iod_MASK 0x00000001
#define wqe_iod_WORD word10
#define LPFC_WQE_IOD_NONE 0
#define LPFC_WQE_IOD_WRITE 0
#define LPFC_WQE_IOD_READ 1
#define wqe_dbde_SHIFT 14
......
drivers/scsi/lpfc/lpfc_init.c
View file @ 5fd11085
......@@ -12583,6 +12583,7 @@ lpfc_init(void)
fc_release_transport(lpfc_transport_template);
return -ENOMEM;
}
lpfc_nvme_cmd_template();
/* Initialize in case vector mapping is needed */
lpfc_used_cpu = NULL;
......
drivers/scsi/lpfc/lpfc_nvme.c
View file @ 5fd11085
......@@ -65,6 +65,136 @@ lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_nvme_buf *);
static struct nvme_fc_port_template lpfc_nvme_template;
union lpfc_wqe128 lpfc_iread_cmd_template;
union lpfc_wqe128 lpfc_iwrite_cmd_template;
union lpfc_wqe128 lpfc_icmnd_cmd_template;
/* Setup WQE templates for NVME IOs */
void
lpfc_nvme_cmd_template()
{
union lpfc_wqe128 *wqe;
/* IREAD template */
wqe = &lpfc_iread_cmd_template;
memset(wqe, 0, sizeof(union lpfc_wqe128));
/* Word 0, 1, 2 - BDE is variable */
/* Word 3 - cmd_buff_len, payload_offset_len is zero */
/* Word 4 - total_xfer_len is variable */
/* Word 5 - is zero */
/* Word 6 - ctxt_tag, xri_tag is variable */
/* Word 7 */
bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE);
bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK);
bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3);
bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI);
/* Word 8 - abort_tag is variable */
/* Word 9 - reqtag is variable */
/* Word 10 - dbde, wqes is variable */
bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1);
bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4);
bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
/* Word 11 - pbde is variable */
bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD);
bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1);
/* Word 12 - is zero */
/* Word 13, 14, 15 - PBDE is variable */
/* IWRITE template */
wqe = &lpfc_iwrite_cmd_template;
memset(wqe, 0, sizeof(union lpfc_wqe128));
/* Word 0, 1, 2 - BDE is variable */
/* Word 3 - cmd_buff_len, payload_offset_len is zero */
/* Word 4 - total_xfer_len is variable */
/* Word 5 - initial_xfer_len is variable */
/* Word 6 - ctxt_tag, xri_tag is variable */
/* Word 7 */
bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE);
bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK);
bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3);
bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI);
/* Word 8 - abort_tag is variable */
/* Word 9 - reqtag is variable */
/* Word 10 - dbde, wqes is variable */
bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1);
bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4);
bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
/* Word 11 - pbde is variable */
bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD);
bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1);
/* Word 12 - is zero */
/* Word 13, 14, 15 - PBDE is variable */
/* ICMND template */
wqe = &lpfc_icmnd_cmd_template;
memset(wqe, 0, sizeof(union lpfc_wqe128));
/* Word 0, 1, 2 - BDE is variable */
/* Word 3 - payload_offset_len is variable */
/* Word 4, 5 - is zero */
/* Word 6 - ctxt_tag, xri_tag is variable */
/* Word 7 */
bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE);
bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3);
bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI);
/* Word 8 - abort_tag is variable */
/* Word 9 - reqtag is variable */
/* Word 10 - dbde, wqes is variable */
bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1);
bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE);
bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE);
bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
/* Word 11 */
bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND);
bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0);
/* Word 12, 13, 14, 15 - is zero */
}
/**
* lpfc_nvme_create_queue -
* @lpfc_pnvme: Pointer to the driver's nvme instance data
......@@ -612,7 +742,7 @@ lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
}
/* Fix up the existing sgls for NVME IO. */
static void
static inline void
lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
struct lpfc_nvme_buf *lpfc_ncmd,
struct nvmefc_fcp_req *nCmd)
......@@ -648,44 +778,8 @@ lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
wqe->generic.bde.tus.f.bdeSize = 56;
wqe->generic.bde.addrHigh = 0;
wqe->generic.bde.addrLow = 64; /* Word 16 */
} else {
sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
/* Word 0-2 - NVME CMND IU Inline BDE */
wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
wqe->generic.bde.addrHigh = sgl->addr_hi;
wqe->generic.bde.addrLow = sgl->addr_lo;
}
sgl++;
/* Setup the physical region for the FCP RSP */
sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
sgl->word2 = le32_to_cpu(sgl->word2);
if (nCmd->sg_cnt)
bf_set(lpfc_sli4_sge_last, sgl, 0);
else
bf_set(lpfc_sli4_sge_last, sgl, 1);
sgl->word2 = cpu_to_le32(sgl->word2);
sgl->sge_len = cpu_to_le32(nCmd->rsplen);
/* Word 3 */
bf_set(payload_offset_len, &wqe->fcp_icmd,
(nCmd->rsplen + nCmd->cmdlen));
/* Word 10 */
bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1);
if (!phba->cfg_nvme_embed_cmd) {
bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 0);
return;
}
bf_set(wqe_dbde, &wqe->generic.wqe_com, 0);
bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
/* Word 10 - dbde is 0, wqes is 1 in template */
/*
* Embed the payload in the last half of the WQE
......@@ -715,6 +809,33 @@ lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
*wptr++ = *dptr++; /* Word 21 */
*wptr++ = *dptr++; /* Word 22 */
*wptr = *dptr; /* Word 23 */
} else {
sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
/* Word 0-2 - NVME CMND IU Inline BDE */
wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
wqe->generic.bde.addrHigh = sgl->addr_hi;
wqe->generic.bde.addrLow = sgl->addr_lo;
/* Word 10 */
bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
}
sgl++;
/* Setup the physical region for the FCP RSP */
sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
sgl->word2 = le32_to_cpu(sgl->word2);
if (nCmd->sg_cnt)
bf_set(lpfc_sli4_sge_last, sgl, 0);
else
bf_set(lpfc_sli4_sge_last, sgl, 1);
sgl->word2 = cpu_to_le32(sgl->word2);
sgl->sge_len = cpu_to_le32(nCmd->rsplen);
}
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
......@@ -1057,9 +1178,16 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
* There are three possibilities here - use scatter-gather segment, use
* the single mapping, or neither.
*/
wqe->fcp_iwrite.initial_xfer_len = 0;
if (nCmd->sg_cnt) {
if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
/* From the iwrite template, initialize words 7 - 11 */
memcpy(&wqe->words[7],
&lpfc_iwrite_cmd_template.words[7],
sizeof(uint32_t) * 5);
/* Word 4 */
wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
/* Word 5 */
if ((phba->cfg_nvme_enable_fb) &&
(pnode->nlp_flag & NLP_FIRSTBURST)) {
......@@ -1070,69 +1198,28 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
else
wqe->fcp_iwrite.initial_xfer_len =
pnode->nvme_fb_size;
} else {
wqe->fcp_iwrite.initial_xfer_len = 0;
}
/* Word 7 */
bf_set(wqe_cmnd, &wqe->generic.wqe_com,
CMD_FCP_IWRITE64_WQE);
bf_set(wqe_pu, &wqe->generic.wqe_com,
PARM_READ_CHECK);
/* Word 10 */
bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com,
LPFC_WQE_IOD_WRITE);
bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com,
LPFC_WQE_LENLOC_WORD4);
if (phba->cfg_nvme_oas)
bf_set(wqe_oas, &wqe->fcp_iwrite.wqe_com, 1);
/* Word 11 */
bf_set(wqe_cmd_type, &wqe->generic.wqe_com,
NVME_WRITE_CMD);
atomic_inc(&phba->fc4NvmeOutputRequests);
} else {
/* Word 7 */
bf_set(wqe_cmnd, &wqe->generic.wqe_com,
CMD_FCP_IREAD64_WQE);
bf_set(wqe_pu, &wqe->generic.wqe_com,
PARM_READ_CHECK);
/* From the iread template, initialize words 7 - 11 */
memcpy(&wqe->words[7],
&lpfc_iread_cmd_template.words[7],
sizeof(uint32_t) * 5);
/* Word 10 */
bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
LPFC_WQE_IOD_READ);
bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com,
LPFC_WQE_LENLOC_WORD4);
if (phba->cfg_nvme_oas)
bf_set(wqe_oas, &wqe->fcp_iread.wqe_com, 1);
/* Word 4 */
wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
/* Word 11 */
bf_set(wqe_cmd_type, &wqe->generic.wqe_com,
NVME_READ_CMD);
/* Word 5 */
wqe->fcp_iread.rsrvd5 = 0;
atomic_inc(&phba->fc4NvmeInputRequests);
}
} else {
/* Word 4 */
wqe->fcp_icmd.rsrvd4 = 0;
/* Word 7 */
bf_set(wqe_cmnd, &wqe->generic.wqe_com, CMD_FCP_ICMND64_WQE);
bf_set(wqe_pu, &wqe->generic.wqe_com, 0);
/* Word 10 */
bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_WRITE);
bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com,
LPFC_WQE_LENLOC_NONE);
if (phba->cfg_nvme_oas)
bf_set(wqe_oas, &wqe->fcp_icmd.wqe_com, 1);
/* Word 11 */
bf_set(wqe_cmd_type, &wqe->generic.wqe_com, NVME_READ_CMD);
/* From the icmnd template, initialize words 4 - 11 */
memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
sizeof(uint32_t) * 8);
atomic_inc(&phba->fc4NvmeControlRequests);
}
/*
......@@ -1140,25 +1227,21 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
* of the nvme_cmnd request_buffer
*/
/* Word 3 */
bf_set(payload_offset_len, &wqe->fcp_icmd,
(nCmd->rsplen + nCmd->cmdlen));
/* Word 6 */
bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
/* Word 7 */
/* Preserve Class data in the ndlp. */
bf_set(wqe_class, &wqe->generic.wqe_com,
(pnode->nlp_fcp_info & 0x0f));
/* Word 8 */
wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
/* Word 9 */
bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
/* Word 11 */
bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
pwqeq->vport = vport;
return 0;
}
......@@ -1269,12 +1352,14 @@ lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
le32_to_cpu(first_data_sgl->sge_len);
bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
bde->tus.w = cpu_to_le32(bde->tus.w);
bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
} else
/* wqe_pbde is 1 in template */
} else {
memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
bf_set(wqe_pbde, &wqe->generic.wqe_com, 0);
}
} else {
bf_set(wqe_pbde, &wqe->generic.wqe_com, 0);
memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
/* For this clause to be valid, the payload_length
* and sg_cnt must zero.
......@@ -1287,12 +1372,6 @@ lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
return 1;
}
}
/*
* Due to difference in data length between DIF/non-DIF paths,
* we need to set word 4 of WQE here
*/
wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
return 0;
}
......@@ -2175,14 +2254,8 @@ lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc)
lpfc_ncmd->cur_iocbq.context1 = lpfc_ncmd;
/* Word 7 */
bf_set(wqe_erp, &wqe->generic.wqe_com, 0);
/* NVME upper layers will time things out, if needed */
bf_set(wqe_tmo, &wqe->generic.wqe_com, 0);
/* Word 10 */
bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0);
bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
/* Initialize WQE */
memset(wqe, 0, sizeof(union lpfc_wqe));
/* add the nvme buffer to a post list */
list_add_tail(&lpfc_ncmd->list, &post_nblist);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7