Skip to content

L
linux
Commit 348bfec2 authored by David S. Miller's avatar David S. Miller
Browse files
Options

Merge branch 'qed-XDP-support' 

Yuval Mintz says:

====================
qed*: Add XDP support

This patch series is intended to add XDP to the qede driver, although
it contains quite a bit of cleanups, refactorings and infrastructure
changes as well.

The content of this series can be roughly divided into:

 - Datapath improvements - mostly focused on having the datapath utilize
parameters which can be more tightly contained in cachelines.
Patches #1, #2, #8, #9 belong to this group.

 - Refactoring - done mostly in favour of XDP. Patches #3, #4, #5, #9.

 - Infrastructure changes - done in favour of XDP. Paches #6 and #7 belong
to this category [#7 being by far the biggest patch in the series].

 - Actual XDP support - last two patches [#10, #11].
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents f54b8cd6 cb6aeb07
Hide whitespace changes
Inline Side-by-side
Showing with 1879 additions and 1291 deletions
drivers/net/ethernet/qlogic/qed/qed.h
View file @ 348bfec2
......@@ -241,15 +241,6 @@ struct qed_hw_info {
enum qed_wol_support b_wol_support;
};
struct qed_hw_cid_data {
u32 cid;
bool b_cid_allocated;
/* Additional identifiers */
u16 opaque_fid;
u8 vport_id;
};
/* maximun size of read/write commands (HW limit) */
#define DMAE_MAX_RW_SIZE 0x2000
......@@ -416,9 +407,6 @@ struct qed_hwfn {
struct qed_dcbx_info *p_dcbx_info;
struct qed_hw_cid_data *p_tx_cids;
struct qed_hw_cid_data *p_rx_cids;
struct qed_dmae_info dmae_info;
/* QM init */
......
drivers/net/ethernet/qlogic/qed/qed_dev.c
View file @ 348bfec2
......@@ -134,15 +134,6 @@ void qed_resc_free(struct qed_dev *cdev)
kfree(cdev->reset_stats);
for_each_hwfn(cdev, i) {
struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
kfree(p_hwfn->p_tx_cids);
p_hwfn->p_tx_cids = NULL;
kfree(p_hwfn->p_rx_cids);
p_hwfn->p_rx_cids = NULL;
}
for_each_hwfn(cdev, i) {
struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
......@@ -425,23 +416,6 @@ int qed_resc_alloc(struct qed_dev *cdev)
if (!cdev->fw_data)
return -ENOMEM;
/* Allocate Memory for the Queue->CID mapping */
for_each_hwfn(cdev, i) {
struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
int tx_size = sizeof(struct qed_hw_cid_data) *
RESC_NUM(p_hwfn, QED_L2_QUEUE);
int rx_size = sizeof(struct qed_hw_cid_data) *
RESC_NUM(p_hwfn, QED_L2_QUEUE);
p_hwfn->p_tx_cids = kzalloc(tx_size, GFP_KERNEL);
if (!p_hwfn->p_tx_cids)
goto alloc_no_mem;
p_hwfn->p_rx_cids = kzalloc(rx_size, GFP_KERNEL);
if (!p_hwfn->p_rx_cids)
goto alloc_no_mem;
}
for_each_hwfn(cdev, i) {
struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
u32 n_eqes, num_cons;
......@@ -2283,12 +2257,12 @@ static void qed_chain_free_pbl(struct qed_dev *cdev, struct qed_chain *p_chain)
{
void **pp_virt_addr_tbl = p_chain->pbl.pp_virt_addr_tbl;
u32 page_cnt = p_chain->page_cnt, i, pbl_size;
u8 *p_pbl_virt = p_chain->pbl.p_virt_table;
u8 *p_pbl_virt = p_chain->pbl_sp.p_virt_table;
if (!pp_virt_addr_tbl)
return;
if (!p_chain->pbl.p_virt_table)
if (!p_pbl_virt)
goto out;
for (i = 0; i < page_cnt; i++) {
......@@ -2306,7 +2280,8 @@ static void qed_chain_free_pbl(struct qed_dev *cdev, struct qed_chain *p_chain)
pbl_size = page_cnt * QED_CHAIN_PBL_ENTRY_SIZE;
dma_free_coherent(&cdev->pdev->dev,
pbl_size,
p_chain->pbl.p_virt_table, p_chain->pbl.p_phys_table);
p_chain->pbl_sp.p_virt_table,
p_chain->pbl_sp.p_phys_table);
out:
vfree(p_chain->pbl.pp_virt_addr_tbl);
}
......
drivers/net/ethernet/qlogic/qed/qed_l2.c
View file @ 348bfec2
......@@ -23,6 +23,7 @@
#include <linux/workqueue.h>
#include <linux/bitops.h>
#include <linux/bug.h>
#include <linux/vmalloc.h>
#include "qed.h"
#include <linux/qed/qed_chain.h>
#include "qed_cxt.h"
......@@ -41,6 +42,124 @@
#define QED_MAX_SGES_NUM 16
#define CRC32_POLY 0x1edc6f41
void qed_eth_queue_cid_release(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid)
{
/* VFs' CIDs are 0-based in PF-view, and uninitialized on VF */
if (!p_cid->is_vf && IS_PF(p_hwfn->cdev))
qed_cxt_release_cid(p_hwfn, p_cid->cid);
vfree(p_cid);
}
/* The internal is only meant to be directly called by PFs initializeing CIDs
* for their VFs.
*/
struct qed_queue_cid *
_qed_eth_queue_to_cid(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
u32 cid,
u8 vf_qid,
struct qed_queue_start_common_params *p_params)
{
bool b_is_same = (p_hwfn->hw_info.opaque_fid == opaque_fid);
struct qed_queue_cid *p_cid;
int rc;
p_cid = vmalloc(sizeof(*p_cid));
if (!p_cid)
return NULL;
memset(p_cid, 0, sizeof(*p_cid));
p_cid->opaque_fid = opaque_fid;
p_cid->cid = cid;
p_cid->vf_qid = vf_qid;
p_cid->rel = *p_params;
/* Don't try calculating the absolute indices for VFs */
if (IS_VF(p_hwfn->cdev)) {
p_cid->abs = p_cid->rel;
goto out;
}
/* Calculate the engine-absolute indices of the resources.
* This would guarantee they're valid later on.
* In some cases [SBs] we already have the right values.
*/
rc = qed_fw_vport(p_hwfn, p_cid->rel.vport_id, &p_cid->abs.vport_id);
if (rc)
goto fail;
rc = qed_fw_l2_queue(p_hwfn, p_cid->rel.queue_id, &p_cid->abs.queue_id);
if (rc)
goto fail;
/* In case of a PF configuring its VF's queues, the stats-id is already
* absolute [since there's a single index that's suitable per-VF].
*/
if (b_is_same) {
rc = qed_fw_vport(p_hwfn, p_cid->rel.stats_id,
&p_cid->abs.stats_id);
if (rc)
goto fail;
} else {
p_cid->abs.stats_id = p_cid->rel.stats_id;
}
/* SBs relevant information was already provided as absolute */
p_cid->abs.sb = p_cid->rel.sb;
p_cid->abs.sb_idx = p_cid->rel.sb_idx;
/* This is tricky - we're actually interested in whehter this is a PF
* entry meant for the VF.
*/
if (!b_is_same)
p_cid->is_vf = true;
out:
DP_VERBOSE(p_hwfn,
QED_MSG_SP,
"opaque_fid: %04x CID %08x vport %02x [%02x] qzone %04x [%04x] stats %02x [%02x] SB %04x PI %02x\n",
p_cid->opaque_fid,
p_cid->cid,
p_cid->rel.vport_id,
p_cid->abs.vport_id,
p_cid->rel.queue_id,
p_cid->abs.queue_id,
p_cid->rel.stats_id,
p_cid->abs.stats_id, p_cid->abs.sb, p_cid->abs.sb_idx);
return p_cid;
fail:
vfree(p_cid);
return NULL;
}
static struct qed_queue_cid *qed_eth_queue_to_cid(struct qed_hwfn *p_hwfn,
u16 opaque_fid, struct
qed_queue_start_common_params
*p_params)
{
struct qed_queue_cid *p_cid;
u32 cid = 0;
/* Get a unique firmware CID for this queue, in case it's a PF.
* VF's don't need a CID as the queue configuration will be done
* by PF.
*/
if (IS_PF(p_hwfn->cdev)) {
if (qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH, &cid)) {
DP_NOTICE(p_hwfn, "Failed to acquire cid\n");
return NULL;
}
}
p_cid = _qed_eth_queue_to_cid(p_hwfn, opaque_fid, cid, 0, p_params);
if (!p_cid && IS_PF(p_hwfn->cdev))
qed_cxt_release_cid(p_hwfn, cid);
return p_cid;
}
int qed_sp_eth_vport_start(struct qed_hwfn *p_hwfn,
struct qed_sp_vport_start_params *p_params)
{
......@@ -496,61 +615,26 @@ static int qed_filter_accept_cmd(struct qed_dev *cdev,
return 0;
}
static int qed_sp_release_queue_cid(
struct qed_hwfn *p_hwfn,
struct qed_hw_cid_data *p_cid_data)
{
if (!p_cid_data->b_cid_allocated)
return 0;
qed_cxt_release_cid(p_hwfn, p_cid_data->cid);
p_cid_data->b_cid_allocated = false;
return 0;
}
int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
u32 cid,
struct qed_queue_start_common_params *p_params,
u8 stats_id,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr,
u16 cqe_pbl_size, bool b_use_zone_a_prod)
int qed_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr, u16 cqe_pbl_size)
{
struct rx_queue_start_ramrod_data *p_ramrod = NULL;
struct qed_spq_entry *p_ent = NULL;
struct qed_sp_init_data init_data;
struct qed_hw_cid_data *p_rx_cid;
u16 abs_rx_q_id = 0;
u8 abs_vport_id = 0;
int rc = -EINVAL;
/* Store information for the stop */
p_rx_cid = &p_hwfn->p_rx_cids[p_params->queue_id];
p_rx_cid->cid = cid;
p_rx_cid->opaque_fid = opaque_fid;
p_rx_cid->vport_id = p_params->vport_id;
rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_vport_id);
if (rc)
return rc;
rc = qed_fw_l2_queue(p_hwfn, p_params->queue_id, &abs_rx_q_id);
if (rc)
return rc;
DP_VERBOSE(p_hwfn, QED_MSG_SP,
"opaque_fid=0x%x, cid=0x%x, rx_qid=0x%x, vport_id=0x%x, sb_id=0x%x\n",
opaque_fid,
cid, p_params->queue_id, p_params->vport_id, p_params->sb);
"opaque_fid=0x%x, cid=0x%x, rx_qzone=0x%x, vport_id=0x%x, sb_id=0x%x\n",
p_cid->opaque_fid, p_cid->cid,
p_cid->abs.queue_id, p_cid->abs.vport_id, p_cid->abs.sb);
/* Get SPQ entry */
memset(&init_data, 0, sizeof(init_data));
init_data.cid = cid;
init_data.opaque_fid = opaque_fid;
init_data.cid = p_cid->cid;
init_data.opaque_fid = p_cid->opaque_fid;
init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
rc = qed_sp_init_request(p_hwfn, &p_ent,
......@@ -561,11 +645,11 @@ int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
p_ramrod = &p_ent->ramrod.rx_queue_start;
p_ramrod->sb_id = cpu_to_le16(p_params->sb);
p_ramrod->sb_index = p_params->sb_idx;
p_ramrod->vport_id = abs_vport_id;
p_ramrod->stats_counter_id = stats_id;
p_ramrod->rx_queue_id = cpu_to_le16(abs_rx_q_id);
p_ramrod->sb_id = cpu_to_le16(p_cid->abs.sb);
p_ramrod->sb_index = p_cid->abs.sb_idx;
p_ramrod->vport_id = p_cid->abs.vport_id;
p_ramrod->stats_counter_id = p_cid->abs.stats_id;
p_ramrod->rx_queue_id = cpu_to_le16(p_cid->abs.queue_id);
p_ramrod->complete_cqe_flg = 0;
p_ramrod->complete_event_flg = 1;
......@@ -575,85 +659,85 @@ int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
p_ramrod->num_of_pbl_pages = cpu_to_le16(cqe_pbl_size);
DMA_REGPAIR_LE(p_ramrod->cqe_pbl_addr, cqe_pbl_addr);
if (p_params->vf_qid || b_use_zone_a_prod) {
p_ramrod->vf_rx_prod_index = p_params->vf_qid;
if (p_cid->is_vf) {
p_ramrod->vf_rx_prod_index = p_cid->vf_qid;
DP_VERBOSE(p_hwfn, QED_MSG_SP,
"Queue%s is meant for VF rxq[%02x]\n",
b_use_zone_a_prod ? " [legacy]" : "",
p_params->vf_qid);
p_ramrod->vf_rx_prod_use_zone_a = b_use_zone_a_prod;
!!p_cid->b_legacy_vf ? " [legacy]" : "",
p_cid->vf_qid);
p_ramrod->vf_rx_prod_use_zone_a = !!p_cid->b_legacy_vf;
}
return qed_spq_post(p_hwfn, p_ent, NULL);
}
static int
qed_sp_eth_rx_queue_start(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
struct qed_queue_start_common_params *p_params,
qed_eth_pf_rx_queue_start(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr,
u16 cqe_pbl_size, void __iomem **pp_prod)
{
struct qed_hw_cid_data *p_rx_cid;
u32 init_prod_val = 0;
u16 abs_l2_queue = 0;
u8 abs_stats_id = 0;
int rc;
if (IS_VF(p_hwfn->cdev)) {
return qed_vf_pf_rxq_start(p_hwfn,
p_params->queue_id,
p_params->sb,
(u8)p_params->sb_idx,
bd_max_bytes,
bd_chain_phys_addr,
cqe_pbl_addr, cqe_pbl_size, pp_prod);
}
rc = qed_fw_l2_queue(p_hwfn, p_params->queue_id, &abs_l2_queue);
if (rc)
return rc;
rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_stats_id);
if (rc)
return rc;
*pp_prod = (u8 __iomem *)p_hwfn->regview +
GTT_BAR0_MAP_REG_MSDM_RAM +
MSTORM_ETH_PF_PRODS_OFFSET(abs_l2_queue);
*pp_prod = p_hwfn->regview +
GTT_BAR0_MAP_REG_MSDM_RAM +
MSTORM_ETH_PF_PRODS_OFFSET(p_cid->abs.queue_id);
/* Init the rcq, rx bd and rx sge (if valid) producers to 0 */
__internal_ram_wr(p_hwfn, *pp_prod, sizeof(u32),
(u32 *)(&init_prod_val));
return qed_eth_rxq_start_ramrod(p_hwfn, p_cid,
bd_max_bytes,
bd_chain_phys_addr,
cqe_pbl_addr, cqe_pbl_size);
}
static int
qed_eth_rx_queue_start(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
struct qed_queue_start_common_params *p_params,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr,
u16 cqe_pbl_size,
struct qed_rxq_start_ret_params *p_ret_params)
{
struct qed_queue_cid *p_cid;
int rc;
/* Allocate a CID for the queue */
p_rx_cid = &p_hwfn->p_rx_cids[p_params->queue_id];
rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH, &p_rx_cid->cid);
if (rc) {
DP_NOTICE(p_hwfn, "Failed to acquire cid\n");
return rc;
}
p_rx_cid->b_cid_allocated = true;
p_cid = qed_eth_queue_to_cid(p_hwfn, opaque_fid, p_params);
if (!p_cid)
return -ENOMEM;
rc = qed_sp_eth_rxq_start_ramrod(p_hwfn,
opaque_fid,
p_rx_cid->cid,
p_params,
abs_stats_id,
if (IS_PF(p_hwfn->cdev)) {
rc = qed_eth_pf_rx_queue_start(p_hwfn, p_cid,
bd_max_bytes,
bd_chain_phys_addr,
cqe_pbl_addr, cqe_pbl_size,
&p_ret_params->p_prod);
} else {
rc = qed_vf_pf_rxq_start(p_hwfn, p_cid,
bd_max_bytes,
bd_chain_phys_addr,
cqe_pbl_addr, cqe_pbl_size, false);
cqe_pbl_addr,
cqe_pbl_size, &p_ret_params->p_prod);
}
/* Provide the caller with a reference to as handler */
if (rc)
qed_sp_release_queue_cid(p_hwfn, p_rx_cid);
qed_eth_queue_cid_release(p_hwfn, p_cid);
else
p_ret_params->p_handle = (void *)p_cid;
return rc;
}
int qed_sp_eth_rx_queues_update(struct qed_hwfn *p_hwfn,
u16 rx_queue_id,
void **pp_rxq_handles,
u8 num_rxqs,
u8 complete_cqe_flg,
u8 complete_event_flg,
......@@ -663,8 +747,7 @@ int qed_sp_eth_rx_queues_update(struct qed_hwfn *p_hwfn,
struct rx_queue_update_ramrod_data *p_ramrod = NULL;
struct qed_spq_entry *p_ent = NULL;
struct qed_sp_init_data init_data;
struct qed_hw_cid_data *p_rx_cid;
u16 qid, abs_rx_q_id = 0;
struct qed_queue_cid *p_cid;
int rc = -EINVAL;
u8 i;
......@@ -673,12 +756,11 @@ int qed_sp_eth_rx_queues_update(struct qed_hwfn *p_hwfn,
init_data.p_comp_data = p_comp_data;
for (i = 0; i < num_rxqs; i++) {
qid = rx_queue_id + i;
p_rx_cid = &p_hwfn->p_rx_cids[qid];
p_cid = ((struct qed_queue_cid **)pp_rxq_handles)[i];
/* Get SPQ entry */
init_data.cid = p_rx_cid->cid;
init_data.opaque_fid = p_rx_cid->opaque_fid;
init_data.cid = p_cid->cid;
init_data.opaque_fid = p_cid->opaque_fid;
rc = qed_sp_init_request(p_hwfn, &p_ent,
ETH_RAMROD_RX_QUEUE_UPDATE,
......@@ -687,10 +769,9 @@ int qed_sp_eth_rx_queues_update(struct qed_hwfn *p_hwfn,
return rc;
p_ramrod = &p_ent->ramrod.rx_queue_update;
p_ramrod->vport_id = p_cid->abs.vport_id;
qed_fw_vport(p_hwfn, p_rx_cid->vport_id, &p_ramrod->vport_id);
qed_fw_l2_queue(p_hwfn, qid, &abs_rx_q_id);
p_ramrod->rx_queue_id = cpu_to_le16(abs_rx_q_id);
p_ramrod->rx_queue_id = cpu_to_le16(p_cid->abs.queue_id);
p_ramrod->complete_cqe_flg = complete_cqe_flg;
p_ramrod->complete_event_flg = complete_event_flg;
......@@ -702,24 +783,19 @@ int qed_sp_eth_rx_queues_update(struct qed_hwfn *p_hwfn,
return rc;
}
int qed_sp_eth_rx_queue_stop(struct qed_hwfn *p_hwfn,
u16 rx_queue_id,
bool eq_completion_only, bool cqe_completion)
static int
qed_eth_pf_rx_queue_stop(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
bool b_eq_completion_only, bool b_cqe_completion)
{
struct qed_hw_cid_data *p_rx_cid = &p_hwfn->p_rx_cids[rx_queue_id];
struct rx_queue_stop_ramrod_data *p_ramrod = NULL;
struct qed_spq_entry *p_ent = NULL;
struct qed_sp_init_data init_data;
u16 abs_rx_q_id = 0;
int rc = -EINVAL;
if (IS_VF(p_hwfn->cdev))
return qed_vf_pf_rxq_stop(p_hwfn, rx_queue_id, cqe_completion);
int rc;
/* Get SPQ entry */
memset(&init_data, 0, sizeof(init_data));
init_data.cid = p_rx_cid->cid;
init_data.opaque_fid = p_rx_cid->opaque_fid;
init_data.cid = p_cid->cid;
init_data.opaque_fid = p_cid->opaque_fid;
init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
rc = qed_sp_init_request(p_hwfn, &p_ent,
......@@ -729,62 +805,53 @@ int qed_sp_eth_rx_queue_stop(struct qed_hwfn *p_hwfn,
return rc;
p_ramrod = &p_ent->ramrod.rx_queue_stop;
qed_fw_vport(p_hwfn, p_rx_cid->vport_id, &p_ramrod->vport_id);
qed_fw_l2_queue(p_hwfn, rx_queue_id, &abs_rx_q_id);
p_ramrod->rx_queue_id = cpu_to_le16(abs_rx_q_id);
p_ramrod->vport_id = p_cid->abs.vport_id;
p_ramrod->rx_queue_id = cpu_to_le16(p_cid->abs.queue_id);
/* Cleaning the queue requires the completion to arrive there.
* In addition, VFs require the answer to come as eqe to PF.
*/
p_ramrod->complete_cqe_flg =
(!!(p_rx_cid->opaque_fid == p_hwfn->hw_info.opaque_fid) &&
!eq_completion_only) || cqe_completion;
p_ramrod->complete_event_flg =
!(p_rx_cid->opaque_fid == p_hwfn->hw_info.opaque_fid) ||
eq_completion_only;
p_ramrod->complete_cqe_flg = (!p_cid->is_vf &&
!b_eq_completion_only) ||
b_cqe_completion;
p_ramrod->complete_event_flg = p_cid->is_vf || b_eq_completion_only;
rc = qed_spq_post(p_hwfn, p_ent, NULL);
if (rc)
return rc;
return qed_spq_post(p_hwfn, p_ent, NULL);
}
int qed_eth_rx_queue_stop(struct qed_hwfn *p_hwfn,
void *p_rxq,
bool eq_completion_only, bool cqe_completion)
{
struct qed_queue_cid *p_cid = (struct qed_queue_cid *)p_rxq;
int rc = -EINVAL;
return qed_sp_release_queue_cid(p_hwfn, p_rx_cid);
if (IS_PF(p_hwfn->cdev))
rc = qed_eth_pf_rx_queue_stop(p_hwfn, p_cid,
eq_completion_only,
cqe_completion);
else
rc = qed_vf_pf_rxq_stop(p_hwfn, p_cid, cqe_completion);
if (!rc)
qed_eth_queue_cid_release(p_hwfn, p_cid);
return rc;
}
int qed_sp_eth_txq_start_ramrod(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
u32 cid,
struct qed_queue_start_common_params *p_params,
u8 stats_id,
dma_addr_t pbl_addr,
u16 pbl_size,
union qed_qm_pq_params *p_pq_params)
int
qed_eth_txq_start_ramrod(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
dma_addr_t pbl_addr, u16 pbl_size, u16 pq_id)
{
struct tx_queue_start_ramrod_data *p_ramrod = NULL;
struct qed_spq_entry *p_ent = NULL;
struct qed_sp_init_data init_data;
struct qed_hw_cid_data *p_tx_cid;
u16 pq_id, abs_tx_q_id = 0;
int rc = -EINVAL;
u8 abs_vport_id;
/* Store information for the stop */
p_tx_cid = &p_hwfn->p_tx_cids[p_params->queue_id];
p_tx_cid->cid = cid;
p_tx_cid->opaque_fid = opaque_fid;
rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_vport_id);
if (rc)
return rc;
rc = qed_fw_l2_queue(p_hwfn, p_params->queue_id, &abs_tx_q_id);
if (rc)
return rc;
/* Get SPQ entry */
memset(&init_data, 0, sizeof(init_data));
init_data.cid = cid;
init_data.opaque_fid = opaque_fid;
init_data.cid = p_cid->cid;
init_data.opaque_fid = p_cid->opaque_fid;
init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
rc = qed_sp_init_request(p_hwfn, &p_ent,
......@@ -794,96 +861,92 @@ int qed_sp_eth_txq_start_ramrod(struct qed_hwfn *p_hwfn,
return rc;
p_ramrod = &p_ent->ramrod.tx_queue_start;
p_ramrod->vport_id = abs_vport_id;
p_ramrod->vport_id = p_cid->abs.vport_id;
p_ramrod->sb_id = cpu_to_le16(p_params->sb);
p_ramrod->sb_index = p_params->sb_idx;
p_ramrod->stats_counter_id = stats_id;
p_ramrod->sb_id = cpu_to_le16(p_cid->abs.sb);
p_ramrod->sb_index = p_cid->abs.sb_idx;
p_ramrod->stats_counter_id = p_cid->abs.stats_id;
p_ramrod->queue_zone_id = cpu_to_le16(abs_tx_q_id);
p_ramrod->queue_zone_id = cpu_to_le16(p_cid->abs.queue_id);
p_ramrod->same_as_last_id = cpu_to_le16(p_cid->abs.queue_id);
p_ramrod->pbl_size = cpu_to_le16(pbl_size);
DMA_REGPAIR_LE(p_ramrod->pbl_base_addr, pbl_addr);
pq_id = qed_get_qm_pq(p_hwfn, PROTOCOLID_ETH, p_pq_params);
p_ramrod->qm_pq_id = cpu_to_le16(pq_id);
return qed_spq_post(p_hwfn, p_ent, NULL);
}
static int
qed_sp_eth_tx_queue_start(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
struct qed_queue_start_common_params *p_params,
qed_eth_pf_tx_queue_start(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
u8 tc,
dma_addr_t pbl_addr,
u16 pbl_size, void __iomem **pp_doorbell)
{
struct qed_hw_cid_data *p_tx_cid;
union qed_qm_pq_params pq_params;
u8 abs_stats_id = 0;
int rc;
if (IS_VF(p_hwfn->cdev)) {
return qed_vf_pf_txq_start(p_hwfn,
p_params->queue_id,
p_params->sb,
p_params->sb_idx,
pbl_addr, pbl_size, pp_doorbell);
}
memset(&pq_params, 0, sizeof(pq_params));
rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_stats_id);
rc = qed_eth_txq_start_ramrod(p_hwfn, p_cid,
pbl_addr, pbl_size,
qed_get_qm_pq(p_hwfn, PROTOCOLID_ETH,
&pq_params));
if (rc)
return rc;
p_tx_cid = &p_hwfn->p_tx_cids[p_params->queue_id];
memset(p_tx_cid, 0, sizeof(*p_tx_cid));
memset(&pq_params, 0, sizeof(pq_params));
/* Provide the caller with the necessary return values */
*pp_doorbell = p_hwfn->doorbells +
qed_db_addr(p_cid->cid, DQ_DEMS_LEGACY);
/* Allocate a CID for the queue */
rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH, &p_tx_cid->cid);
if (rc) {
DP_NOTICE(p_hwfn, "Failed to acquire cid\n");
return rc;
}
p_tx_cid->b_cid_allocated = true;
return 0;
}
DP_VERBOSE(p_hwfn, QED_MSG_SP,
"opaque_fid=0x%x, cid=0x%x, tx_qid=0x%x, vport_id=0x%x, sb_id=0x%x\n",
opaque_fid, p_tx_cid->cid,
p_params->queue_id, p_params->vport_id, p_params->sb);
rc = qed_sp_eth_txq_start_ramrod(p_hwfn,
opaque_fid,
p_tx_cid->cid,
p_params,
abs_stats_id,
pbl_addr,
pbl_size,
&pq_params);
*pp_doorbell = (u8 __iomem *)p_hwfn->doorbells +
qed_db_addr(p_tx_cid->cid, DQ_DEMS_LEGACY);
static int
qed_eth_tx_queue_start(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
struct qed_queue_start_common_params *p_params,
u8 tc,
dma_addr_t pbl_addr,
u16 pbl_size,
struct qed_txq_start_ret_params *p_ret_params)
{
struct qed_queue_cid *p_cid;
int rc;
p_cid = qed_eth_queue_to_cid(p_hwfn, opaque_fid, p_params);
if (!p_cid)
return -EINVAL;
if (IS_PF(p_hwfn->cdev))
rc = qed_eth_pf_tx_queue_start(p_hwfn, p_cid, tc,
pbl_addr, pbl_size,
&p_ret_params->p_doorbell);
else
rc = qed_vf_pf_txq_start(p_hwfn, p_cid,
pbl_addr, pbl_size,
&p_ret_params->p_doorbell);
if (rc)
qed_sp_release_queue_cid(p_hwfn, p_tx_cid);
qed_eth_queue_cid_release(p_hwfn, p_cid);
else
p_ret_params->p_handle = (void *)p_cid;
return rc;
}
int qed_sp_eth_tx_queue_stop(struct qed_hwfn *p_hwfn, u16 tx_queue_id)
static int
qed_eth_pf_tx_queue_stop(struct qed_hwfn *p_hwfn, struct qed_queue_cid *p_cid)
{
struct qed_hw_cid_data *p_tx_cid = &p_hwfn->p_tx_cids[tx_queue_id];
struct qed_spq_entry *p_ent = NULL;
struct qed_sp_init_data init_data;
int rc = -EINVAL;
if (IS_VF(p_hwfn->cdev))
return qed_vf_pf_txq_stop(p_hwfn, tx_queue_id);
int rc;
/* Get SPQ entry */
memset(&init_data, 0, sizeof(init_data));
init_data.cid = p_tx_cid->cid;
init_data.opaque_fid = p_tx_cid->opaque_fid;
init_data.cid = p_cid->cid;
init_data.opaque_fid = p_cid->opaque_fid;
init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
rc = qed_sp_init_request(p_hwfn, &p_ent,
......@@ -892,11 +955,22 @@ int qed_sp_eth_tx_queue_stop(struct qed_hwfn *p_hwfn, u16 tx_queue_id)
if (rc)
return rc;
rc = qed_spq_post(p_hwfn, p_ent, NULL);
if (rc)
return rc;
return qed_spq_post(p_hwfn, p_ent, NULL);
}
int qed_eth_tx_queue_stop(struct qed_hwfn *p_hwfn, void *p_handle)
{
struct qed_queue_cid *p_cid = (struct qed_queue_cid *)p_handle;
int rc;
if (IS_PF(p_hwfn->cdev))
rc = qed_eth_pf_tx_queue_stop(p_hwfn, p_cid);
else
rc = qed_vf_pf_txq_stop(p_hwfn, p_cid);
return qed_sp_release_queue_cid(p_hwfn, p_tx_cid);
if (!rc)
qed_eth_queue_cid_release(p_hwfn, p_cid);
return rc;
}
static enum eth_filter_action qed_filter_action(enum qed_filter_opcode opcode)
......@@ -1880,58 +1954,53 @@ static int qed_update_vport(struct qed_dev *cdev,
}
static int qed_start_rxq(struct qed_dev *cdev,
struct qed_queue_start_common_params *params,
u8 rss_num,
struct qed_queue_start_common_params *p_params,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr,
u16 cqe_pbl_size,
void __iomem **pp_prod)
struct qed_rxq_start_ret_params *ret_params)
{
struct qed_hwfn *p_hwfn;
int rc, hwfn_index;
hwfn_index = params->rss_id % cdev->num_hwfns;
hwfn_index = rss_num % cdev->num_hwfns;
p_hwfn = &cdev->hwfns[hwfn_index];
/* Fix queue ID in 100g mode */
params->queue_id /= cdev->num_hwfns;
rc = qed_sp_eth_rx_queue_start(p_hwfn,
p_hwfn->hw_info.opaque_fid,
params,
bd_max_bytes,
bd_chain_phys_addr,
cqe_pbl_addr,
cqe_pbl_size,
pp_prod);
p_params->queue_id = p_params->queue_id / cdev->num_hwfns;
p_params->stats_id = p_params->vport_id;
rc = qed_eth_rx_queue_start(p_hwfn,
p_hwfn->hw_info.opaque_fid,
p_params,
bd_max_bytes,
bd_chain_phys_addr,
cqe_pbl_addr, cqe_pbl_size, ret_params);
if (rc) {
DP_ERR(cdev, "Failed to start RXQ#%d\n", params->queue_id);
DP_ERR(cdev, "Failed to start RXQ#%d\n", p_params->queue_id);
return rc;
}
DP_VERBOSE(cdev, (QED_MSG_SPQ | NETIF_MSG_IFUP),
"Started RX-Q %d [rss %d] on V-PORT %d and SB %d\n",
params->queue_id, params->rss_id, params->vport_id,
params->sb);
"Started RX-Q %d [rss_num %d] on V-PORT %d and SB %d\n",
p_params->queue_id, rss_num, p_params->vport_id,
p_params->sb);
return 0;
}
static int qed_stop_rxq(struct qed_dev *cdev,
struct qed_stop_rxq_params *params)
static int qed_stop_rxq(struct qed_dev *cdev, u8 rss_id, void *handle)
{
int rc, hwfn_index;
struct qed_hwfn *p_hwfn;
hwfn_index = params->rss_id % cdev->num_hwfns;
p_hwfn = &cdev->hwfns[hwfn_index];
hwfn_index = rss_id % cdev->num_hwfns;
p_hwfn = &cdev->hwfns[hwfn_index];
rc = qed_sp_eth_rx_queue_stop(p_hwfn,
params->rx_queue_id / cdev->num_hwfns,
params->eq_completion_only, false);
rc = qed_eth_rx_queue_stop(p_hwfn, handle, false, false);
if (rc) {
DP_ERR(cdev, "Failed to stop RXQ#%d\n", params->rx_queue_id);
DP_ERR(cdev, "Failed to stop RXQ#%02x\n", rss_id);
return rc;
}
......@@ -1939,26 +2008,24 @@ static int qed_stop_rxq(struct qed_dev *cdev,
}
static int qed_start_txq(struct qed_dev *cdev,
u8 rss_num,
struct qed_queue_start_common_params *p_params,
dma_addr_t pbl_addr,
u16 pbl_size,
void __iomem **pp_doorbell)
struct qed_txq_start_ret_params *ret_params)
{
struct qed_hwfn *p_hwfn;
int rc, hwfn_index;
hwfn_index = p_params->rss_id % cdev->num_hwfns;
p_hwfn = &cdev->hwfns[hwfn_index];
/* Fix queue ID in 100g mode */
p_params->queue_id /= cdev->num_hwfns;
hwfn_index = rss_num % cdev->num_hwfns;
p_hwfn = &cdev->hwfns[hwfn_index];
p_params->queue_id = p_params->queue_id / cdev->num_hwfns;
p_params->stats_id = p_params->vport_id;
rc = qed_sp_eth_tx_queue_start(p_hwfn,
p_hwfn->hw_info.opaque_fid,
p_params,
pbl_addr,
pbl_size,
pp_doorbell);
rc = qed_eth_tx_queue_start(p_hwfn,
p_hwfn->hw_info.opaque_fid,
p_params, 0,
pbl_addr, pbl_size, ret_params);
if (rc) {
DP_ERR(cdev, "Failed to start TXQ#%d\n", p_params->queue_id);
......@@ -1966,8 +2033,8 @@ static int qed_start_txq(struct qed_dev *cdev,
}
DP_VERBOSE(cdev, (QED_MSG_SPQ | NETIF_MSG_IFUP),
"Started TX-Q %d [rss %d] on V-PORT %d and SB %d\n",
p_params->queue_id, p_params->rss_id, p_params->vport_id,
"Started TX-Q %d [rss_num %d] on V-PORT %d and SB %d\n",
p_params->queue_id, rss_num, p_params->vport_id,
p_params->sb);
return 0;
......@@ -1981,19 +2048,17 @@ static int qed_fastpath_stop(struct qed_dev *cdev)
return 0;
}
static int qed_stop_txq(struct qed_dev *cdev,
struct qed_stop_txq_params *params)
static int qed_stop_txq(struct qed_dev *cdev, u8 rss_id, void *handle)
{
struct qed_hwfn *p_hwfn;
int rc, hwfn_index;
hwfn_index = params->rss_id % cdev->num_hwfns;
p_hwfn = &cdev->hwfns[hwfn_index];
hwfn_index = rss_id % cdev->num_hwfns;
p_hwfn = &cdev->hwfns[hwfn_index];
rc = qed_sp_eth_tx_queue_stop(p_hwfn,
params->tx_queue_id / cdev->num_hwfns);
rc = qed_eth_tx_queue_stop(p_hwfn, handle);
if (rc) {
DP_ERR(cdev, "Failed to stop TXQ#%d\n", params->tx_queue_id);
DP_ERR(cdev, "Failed to stop TXQ#%02x\n", rss_id);
return rc;
}
......
drivers/net/ethernet/qlogic/qed/qed_l2.h
View file @ 348bfec2
......@@ -78,11 +78,34 @@ struct qed_filter_mcast {
unsigned char mac[QED_MAX_MC_ADDRS][ETH_ALEN];
};
int qed_sp_eth_rx_queue_stop(struct qed_hwfn *p_hwfn,
u16 rx_queue_id,
bool eq_completion_only, bool cqe_completion);
/**
* @brief qed_eth_rx_queue_stop - This ramrod closes an Rx queue
*
* @param p_hwfn
* @param p_rxq Handler of queue to close
* @param eq_completion_only If True completion will be on
* EQe, if False completion will be
* on EQe if p_hwfn opaque
* different from the RXQ opaque
* otherwise on CQe.
* @param cqe_completion If True completion will be
* receive on CQe.
* @return int
*/
int
qed_eth_rx_queue_stop(struct qed_hwfn *p_hwfn,
void *p_rxq,
bool eq_completion_only, bool cqe_completion);
int qed_sp_eth_tx_queue_stop(struct qed_hwfn *p_hwfn, u16 tx_queue_id);
/**
* @brief qed_eth_tx_queue_stop - closes a Tx queue
*
* @param p_hwfn
* @param p_txq - handle to Tx queue needed to be closed
*
* @return int
*/
int qed_eth_tx_queue_stop(struct qed_hwfn *p_hwfn, void *p_txq);
enum qed_tpa_mode {
QED_TPA_MODE_NONE,
......@@ -196,19 +219,19 @@ int qed_sp_eth_filter_ucast(struct qed_hwfn *p_hwfn,
* @note At the moment - only used by non-linux VFs.
*
* @param p_hwfn
* @param rx_queue_id RX Queue ID
* @param num_rxqs Allow to update multiple rx
* queues, from rx_queue_id to
* (rx_queue_id + num_rxqs)
* @param pp_rxq_handlers An array of queue handlers to be updated.
* @param num_rxqs number of queues to update.
* @param complete_cqe_flg Post completion to the CQE Ring if set
* @param complete_event_flg Post completion to the Event Ring if set
* @param comp_mode
* @param p_comp_data
*
* @return int
*/
int
qed_sp_eth_rx_queues_update(struct qed_hwfn *p_hwfn,
u16 rx_queue_id,
void **pp_rxq_handlers,
u8 num_rxqs,
u8 complete_cqe_flg,
u8 complete_event_flg,
......@@ -217,27 +240,79 @@ qed_sp_eth_rx_queues_update(struct qed_hwfn *p_hwfn,
void qed_get_vport_stats(struct qed_dev *cdev, struct qed_eth_stats *stats);
int qed_sp_eth_vport_start(struct qed_hwfn *p_hwfn,
struct qed_sp_vport_start_params *p_params);
void qed_reset_vport_stats(struct qed_dev *cdev);
struct qed_queue_cid {
/* 'Relative' is a relative term ;-). Usually the indices [not counting
* SBs] would be PF-relative, but there are some cases where that isn't
* the case - specifically for a PF configuring its VF indices it's
* possible some fields [E.g., stats-id] in 'rel' would already be abs.
*/
struct qed_queue_start_common_params rel;
struct qed_queue_start_common_params abs;
u32 cid;
u16 opaque_fid;
/* VFs queues are mapped differently, so we need to know the
* relative queue associated with them [0-based].
* Notice this is relevant on the *PF* queue-cid of its VF's queues,
* and not on the VF itself.
*/
bool is_vf;
u8 vf_qid;
/* Legacy VFs might have Rx producer located elsewhere */
bool b_legacy_vf;
};
int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
u32 cid,
struct qed_queue_start_common_params *params,
u8 stats_id,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr,
u16 cqe_pbl_size, bool b_use_zone_a_prod);
int qed_sp_eth_txq_start_ramrod(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
u32 cid,
struct qed_queue_start_common_params *p_params,
u8 stats_id,
dma_addr_t pbl_addr,
u16 pbl_size,
union qed_qm_pq_params *p_pq_params);
void qed_eth_queue_cid_release(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid);
struct qed_queue_cid *_qed_eth_queue_to_cid(struct qed_hwfn *p_hwfn,
u16 opaque_fid,
u32 cid,
u8 vf_qid,
struct qed_queue_start_common_params
*p_params);
int
qed_sp_eth_vport_start(struct qed_hwfn *p_hwfn,
struct qed_sp_vport_start_params *p_params);
/**
* @brief - Starts an Rx queue, when queue_cid is already prepared
*
* @param p_hwfn
* @param p_cid
* @param bd_max_bytes
* @param bd_chain_phys_addr
* @param cqe_pbl_addr
* @param cqe_pbl_size
*
* @return int
*/
int
qed_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr, u16 cqe_pbl_size);
/**
* @brief - Starts a Tx queue, where queue_cid is already prepared
*
* @param p_hwfn
* @param p_cid
* @param pbl_addr
* @param pbl_size
* @param p_pq_params - parameters for choosing the PQ for this Tx queue
*
* @return int
*/
int
qed_eth_txq_start_ramrod(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
dma_addr_t pbl_addr, u16 pbl_size, u16 pq_id);
u8 qed_mcast_bin_from_mac(u8 *mac);
......
drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
View file @ 348bfec2
......@@ -347,11 +347,11 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
/* Place EQ address in RAMROD */
DMA_REGPAIR_LE(p_ramrod->event_ring_pbl_addr,
p_hwfn->p_eq->chain.pbl.p_phys_table);
p_hwfn->p_eq->chain.pbl_sp.p_phys_table);
page_cnt = (u8)qed_chain_get_page_cnt(&p_hwfn->p_eq->chain);
p_ramrod->event_ring_num_pages = page_cnt;
DMA_REGPAIR_LE(p_ramrod->consolid_q_pbl_addr,
p_hwfn->p_consq->chain.pbl.p_phys_table);
p_hwfn->p_consq->chain.pbl_sp.p_phys_table);
qed_tunn_set_pf_start_params(p_hwfn, p_tunn, &p_ramrod->tunnel_config);
......
drivers/net/ethernet/qlogic/qed/qed_sriov.c
View file @ 348bfec2
......@@ -808,37 +808,70 @@ static void qed_iov_free_vf_igu_sbs(struct qed_hwfn *p_hwfn,
static int qed_iov_init_hw_for_vf(struct qed_hwfn *p_hwfn,
struct qed_ptt *p_ptt,
u16 rel_vf_id, u16 num_rx_queues)
struct qed_iov_vf_init_params *p_params)
{
u8 num_of_vf_avaiable_chains = 0;
struct qed_vf_info *vf = NULL;
u16 qid, num_irqs;
int rc = 0;
u32 cids;
u8 i;
vf = qed_iov_get_vf_info(p_hwfn, rel_vf_id, false);
vf = qed_iov_get_vf_info(p_hwfn, p_params->rel_vf_id, false);
if (!vf) {
DP_ERR(p_hwfn, "qed_iov_init_hw_for_vf : vf is NULL\n");
return -EINVAL;
}
if (vf->b_init) {
DP_NOTICE(p_hwfn, "VF[%d] is already active.\n", rel_vf_id);
DP_NOTICE(p_hwfn, "VF[%d] is already active.\n",
p_params->rel_vf_id);
return -EINVAL;
}
/* Perform sanity checking on the requested queue_id */
for (i = 0; i < p_params->num_queues; i++) {
u16 min_vf_qzone = FEAT_NUM(p_hwfn, QED_PF_L2_QUE);
u16 max_vf_qzone = min_vf_qzone +
FEAT_NUM(p_hwfn, QED_VF_L2_QUE) - 1;
qid = p_params->req_rx_queue[i];
if (qid < min_vf_qzone || qid > max_vf_qzone) {
DP_NOTICE(p_hwfn,
"Can't enable Rx qid [%04x] for VF[%d]: qids [0x%04x,...,0x%04x] available\n",
qid,
p_params->rel_vf_id,
min_vf_qzone, max_vf_qzone);
return -EINVAL;
}
qid = p_params->req_tx_queue[i];
if (qid > max_vf_qzone) {
DP_NOTICE(p_hwfn,
"Can't enable Tx qid [%04x] for VF[%d]: max qid 0x%04x\n",
qid, p_params->rel_vf_id, max_vf_qzone);
return -EINVAL;
}
/* If client *really* wants, Tx qid can be shared with PF */
if (qid < min_vf_qzone)
DP_VERBOSE(p_hwfn,
QED_MSG_IOV,
"VF[%d] is using PF qid [0x%04x] for Txq[0x%02x]\n",
p_params->rel_vf_id, qid, i);
}
/* Limit number of queues according to number of CIDs */
qed_cxt_get_proto_cid_count(p_hwfn, PROTOCOLID_ETH, &cids);
DP_VERBOSE(p_hwfn,
QED_MSG_IOV,
"VF[%d] - requesting to initialize for 0x%04x queues [0x%04x CIDs available]\n",
vf->relative_vf_id, num_rx_queues, (u16) cids);
num_rx_queues = min_t(u16, num_rx_queues, ((u16) cids));
vf->relative_vf_id, p_params->num_queues, (u16)cids);
num_irqs = min_t(u16, p_params->num_queues, ((u16)cids));
num_of_vf_avaiable_chains = qed_iov_alloc_vf_igu_sbs(p_hwfn,
p_ptt,
vf,
num_rx_queues);
vf, num_irqs);
if (!num_of_vf_avaiable_chains) {
DP_ERR(p_hwfn, "no available igu sbs\n");
return -ENOMEM;
......@@ -849,25 +882,22 @@ static int qed_iov_init_hw_for_vf(struct qed_hwfn *p_hwfn,
vf->num_txqs = num_of_vf_avaiable_chains;
for (i = 0; i < vf->num_rxqs; i++) {
u16 queue_id = qed_int_queue_id_from_sb_id(p_hwfn,
vf->igu_sbs[i]);
struct qed_vf_q_info *p_queue = &vf->vf_queues[i];
if (queue_id > RESC_NUM(p_hwfn, QED_L2_QUEUE)) {
DP_NOTICE(p_hwfn,
"VF[%d] will require utilizing of out-of-bounds queues - %04x\n",
vf->relative_vf_id, queue_id);
return -EINVAL;
}
p_queue->fw_rx_qid = p_params->req_rx_queue[i];
p_queue->fw_tx_qid = p_params->req_tx_queue[i];
/* CIDs are per-VF, so no problem having them 0-based. */
vf->vf_queues[i].fw_rx_qid = queue_id;
vf->vf_queues[i].fw_tx_qid = queue_id;
vf->vf_queues[i].fw_cid = i;
p_queue->fw_cid = i;
DP_VERBOSE(p_hwfn, QED_MSG_IOV,
"VF[%d] - [%d] SB %04x, Tx/Rx queue %04x CID %04x\n",
vf->relative_vf_id, i, vf->igu_sbs[i], queue_id, i);
"VF[%d] - Q[%d] SB %04x, qid [Rx %04x Tx %04x] CID %04x\n",
vf->relative_vf_id,
i, vf->igu_sbs[i],
p_queue->fw_rx_qid,
p_queue->fw_tx_qid, p_queue->fw_cid);
}
rc = qed_iov_enable_vf_access(p_hwfn, p_ptt, vf);
if (!rc) {
vf->b_init = true;
......@@ -1187,8 +1217,19 @@ static void qed_iov_vf_cleanup(struct qed_hwfn *p_hwfn,
p_vf->num_active_rxqs = 0;
for (i = 0; i < QED_MAX_VF_CHAINS_PER_PF; i++)
p_vf->vf_queues[i].rxq_active = 0;
for (i = 0; i < QED_MAX_VF_CHAINS_PER_PF; i++) {
struct qed_vf_q_info *p_queue = &p_vf->vf_queues[i];
if (p_queue->p_rx_cid) {
qed_eth_queue_cid_release(p_hwfn, p_queue->p_rx_cid);
p_queue->p_rx_cid = NULL;
}
if (p_queue->p_tx_cid) {
qed_eth_queue_cid_release(p_hwfn, p_queue->p_tx_cid);
p_queue->p_tx_cid = NULL;
}
}
memset(&p_vf->shadow_config, 0, sizeof(p_vf->shadow_config));
memset(&p_vf->acquire, 0, sizeof(p_vf->acquire));
......@@ -1594,21 +1635,21 @@ static int qed_iov_configure_vport_forced(struct qed_hwfn *p_hwfn,
/* Update all the Rx queues */
for (i = 0; i < QED_MAX_VF_CHAINS_PER_PF; i++) {
u16 qid;
struct qed_queue_cid *p_cid;
if (!p_vf->vf_queues[i].rxq_active)
p_cid = p_vf->vf_queues[i].p_rx_cid;
if (!p_cid)
continue;
qid = p_vf->vf_queues[i].fw_rx_qid;
rc = qed_sp_eth_rx_queues_update(p_hwfn, qid,
rc = qed_sp_eth_rx_queues_update(p_hwfn,
(void **)&p_cid,
1, 0, 1,
QED_SPQ_MODE_EBLOCK,
NULL);
if (rc) {
DP_NOTICE(p_hwfn,
"Failed to send Rx update fo queue[0x%04x]\n",
qid);
p_cid->rel.queue_id);
return rc;
}
}
......@@ -1782,23 +1823,34 @@ static void qed_iov_vf_mbx_start_rxq(struct qed_hwfn *p_hwfn,
struct qed_queue_start_common_params params;
struct qed_iov_vf_mbx *mbx = &vf->vf_mbx;
u8 status = PFVF_STATUS_NO_RESOURCE;
struct qed_vf_q_info *p_queue;
struct vfpf_start_rxq_tlv *req;
bool b_legacy_vf = false;
int rc;
memset(&params, 0, sizeof(params));
req = &mbx->req_virt->start_rxq;
if (!qed_iov_validate_rxq(p_hwfn, vf, req->rx_qid) ||
!qed_iov_validate_sb(p_hwfn, vf, req->hw_sb))
goto out;
params.queue_id = vf->vf_queues[req->rx_qid].fw_rx_qid;
params.vf_qid = req->rx_qid;
/* Acquire a new queue-cid */
p_queue = &vf->vf_queues[req->rx_qid];
memset(&params, 0, sizeof(params));
params.queue_id = p_queue->fw_rx_qid;
params.vport_id = vf->vport_id;
params.stats_id = vf->abs_vf_id + 0x10;
params.sb = req->hw_sb;
params.sb_idx = req->sb_index;
p_queue->p_rx_cid = _qed_eth_queue_to_cid(p_hwfn,
vf->opaque_fid,
p_queue->fw_cid,
req->rx_qid, &params);
if (!p_queue->p_rx_cid)
goto out;
/* Legacy VFs have their Producers in a different location, which they
* calculate on their own and clean the producer prior to this.
*/
......@@ -1811,21 +1863,19 @@ static void qed_iov_vf_mbx_start_rxq(struct qed_hwfn *p_hwfn,
MSTORM_ETH_VF_PRODS_OFFSET(vf->abs_vf_id, req->rx_qid),
0);
}
p_queue->p_rx_cid->b_legacy_vf = b_legacy_vf;
rc = qed_sp_eth_rxq_start_ramrod(p_hwfn, vf->opaque_fid,
vf->vf_queues[req->rx_qid].fw_cid,
&params,
vf->abs_vf_id + 0x10,
req->bd_max_bytes,
req->rxq_addr,
req->cqe_pbl_addr, req->cqe_pbl_size,
b_legacy_vf);
rc = qed_eth_rxq_start_ramrod(p_hwfn,
p_queue->p_rx_cid,
req->bd_max_bytes,
req->rxq_addr,
req->cqe_pbl_addr, req->cqe_pbl_size);
if (rc) {
status = PFVF_STATUS_FAILURE;
qed_eth_queue_cid_release(p_hwfn, p_queue->p_rx_cid);
p_queue->p_rx_cid = NULL;
} else {
status = PFVF_STATUS_SUCCESS;
vf->vf_queues[req->rx_qid].rxq_active = true;
vf->num_active_rxqs++;
}
......@@ -1882,7 +1932,9 @@ static void qed_iov_vf_mbx_start_txq(struct qed_hwfn *p_hwfn,
u8 status = PFVF_STATUS_NO_RESOURCE;
union qed_qm_pq_params pq_params;
struct vfpf_start_txq_tlv *req;
struct qed_vf_q_info *p_queue;
int rc;
u16 pq;
/* Prepare the parameters which would choose the right PQ */
memset(&pq_params, 0, sizeof(pq_params));
......@@ -1896,24 +1948,31 @@ static void qed_iov_vf_mbx_start_txq(struct qed_hwfn *p_hwfn,
!qed_iov_validate_sb(p_hwfn, vf, req->hw_sb))
goto out;
params.queue_id = vf->vf_queues[req->tx_qid].fw_tx_qid;
/* Acquire a new queue-cid */
p_queue = &vf->vf_queues[req->tx_qid];
params.queue_id = p_queue->fw_tx_qid;
params.vport_id = vf->vport_id;
params.stats_id = vf->abs_vf_id + 0x10;
params.sb = req->hw_sb;
params.sb_idx = req->sb_index;
rc = qed_sp_eth_txq_start_ramrod(p_hwfn,
vf->opaque_fid,
vf->vf_queues[req->tx_qid].fw_cid,
&params,
vf->abs_vf_id + 0x10,
req->pbl_addr,
req->pbl_size, &pq_params);
p_queue->p_tx_cid = _qed_eth_queue_to_cid(p_hwfn,
vf->opaque_fid,
p_queue->fw_cid,
req->tx_qid, &params);
if (!p_queue->p_tx_cid)
goto out;
pq = qed_get_qm_pq(p_hwfn, PROTOCOLID_ETH, &pq_params);
rc = qed_eth_txq_start_ramrod(p_hwfn, p_queue->p_tx_cid,
req->pbl_addr, req->pbl_size, pq);
if (rc) {
status = PFVF_STATUS_FAILURE;
qed_eth_queue_cid_release(p_hwfn, p_queue->p_tx_cid);
p_queue->p_tx_cid = NULL;
} else {
status = PFVF_STATUS_SUCCESS;
vf->vf_queues[req->tx_qid].txq_active = true;
}
out:
......@@ -1924,6 +1983,7 @@ static int qed_iov_vf_stop_rxqs(struct qed_hwfn *p_hwfn,
struct qed_vf_info *vf,
u16 rxq_id, u8 num_rxqs, bool cqe_completion)
{
struct qed_vf_q_info *p_queue;
int rc = 0;
int qid;
......@@ -1931,16 +1991,18 @@ static int qed_iov_vf_stop_rxqs(struct qed_hwfn *p_hwfn,
return -EINVAL;
for (qid = rxq_id; qid < rxq_id + num_rxqs; qid++) {
if (vf->vf_queues[qid].rxq_active) {
rc = qed_sp_eth_rx_queue_stop(p_hwfn,
vf->vf_queues[qid].
fw_rx_qid, false,
cqe_completion);
p_queue = &vf->vf_queues[qid];
if (rc)
return rc;
}
vf->vf_queues[qid].rxq_active = false;
if (!p_queue->p_rx_cid)
continue;
rc = qed_eth_rx_queue_stop(p_hwfn,
p_queue->p_rx_cid,
false, cqe_completion);
if (rc)
return rc;
vf->vf_queues[qid].p_rx_cid = NULL;
vf->num_active_rxqs--;
}
......@@ -1951,22 +2013,24 @@ static int qed_iov_vf_stop_txqs(struct qed_hwfn *p_hwfn,
struct qed_vf_info *vf, u16 txq_id, u8 num_txqs)
{
int rc = 0;
struct qed_vf_q_info *p_queue;
int qid;
if (txq_id + num_txqs > ARRAY_SIZE(vf->vf_queues))
return -EINVAL;
for (qid = txq_id; qid < txq_id + num_txqs; qid++) {
if (vf->vf_queues[qid].txq_active) {
rc = qed_sp_eth_tx_queue_stop(p_hwfn,
vf->vf_queues[qid].
fw_tx_qid);
p_queue = &vf->vf_queues[qid];
if (!p_queue->p_tx_cid)
continue;
if (rc)
return rc;
}
vf->vf_queues[qid].txq_active = false;
rc = qed_eth_tx_queue_stop(p_hwfn, p_queue->p_tx_cid);
if (rc)
return rc;
p_queue->p_tx_cid = NULL;
}
return rc;
}
......@@ -2021,10 +2085,11 @@ static void qed_iov_vf_mbx_update_rxqs(struct qed_hwfn *p_hwfn,
struct qed_ptt *p_ptt,
struct qed_vf_info *vf)
{
struct qed_queue_cid *handlers[QED_MAX_VF_CHAINS_PER_PF];
u16 length = sizeof(struct pfvf_def_resp_tlv);
struct qed_iov_vf_mbx *mbx = &vf->vf_mbx;
struct vfpf_update_rxq_tlv *req;
u8 status = PFVF_STATUS_SUCCESS;
u8 status = PFVF_STATUS_FAILURE;
u8 complete_event_flg;
u8 complete_cqe_flg;
u16 qid;
......@@ -2035,29 +2100,36 @@ static void qed_iov_vf_mbx_update_rxqs(struct qed_hwfn *p_hwfn,
complete_cqe_flg = !!(req->flags & VFPF_RXQ_UPD_COMPLETE_CQE_FLAG);
complete_event_flg = !!(req->flags & VFPF_RXQ_UPD_COMPLETE_EVENT_FLAG);
/* Validate inputs */
if (req->num_rxqs + req->rx_qid > QED_MAX_VF_CHAINS_PER_PF ||
!qed_iov_validate_rxq(p_hwfn, vf, req->rx_qid)) {
DP_INFO(p_hwfn, "VF[%d]: Incorrect Rxqs [%04x, %02x]\n",
vf->relative_vf_id, req->rx_qid, req->num_rxqs);
goto out;
}
for (i = 0; i < req->num_rxqs; i++) {
qid = req->rx_qid + i;
if (!vf->vf_queues[qid].rxq_active) {
DP_NOTICE(p_hwfn, "VF rx_qid = %d isn`t active!\n",
qid);
status = PFVF_STATUS_FAILURE;
break;
if (!vf->vf_queues[qid].p_rx_cid) {
DP_INFO(p_hwfn,
"VF[%d] rx_qid = %d isn`t active!\n",
vf->relative_vf_id, qid);
goto out;
}
rc = qed_sp_eth_rx_queues_update(p_hwfn,
vf->vf_queues[qid].fw_rx_qid,
1,
complete_cqe_flg,
complete_event_flg,
QED_SPQ_MODE_EBLOCK, NULL);
if (rc) {
status = PFVF_STATUS_FAILURE;
break;
}
handlers[i] = vf->vf_queues[qid].p_rx_cid;
}
rc = qed_sp_eth_rx_queues_update(p_hwfn, (void **)&handlers,
req->num_rxqs,
complete_cqe_flg,
complete_event_flg,
QED_SPQ_MODE_EBLOCK, NULL);
if (rc)
goto out;
status = PFVF_STATUS_SUCCESS;
out:
qed_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_UPDATE_RXQ,
length, status);
}
......@@ -2268,7 +2340,7 @@ qed_iov_vp_update_rss_param(struct qed_hwfn *p_hwfn,
DP_NOTICE(p_hwfn,
"rss_ind_table[%d] = %d, rxq is out of range\n",
i, q_idx);
else if (!vf->vf_queues[q_idx].rxq_active)
else if (!vf->vf_queues[q_idx].p_rx_cid)
DP_NOTICE(p_hwfn,
"rss_ind_table[%d] = %d, rxq is not active\n",
i, q_idx);
......@@ -3468,8 +3540,28 @@ int qed_sriov_disable(struct qed_dev *cdev, bool pci_enabled)
return 0;
}
static void qed_sriov_enable_qid_config(struct qed_hwfn *hwfn,
u16 vfid,
struct qed_iov_vf_init_params *params)
{
u16 base, i;
/* Since we have an equal resource distribution per-VF, and we assume
* PF has acquired the QED_PF_L2_QUE first queues, we start setting
* sequentially from there.
*/
base = FEAT_NUM(hwfn, QED_PF_L2_QUE) + vfid * params->num_queues;
params->rel_vf_id = vfid;
for (i = 0; i < params->num_queues; i++) {
params->req_rx_queue[i] = base + i;
params->req_tx_queue[i] = base + i;
}
}
static int qed_sriov_enable(struct qed_dev *cdev, int num)
{
struct qed_iov_vf_init_params params;
int i, j, rc;
if (num >= RESC_NUM(&cdev->hwfns[0], QED_VPORT)) {
......@@ -3478,15 +3570,17 @@ static int qed_sriov_enable(struct qed_dev *cdev, int num)
return -EINVAL;
}
memset(&params, 0, sizeof(params));
/* Initialize HW for VF access */
for_each_hwfn(cdev, j) {
struct qed_hwfn *hwfn = &cdev->hwfns[j];
struct qed_ptt *ptt = qed_ptt_acquire(hwfn);
int num_queues;
/* Make sure not to use more than 16 queues per VF */
num_queues = min_t(int,
FEAT_NUM(hwfn, QED_VF_L2_QUE) / num, 16);
params.num_queues = min_t(int,
FEAT_NUM(hwfn, QED_VF_L2_QUE) / num,
16);
if (!ptt) {
DP_ERR(hwfn, "Failed to acquire ptt\n");
......@@ -3498,7 +3592,8 @@ static int qed_sriov_enable(struct qed_dev *cdev, int num)
if (!qed_iov_is_valid_vfid(hwfn, i, false, true))
continue;
rc = qed_iov_init_hw_for_vf(hwfn, ptt, i, num_queues);
qed_sriov_enable_qid_config(hwfn, i, &params);
rc = qed_iov_init_hw_for_vf(hwfn, ptt, &params);
if (rc) {
DP_ERR(cdev, "Failed to enable VF[%d]\n", i);
qed_ptt_release(hwfn, ptt);
......
drivers/net/ethernet/qlogic/qed/qed_sriov.h
View file @ 348bfec2
......@@ -58,6 +58,23 @@ struct qed_public_vf_info {
int tx_rate;
};
struct qed_iov_vf_init_params {
u16 rel_vf_id;
/* Number of requested Queues; Currently, don't support different
* number of Rx/Tx queues.
*/
u16 num_queues;
/* Allow the client to choose which qzones to use for Rx/Tx,
* and which queue_base to use for Tx queues on a per-queue basis.
* Notice values should be relative to the PF resources.
*/
u16 req_rx_queue[QED_MAX_VF_CHAINS_PER_PF];
u16 req_tx_queue[QED_MAX_VF_CHAINS_PER_PF];
};
/* This struct is part of qed_dev and contains data relevant to all hwfns;
* Initialized only if SR-IOV cpabability is exposed in PCIe config space.
*/
......@@ -99,10 +116,10 @@ struct qed_iov_vf_mbx {
struct qed_vf_q_info {
u16 fw_rx_qid;
struct qed_queue_cid *p_rx_cid;
u16 fw_tx_qid;
struct qed_queue_cid *p_tx_cid;
u8 fw_cid;
u8 rxq_active;
u8 txq_active;
};
enum vf_state {
......
drivers/net/ethernet/qlogic/qed/qed_vf.c
View file @ 348bfec2
......@@ -388,18 +388,18 @@ int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
#define MSTORM_QZONE_START(dev) (TSTORM_QZONE_START + \
(TSTORM_QZONE_SIZE * NUM_OF_L2_QUEUES(dev)))
int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
u8 rx_qid,
u16 sb,
u8 sb_index,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr,
u16 cqe_pbl_size, void __iomem **pp_prod)
int
qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr,
u16 cqe_pbl_size, void __iomem **pp_prod)
{
struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
struct pfvf_start_queue_resp_tlv *resp;
struct vfpf_start_rxq_tlv *req;
u8 rx_qid = p_cid->rel.queue_id;
int rc;
/* clear mailbox and prep first tlv */
......@@ -409,21 +409,22 @@ int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
req->cqe_pbl_addr = cqe_pbl_addr;
req->cqe_pbl_size = cqe_pbl_size;
req->rxq_addr = bd_chain_phys_addr;
req->hw_sb = sb;
req->sb_index = sb_index;
req->hw_sb = p_cid->rel.sb;
req->sb_index = p_cid->rel.sb_idx;
req->bd_max_bytes = bd_max_bytes;
req->stat_id = -1;
/* If PF is legacy, we'll need to calculate producers ourselves
* as well as clean them.
*/
if (pp_prod && p_iov->b_pre_fp_hsi) {
if (p_iov->b_pre_fp_hsi) {
u8 hw_qid = p_iov->acquire_resp.resc.hw_qid[rx_qid];
u32 init_prod_val = 0;
*pp_prod = (u8 __iomem *)p_hwfn->regview +
MSTORM_QZONE_START(p_hwfn->cdev) +
hw_qid * MSTORM_QZONE_SIZE;
*pp_prod = (u8 __iomem *)
p_hwfn->regview +
MSTORM_QZONE_START(p_hwfn->cdev) +
hw_qid * MSTORM_QZONE_SIZE;
/* Init the rcq, rx bd and rx sge (if valid) producers to 0 */
__internal_ram_wr(p_hwfn, *pp_prod, sizeof(u32),
......@@ -444,7 +445,7 @@ int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
}
/* Learn the address of the producer from the response */
if (pp_prod && !p_iov->b_pre_fp_hsi) {
if (!p_iov->b_pre_fp_hsi) {
u32 init_prod_val = 0;
*pp_prod = (u8 __iomem *)p_hwfn->regview + resp->offset;
......@@ -462,7 +463,8 @@ int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
return rc;
}
int qed_vf_pf_rxq_stop(struct qed_hwfn *p_hwfn, u16 rx_qid, bool cqe_completion)
int qed_vf_pf_rxq_stop(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid, bool cqe_completion)
{
struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
struct vfpf_stop_rxqs_tlv *req;
......@@ -472,7 +474,7 @@ int qed_vf_pf_rxq_stop(struct qed_hwfn *p_hwfn, u16 rx_qid, bool cqe_completion)
/* clear mailbox and prep first tlv */
req = qed_vf_pf_prep(p_hwfn, CHANNEL_TLV_STOP_RXQS, sizeof(*req));
req->rx_qid = rx_qid;
req->rx_qid = p_cid->rel.queue_id;
req->num_rxqs = 1;
req->cqe_completion = cqe_completion;
......@@ -496,28 +498,28 @@ int qed_vf_pf_rxq_stop(struct qed_hwfn *p_hwfn, u16 rx_qid, bool cqe_completion)
return rc;
}
int qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
u16 tx_queue_id,
u16 sb,
u8 sb_index,
dma_addr_t pbl_addr,
u16 pbl_size, void __iomem **pp_doorbell)
int
qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
dma_addr_t pbl_addr,
u16 pbl_size, void __iomem **pp_doorbell)
{
struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
struct pfvf_start_queue_resp_tlv *resp;
struct vfpf_start_txq_tlv *req;
u16 qid = p_cid->rel.queue_id;
int rc;
/* clear mailbox and prep first tlv */
req = qed_vf_pf_prep(p_hwfn, CHANNEL_TLV_START_TXQ, sizeof(*req));
req->tx_qid = tx_queue_id;
req->tx_qid = qid;
/* Tx */
req->pbl_addr = pbl_addr;
req->pbl_size = pbl_size;
req->hw_sb = sb;
req->sb_index = sb_index;
req->hw_sb = p_cid->rel.sb;
req->sb_index = p_cid->rel.sb_idx;
/* add list termination tlv */
qed_add_tlv(p_hwfn, &p_iov->offset,
......@@ -533,33 +535,29 @@ int qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
goto exit;
}
if (pp_doorbell) {
/* Modern PFs provide the actual offsets, while legacy
* provided only the queue id.
*/
if (!p_iov->b_pre_fp_hsi) {
*pp_doorbell = (u8 __iomem *)p_hwfn->doorbells +
resp->offset;
} else {
u8 cid = p_iov->acquire_resp.resc.cid[tx_queue_id];
u32 db_addr;
db_addr = qed_db_addr_vf(cid, DQ_DEMS_LEGACY);
*pp_doorbell = (u8 __iomem *)p_hwfn->doorbells +
db_addr;
}
/* Modern PFs provide the actual offsets, while legacy
* provided only the queue id.
*/
if (!p_iov->b_pre_fp_hsi) {
*pp_doorbell = (u8 __iomem *)p_hwfn->doorbells + resp->offset;
} else {
u8 cid = p_iov->acquire_resp.resc.cid[qid];
DP_VERBOSE(p_hwfn, QED_MSG_IOV,
"Txq[0x%02x]: doorbell at %p [offset 0x%08x]\n",
tx_queue_id, *pp_doorbell, resp->offset);
*pp_doorbell = (u8 __iomem *)p_hwfn->doorbells +
qed_db_addr_vf(cid,
DQ_DEMS_LEGACY);
}
DP_VERBOSE(p_hwfn, QED_MSG_IOV,
"Txq[0x%02x]: doorbell at %p [offset 0x%08x]\n",
qid, *pp_doorbell, resp->offset);
exit:
qed_vf_pf_req_end(p_hwfn, rc);
return rc;
}
int qed_vf_pf_txq_stop(struct qed_hwfn *p_hwfn, u16 tx_qid)
int qed_vf_pf_txq_stop(struct qed_hwfn *p_hwfn, struct qed_queue_cid *p_cid)
{
struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
struct vfpf_stop_txqs_tlv *req;
......@@ -569,7 +567,7 @@ int qed_vf_pf_txq_stop(struct qed_hwfn *p_hwfn, u16 tx_qid)
/* clear mailbox and prep first tlv */
req = qed_vf_pf_prep(p_hwfn, CHANNEL_TLV_STOP_TXQS, sizeof(*req));
req->tx_qid = tx_qid;
req->tx_qid = p_cid->rel.queue_id;
req->num_txqs = 1;
/* add list termination tlv */
......
drivers/net/ethernet/qlogic/qed/qed_vf.h
View file @ 348bfec2
......@@ -666,10 +666,7 @@ int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn);
/**
* @brief VF - start the RX Queue by sending a message to the PF
* @param p_hwfn
* @param cid - zero based within the VF
* @param rx_queue_id - zero based within the VF
* @param sb - VF status block for this queue
* @param sb_index - Index within the status block
* @param p_cid - Only relative fields are relevant
* @param bd_max_bytes - maximum number of bytes per bd
* @param bd_chain_phys_addr - physical address of bd chain
* @param cqe_pbl_addr - physical address of pbl
......@@ -680,9 +677,7 @@ int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn);
* @return int
*/
int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
u8 rx_queue_id,
u16 sb,
u8 sb_index,
struct qed_queue_cid *p_cid,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr,
......@@ -702,24 +697,23 @@ int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
*
* @return int
*/
int qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
u16 tx_queue_id,
u16 sb,
u8 sb_index,
dma_addr_t pbl_addr,
u16 pbl_size, void __iomem **pp_doorbell);
int
qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid,
dma_addr_t pbl_addr,
u16 pbl_size, void __iomem **pp_doorbell);
/**
* @brief VF - stop the RX queue by sending a message to the PF
*
* @param p_hwfn
* @param rx_qid
* @param p_cid
* @param cqe_completion
*
* @return int
*/
int qed_vf_pf_rxq_stop(struct qed_hwfn *p_hwfn,
u16 rx_qid, bool cqe_completion);
struct qed_queue_cid *p_cid, bool cqe_completion);
/**
* @brief VF - stop the TX queue by sending a message to the PF
......@@ -729,7 +723,7 @@ int qed_vf_pf_rxq_stop(struct qed_hwfn *p_hwfn,
*
* @return int
*/
int qed_vf_pf_txq_stop(struct qed_hwfn *p_hwfn, u16 tx_qid);
int qed_vf_pf_txq_stop(struct qed_hwfn *p_hwfn, struct qed_queue_cid *p_cid);
/**
* @brief VF - send a vport update command
......@@ -902,9 +896,7 @@ static inline int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
}
static inline int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
u8 rx_queue_id,
u16 sb,
u8 sb_index,
struct qed_queue_cid *p_cid,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_adr,
dma_addr_t cqe_pbl_addr,
......@@ -914,9 +906,7 @@ static inline int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
}
static inline int qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
u16 tx_queue_id,
u16 sb,
u8 sb_index,
struct qed_queue_cid *p_cid,
dma_addr_t pbl_addr,
u16 pbl_size, void __iomem **pp_doorbell)
{
......@@ -924,12 +914,14 @@ static inline int qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
}
static inline int qed_vf_pf_rxq_stop(struct qed_hwfn *p_hwfn,
u16 rx_qid, bool cqe_completion)
struct qed_queue_cid *p_cid,
bool cqe_completion)
{
return -EINVAL;
}
static inline int qed_vf_pf_txq_stop(struct qed_hwfn *p_hwfn, u16 tx_qid)
static inline int qed_vf_pf_txq_stop(struct qed_hwfn *p_hwfn,
struct qed_queue_cid *p_cid)
{
return -EINVAL;
}
......
drivers/net/ethernet/qlogic/qede/qede.h
View file @ 348bfec2
......@@ -16,6 +16,7 @@
#include <linux/bitmap.h>
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/bpf.h>
#include <linux/io.h>
#include <linux/qed/common_hsi.h>
#include <linux/qed/eth_common.h>
......@@ -127,10 +128,9 @@ struct qede_dev {
const struct qed_eth_ops *ops;
struct qed_dev_eth_info dev_info;
struct qed_dev_eth_info dev_info;
#define QEDE_MAX_RSS_CNT(edev) ((edev)->dev_info.num_queues)
#define QEDE_MAX_TSS_CNT(edev) ((edev)->dev_info.num_queues * \
(edev)->dev_info.num_tc)
#define QEDE_MAX_TSS_CNT(edev) ((edev)->dev_info.num_queues)
struct qede_fastpath *fp_array;
u8 req_num_tx;
......@@ -139,17 +139,9 @@ struct qede_dev {
u8 fp_num_rx;
u16 req_queues;
u16 num_queues;
u8 num_tc;
#define QEDE_QUEUE_CNT(edev) ((edev)->num_queues)
#define QEDE_RSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_tx)
#define QEDE_TSS_COUNT(edev) (((edev)->num_queues - (edev)->fp_num_rx) * \
(edev)->num_tc)
#define QEDE_TX_IDX(edev, txqidx) ((edev)->fp_num_rx + (txqidx) % \
QEDE_TSS_COUNT(edev))
#define QEDE_TC_IDX(edev, txqidx) ((txqidx) / QEDE_TSS_COUNT(edev))
#define QEDE_TX_QUEUE(edev, txqidx) \
(&(edev)->fp_array[QEDE_TX_IDX((edev), (txqidx))].txqs[QEDE_TC_IDX(\
(edev), (txqidx))])
#define QEDE_TSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_rx)
struct qed_int_info int_info;
unsigned char primary_mac[ETH_ALEN];
......@@ -196,6 +188,8 @@ struct qede_dev {
bool wol_enabled;
struct qede_rdma_dev rdma_info;
struct bpf_prog *xdp_prog;
};
enum QEDE_STATE {
......@@ -225,39 +219,67 @@ enum qede_agg_state {
};
struct qede_agg_info {
struct sw_rx_data replace_buf;
dma_addr_t replace_buf_mapping;
struct sw_rx_data start_buf;
dma_addr_t start_buf_mapping;
struct eth_fast_path_rx_tpa_start_cqe start_cqe;
enum qede_agg_state agg_state;
/* rx_buf is a data buffer that can be placed / consumed from rx bd
* chain. It has two purposes: We will preallocate the data buffer
* for each aggregation when we open the interface and will place this
* buffer on the rx-bd-ring when we receive TPA_START. We don't want
* to be in a state where allocation fails, as we can't reuse the
* consumer buffer in the rx-chain since FW may still be writing to it
* (since header needs to be modified for TPA).
* The second purpose is to keep a pointer to the bd buffer during
* aggregation.
*/
struct sw_rx_data buffer;
dma_addr_t buffer_mapping;
struct sk_buff *skb;
int frag_id;
/* We need some structs from the start cookie until termination */
u16 vlan_tag;
u16 start_cqe_bd_len;
u8 start_cqe_placement_offset;
u8 state;
u8 frag_id;
u8 tunnel_type;
};
struct qede_rx_queue {
__le16 *hw_cons_ptr;
struct sw_rx_data *sw_rx_ring;
u16 sw_rx_cons;
u16 sw_rx_prod;
struct qed_chain rx_bd_ring;
struct qed_chain rx_comp_ring;
void __iomem *hw_rxq_prod_addr;
__le16 *hw_cons_ptr;
void __iomem *hw_rxq_prod_addr;
/* Required for the allocation of replacement buffers */
struct device *dev;
struct bpf_prog *xdp_prog;
u16 sw_rx_cons;
u16 sw_rx_prod;
u16 num_rx_buffers; /* Slowpath */
u8 data_direction;
u8 rxq_id;
u32 rx_buf_size;
u32 rx_buf_seg_size;
u64 rcv_pkts;
struct sw_rx_data *sw_rx_ring;
struct qed_chain rx_bd_ring;
struct qed_chain rx_comp_ring ____cacheline_aligned;
/* GRO */
struct qede_agg_info tpa_info[ETH_TPA_MAX_AGGS_NUM];
struct qede_agg_info tpa_info[ETH_TPA_MAX_AGGS_NUM];
int rx_buf_size;
unsigned int rx_buf_seg_size;
u64 rx_hw_errors;
u64 rx_alloc_errors;
u64 rx_ip_frags;
u16 num_rx_buffers;
u16 rxq_id;
u64 xdp_no_pass;
u64 rcv_pkts;
u64 rx_hw_errors;
u64 rx_alloc_errors;
u64 rx_ip_frags;
void *handle;
};
union db_prod {
......@@ -273,20 +295,39 @@ struct sw_tx_bd {
};
struct qede_tx_queue {
int index; /* Queue index */
__le16 *hw_cons_ptr;
struct sw_tx_bd *sw_tx_ring;
u16 sw_tx_cons;
u16 sw_tx_prod;
struct qed_chain tx_pbl;
void __iomem *doorbell_addr;
union db_prod tx_db;
u16 num_tx_buffers;
u64 xmit_pkts;
u64 stopped_cnt;
bool is_legacy;
u8 is_xdp;
bool is_legacy;
u16 sw_tx_cons;
u16 sw_tx_prod;
u16 num_tx_buffers; /* Slowpath only */
u64 xmit_pkts;
u64 stopped_cnt;
__le16 *hw_cons_ptr;
/* Needed for the mapping of packets */
struct device *dev;
void __iomem *doorbell_addr;
union db_prod tx_db;
int index; /* Slowpath only */
#define QEDE_TXQ_XDP_TO_IDX(edev, txq) ((txq)->index - \
QEDE_MAX_TSS_CNT(edev))
#define QEDE_TXQ_IDX_TO_XDP(edev, idx) ((idx) + QEDE_MAX_TSS_CNT(edev))
/* Regular Tx requires skb + metadata for release purpose,
* while XDP requires only the pages themselves.
*/
union {
struct sw_tx_bd *skbs;
struct page **pages;
} sw_tx_ring;
struct qed_chain tx_pbl;
/* Slowpath; Should be kept in end [unless missing padding] */
void *handle;
};
#define BD_UNMAP_ADDR(bd) HILO_U64(le32_to_cpu((bd)->addr.hi), \
......@@ -303,13 +344,16 @@ struct qede_fastpath {
struct qede_dev *edev;
#define QEDE_FASTPATH_TX BIT(0)
#define QEDE_FASTPATH_RX BIT(1)
#define QEDE_FASTPATH_XDP BIT(2)
#define QEDE_FASTPATH_COMBINED (QEDE_FASTPATH_TX | QEDE_FASTPATH_RX)
u8 type;
u8 id;
u8 xdp_xmit;
struct napi_struct napi;
struct qed_sb_info *sb_info;
struct qede_rx_queue *rxq;
struct qede_tx_queue *txqs;
struct qede_tx_queue *txq;
struct qede_tx_queue *xdp_tx;
#define VEC_NAME_SIZE (sizeof(((struct net_device *)0)->name) + 8)
char name[VEC_NAME_SIZE];
......@@ -332,8 +376,13 @@ struct qede_fastpath {
#define QEDE_SP_VXLAN_PORT_CONFIG 2
#define QEDE_SP_GENEVE_PORT_CONFIG 3
union qede_reload_args {
u16 mtu;
struct qede_reload_args {
void (*func)(struct qede_dev *edev, struct qede_reload_args *args);
union {
netdev_features_t features;
struct bpf_prog *new_prog;
u16 mtu;
} u;
};
#ifdef CONFIG_DCB
......@@ -342,15 +391,14 @@ void qede_set_dcbnl_ops(struct net_device *ndev);
void qede_config_debug(uint debug, u32 *p_dp_module, u8 *p_dp_level);
void qede_set_ethtool_ops(struct net_device *netdev);
void qede_reload(struct qede_dev *edev,
void (*func)(struct qede_dev *edev,
union qede_reload_args *args),
union qede_reload_args *args);
struct qede_reload_args *args, bool is_locked);
int qede_change_mtu(struct net_device *dev, int new_mtu);
void qede_fill_by_demand_stats(struct qede_dev *edev);
void __qede_lock(struct qede_dev *edev);
void __qede_unlock(struct qede_dev *edev);
bool qede_has_rx_work(struct qede_rx_queue *rxq);
int qede_txq_has_work(struct qede_tx_queue *txq);
void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, struct qede_dev *edev,
u8 count);
void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, u8 count);
void qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq);
#define RX_RING_SIZE_POW 13
......
drivers/net/ethernet/qlogic/qede/qede_ethtool.c
View file @ 348bfec2
......@@ -16,13 +16,6 @@
#include <linux/capability.h>
#include "qede.h"
#define QEDE_STAT_OFFSET(stat_name) (offsetof(struct qede_stats, stat_name))
#define QEDE_STAT_STRING(stat_name) (#stat_name)
#define _QEDE_STAT(stat_name, pf_only) \
{QEDE_STAT_OFFSET(stat_name), QEDE_STAT_STRING(stat_name), pf_only}
#define QEDE_PF_STAT(stat_name) _QEDE_STAT(stat_name, true)
#define QEDE_STAT(stat_name) _QEDE_STAT(stat_name, false)
#define QEDE_RQSTAT_OFFSET(stat_name) \
(offsetof(struct qede_rx_queue, stat_name))
#define QEDE_RQSTAT_STRING(stat_name) (#stat_name)
......@@ -39,12 +32,10 @@ static const struct {
QEDE_RQSTAT(rx_hw_errors),
QEDE_RQSTAT(rx_alloc_errors),
QEDE_RQSTAT(rx_ip_frags),
QEDE_RQSTAT(xdp_no_pass),
};
#define QEDE_NUM_RQSTATS ARRAY_SIZE(qede_rqstats_arr)
#define QEDE_RQSTATS_DATA(dev, sindex, rqindex) \
(*((u64 *)(((char *)(dev->fp_array[(rqindex)].rxq)) +\
qede_rqstats_arr[(sindex)].offset)))
#define QEDE_TQSTAT_OFFSET(stat_name) \
(offsetof(struct qede_tx_queue, stat_name))
#define QEDE_TQSTAT_STRING(stat_name) (#stat_name)
......@@ -59,10 +50,12 @@ static const struct {
QEDE_TQSTAT(stopped_cnt),
};
#define QEDE_TQSTATS_DATA(dev, sindex, tssid, tcid) \
(*((u64 *)(((void *)(&dev->fp_array[tssid].txqs[tcid])) +\
qede_tqstats_arr[(sindex)].offset)))
#define QEDE_STAT_OFFSET(stat_name) (offsetof(struct qede_stats, stat_name))
#define QEDE_STAT_STRING(stat_name) (#stat_name)
#define _QEDE_STAT(stat_name, pf_only) \
{QEDE_STAT_OFFSET(stat_name), QEDE_STAT_STRING(stat_name), pf_only}
#define QEDE_PF_STAT(stat_name) _QEDE_STAT(stat_name, true)
#define QEDE_STAT(stat_name) _QEDE_STAT(stat_name, false)
static const struct {
u64 offset;
char string[ETH_GSTRING_LEN];
......@@ -136,10 +129,6 @@ static const struct {
QEDE_STAT(coalesced_bytes),
};
#define QEDE_STATS_DATA(dev, index) \
(*((u64 *)(((char *)(dev)) + offsetof(struct qede_dev, stats) \
+ qede_stats_arr[(index)].offset)))
#define QEDE_NUM_STATS ARRAY_SIZE(qede_stats_arr)
enum {
......@@ -170,39 +159,60 @@ static const char qede_tests_str_arr[QEDE_ETHTOOL_TEST_MAX][ETH_GSTRING_LEN] = {
"Nvram (online)\t\t",
};
static void qede_get_strings_stats_txq(struct qede_dev *edev,
struct qede_tx_queue *txq, u8 **buf)
{
int i;
for (i = 0; i < QEDE_NUM_TQSTATS; i++) {
if (txq->is_xdp)
sprintf(*buf, "%d [XDP]: %s",
QEDE_TXQ_XDP_TO_IDX(edev, txq),
qede_tqstats_arr[i].string);
else
sprintf(*buf, "%d: %s", txq->index,
qede_tqstats_arr[i].string);
*buf += ETH_GSTRING_LEN;
}
}
static void qede_get_strings_stats_rxq(struct qede_dev *edev,
struct qede_rx_queue *rxq, u8 **buf)
{
int i;
for (i = 0; i < QEDE_NUM_RQSTATS; i++) {
sprintf(*buf, "%d: %s", rxq->rxq_id,
qede_rqstats_arr[i].string);
*buf += ETH_GSTRING_LEN;
}
}
static void qede_get_strings_stats(struct qede_dev *edev, u8 *buf)
{
int i, j, k;
struct qede_fastpath *fp;
int i;
for (i = 0, k = 0; i < QEDE_QUEUE_CNT(edev); i++) {
int tc;
/* Account for queue statistics */
for (i = 0; i < QEDE_QUEUE_CNT(edev); i++) {
fp = &edev->fp_array[i];
if (edev->fp_array[i].type & QEDE_FASTPATH_RX) {
for (j = 0; j < QEDE_NUM_RQSTATS; j++)
sprintf(buf + (k + j) * ETH_GSTRING_LEN,
"%d: %s", i,
qede_rqstats_arr[j].string);
k += QEDE_NUM_RQSTATS;
}
if (fp->type & QEDE_FASTPATH_RX)
qede_get_strings_stats_rxq(edev, fp->rxq, &buf);
if (edev->fp_array[i].type & QEDE_FASTPATH_TX) {
for (tc = 0; tc < edev->num_tc; tc++) {
for (j = 0; j < QEDE_NUM_TQSTATS; j++)
sprintf(buf + (k + j) *
ETH_GSTRING_LEN,
"%d.%d: %s", i, tc,
qede_tqstats_arr[j].string);
k += QEDE_NUM_TQSTATS;
}
}
if (fp->type & QEDE_FASTPATH_XDP)
qede_get_strings_stats_txq(edev, fp->xdp_tx, &buf);
if (fp->type & QEDE_FASTPATH_TX)
qede_get_strings_stats_txq(edev, fp->txq, &buf);
}
for (i = 0, j = 0; i < QEDE_NUM_STATS; i++) {
/* Account for non-queue statistics */
for (i = 0; i < QEDE_NUM_STATS; i++) {
if (IS_VF(edev) && qede_stats_arr[i].pf_only)
continue;
strcpy(buf + (k + j) * ETH_GSTRING_LEN,
qede_stats_arr[i].string);
j++;
strcpy(buf, qede_stats_arr[i].string);
buf += ETH_GSTRING_LEN;
}
}
......@@ -228,42 +238,61 @@ static void qede_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
}
}
static void qede_get_ethtool_stats_txq(struct qede_tx_queue *txq, u64 **buf)
{
int i;
for (i = 0; i < QEDE_NUM_TQSTATS; i++) {
**buf = *((u64 *)(((void *)txq) + qede_tqstats_arr[i].offset));
(*buf)++;
}
}
static void qede_get_ethtool_stats_rxq(struct qede_rx_queue *rxq, u64 **buf)
{
int i;
for (i = 0; i < QEDE_NUM_RQSTATS; i++) {
**buf = *((u64 *)(((void *)rxq) + qede_rqstats_arr[i].offset));
(*buf)++;
}
}
static void qede_get_ethtool_stats(struct net_device *dev,
struct ethtool_stats *stats, u64 *buf)
{
struct qede_dev *edev = netdev_priv(dev);
int sidx, cnt = 0;
int qid;
struct qede_fastpath *fp;
int i;
qede_fill_by_demand_stats(edev);
mutex_lock(&edev->qede_lock);
/* Need to protect the access to the fastpath array */
__qede_lock(edev);
for (qid = 0; qid < QEDE_QUEUE_CNT(edev); qid++) {
int tc;
for (i = 0; i < QEDE_QUEUE_CNT(edev); i++) {
fp = &edev->fp_array[i];
if (edev->fp_array[qid].type & QEDE_FASTPATH_RX) {
for (sidx = 0; sidx < QEDE_NUM_RQSTATS; sidx++)
buf[cnt++] = QEDE_RQSTATS_DATA(edev, sidx, qid);
}
if (fp->type & QEDE_FASTPATH_RX)
qede_get_ethtool_stats_rxq(fp->rxq, &buf);
if (edev->fp_array[qid].type & QEDE_FASTPATH_TX) {
for (tc = 0; tc < edev->num_tc; tc++) {
for (sidx = 0; sidx < QEDE_NUM_TQSTATS; sidx++)
buf[cnt++] = QEDE_TQSTATS_DATA(edev,
sidx,
qid, tc);
}
}
if (fp->type & QEDE_FASTPATH_XDP)
qede_get_ethtool_stats_txq(fp->xdp_tx, &buf);
if (fp->type & QEDE_FASTPATH_TX)
qede_get_ethtool_stats_txq(fp->txq, &buf);
}
for (sidx = 0; sidx < QEDE_NUM_STATS; sidx++) {
if (IS_VF(edev) && qede_stats_arr[sidx].pf_only)
for (i = 0; i < QEDE_NUM_STATS; i++) {
if (IS_VF(edev) && qede_stats_arr[i].pf_only)
continue;
buf[cnt++] = QEDE_STATS_DATA(edev, sidx);
*buf = *((u64 *)(((void *)&edev->stats) +
qede_stats_arr[i].offset));
buf++;
}
mutex_unlock(&edev->qede_lock);
__qede_unlock(edev);
}
static int qede_get_sset_count(struct net_device *dev, int stringset)
......@@ -280,8 +309,18 @@ static int qede_get_sset_count(struct net_device *dev, int stringset)
if (qede_stats_arr[i].pf_only)
num_stats--;
}
return num_stats + QEDE_RSS_COUNT(edev) * QEDE_NUM_RQSTATS +
QEDE_TSS_COUNT(edev) * QEDE_NUM_TQSTATS * edev->num_tc;
/* Account for the Regular Tx statistics */
num_stats += QEDE_TSS_COUNT(edev) * QEDE_NUM_TQSTATS;
/* Account for the Regular Rx statistics */
num_stats += QEDE_RSS_COUNT(edev) * QEDE_NUM_RQSTATS;
/* Account for XDP statistics [if needed] */
if (edev->xdp_prog)
num_stats += QEDE_RSS_COUNT(edev) * QEDE_NUM_TQSTATS;
return num_stats;
case ETH_SS_PRIV_FLAGS:
return QEDE_PRI_FLAG_LEN;
case ETH_SS_TEST:
......@@ -352,6 +391,8 @@ static int qede_get_link_ksettings(struct net_device *dev,
struct qede_dev *edev = netdev_priv(dev);
struct qed_link_output current_link;
__qede_lock(edev);
memset(&current_link, 0, sizeof(current_link));
edev->ops->common->get_link(edev->cdev, &current_link);
......@@ -371,6 +412,9 @@ static int qede_get_link_ksettings(struct net_device *dev,
base->speed = SPEED_UNKNOWN;
base->duplex = DUPLEX_UNKNOWN;
}
__qede_unlock(edev);
base->port = current_link.port;
base->autoneg = (current_link.autoneg) ? AUTONEG_ENABLE :
AUTONEG_DISABLE;
......@@ -679,8 +723,7 @@ static int qede_set_ringparam(struct net_device *dev,
edev->q_num_rx_buffers = ering->rx_pending;
edev->q_num_tx_buffers = ering->tx_pending;
if (netif_running(edev->ndev))
qede_reload(edev, NULL, NULL);
qede_reload(edev, NULL, false);
return 0;
}
......@@ -765,29 +808,27 @@ static int qede_get_regs_len(struct net_device *ndev)
return -EINVAL;
}
static void qede_update_mtu(struct qede_dev *edev, union qede_reload_args *args)
static void qede_update_mtu(struct qede_dev *edev,
struct qede_reload_args *args)
{
edev->ndev->mtu = args->mtu;
edev->ndev->mtu = args->u.mtu;
}
/* Netdevice NDOs */
int qede_change_mtu(struct net_device *ndev, int new_mtu)
{
struct qede_dev *edev = netdev_priv(ndev);
union qede_reload_args args;
struct qede_reload_args args;
DP_VERBOSE(edev, (NETIF_MSG_IFUP | NETIF_MSG_IFDOWN),
"Configuring MTU size of %d\n", new_mtu);
/* Set the mtu field and re-start the interface if needed*/
args.mtu = new_mtu;
if (netif_running(edev->ndev))
qede_reload(edev, &qede_update_mtu, &args);
qede_update_mtu(edev, &args);
/* Set the mtu field and re-start the interface if needed */
args.u.mtu = new_mtu;
args.func = &qede_update_mtu;
qede_reload(edev, &args, false);
edev->ops->common->update_mtu(edev->cdev, args.mtu);
edev->ops->common->update_mtu(edev->cdev, new_mtu);
return 0;
}
......@@ -871,8 +912,7 @@ static int qede_set_channels(struct net_device *dev,
sizeof(edev->rss_params.rss_ind_table));
}
if (netif_running(dev))
qede_reload(edev, NULL, NULL);
qede_reload(edev, NULL, false);
return 0;
}
......@@ -1178,7 +1218,7 @@ static int qede_selftest_transmit_traffic(struct qede_dev *edev,
for_each_queue(i) {
if (edev->fp_array[i].type & QEDE_FASTPATH_TX) {
txq = edev->fp_array[i].txqs;
txq = edev->fp_array[i].txq;
break;
}
}
......@@ -1190,7 +1230,7 @@ static int qede_selftest_transmit_traffic(struct qede_dev *edev,
/* Fill the entry in the SW ring and the BDs in the FW ring */
idx = txq->sw_tx_prod & NUM_TX_BDS_MAX;
txq->sw_tx_ring[idx].skb = skb;
txq->sw_tx_ring.skbs[idx].skb = skb;
first_bd = qed_chain_produce(&txq->tx_pbl);
memset(first_bd, 0, sizeof(*first_bd));
val = 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT;
......@@ -1244,7 +1284,7 @@ static int qede_selftest_transmit_traffic(struct qede_dev *edev,
dma_unmap_single(&edev->pdev->dev, BD_UNMAP_ADDR(first_bd),
BD_UNMAP_LEN(first_bd), DMA_TO_DEVICE);
txq->sw_tx_cons++;
txq->sw_tx_ring[idx].skb = NULL;
txq->sw_tx_ring.skbs[idx].skb = NULL;
return 0;
}
......@@ -1312,13 +1352,13 @@ static int qede_selftest_receive_traffic(struct qede_dev *edev)
break;
}
qede_recycle_rx_bd_ring(rxq, edev, 1);
qede_recycle_rx_bd_ring(rxq, 1);
qed_chain_recycle_consumed(&rxq->rx_comp_ring);
break;
}
DP_INFO(edev, "Not the transmitted packet\n");
qede_recycle_rx_bd_ring(rxq, edev, 1);
qede_recycle_rx_bd_ring(rxq, 1);
qed_chain_recycle_consumed(&rxq->rx_comp_ring);
}
......
drivers/net/ethernet/qlogic/qede/qede_main.c
View file @ 348bfec2
......@@ -94,12 +94,26 @@ static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id);
#define TX_TIMEOUT (5 * HZ)
/* Utilize last protocol index for XDP */
#define XDP_PI 11
static void qede_remove(struct pci_dev *pdev);
static void qede_shutdown(struct pci_dev *pdev);
static int qede_alloc_rx_buffer(struct qede_dev *edev,
struct qede_rx_queue *rxq);
static void qede_link_update(void *dev, struct qed_link_output *link);
/* The qede lock is used to protect driver state change and driver flows that
* are not reentrant.
*/
void __qede_lock(struct qede_dev *edev)
{
mutex_lock(&edev->qede_lock);
}
void __qede_unlock(struct qede_dev *edev)
{
mutex_unlock(&edev->qede_lock);
}
#ifdef CONFIG_QED_SRIOV
static int qede_set_vf_vlan(struct net_device *ndev, int vf, u16 vlan, u8 qos,
__be16 vlan_proto)
......@@ -290,12 +304,12 @@ static int qede_free_tx_pkt(struct qede_dev *edev,
struct qede_tx_queue *txq, int *len)
{
u16 idx = txq->sw_tx_cons & NUM_TX_BDS_MAX;
struct sk_buff *skb = txq->sw_tx_ring[idx].skb;
struct sk_buff *skb = txq->sw_tx_ring.skbs[idx].skb;
struct eth_tx_1st_bd *first_bd;
struct eth_tx_bd *tx_data_bd;
int bds_consumed = 0;
int nbds;
bool data_split = txq->sw_tx_ring[idx].flags & QEDE_TSO_SPLIT_BD;
bool data_split = txq->sw_tx_ring.skbs[idx].flags & QEDE_TSO_SPLIT_BD;
int i, split_bd_len = 0;
if (unlikely(!skb)) {
......@@ -335,20 +349,19 @@ static int qede_free_tx_pkt(struct qede_dev *edev,
/* Free skb */
dev_kfree_skb_any(skb);
txq->sw_tx_ring[idx].skb = NULL;
txq->sw_tx_ring[idx].flags = 0;
txq->sw_tx_ring.skbs[idx].skb = NULL;
txq->sw_tx_ring.skbs[idx].flags = 0;
return 0;
}
/* Unmap the data and free skb when mapping failed during start_xmit */
static void qede_free_failed_tx_pkt(struct qede_dev *edev,
struct qede_tx_queue *txq,
static void qede_free_failed_tx_pkt(struct qede_tx_queue *txq,
struct eth_tx_1st_bd *first_bd,
int nbd, bool data_split)
{
u16 idx = txq->sw_tx_prod & NUM_TX_BDS_MAX;
struct sk_buff *skb = txq->sw_tx_ring[idx].skb;
struct sk_buff *skb = txq->sw_tx_ring.skbs[idx].skb;
struct eth_tx_bd *tx_data_bd;
int i, split_bd_len = 0;
......@@ -365,7 +378,7 @@ static void qede_free_failed_tx_pkt(struct qede_dev *edev,
nbd--;
}
dma_unmap_single(&edev->pdev->dev, BD_UNMAP_ADDR(first_bd),
dma_unmap_single(txq->dev, BD_UNMAP_ADDR(first_bd),
BD_UNMAP_LEN(first_bd) + split_bd_len, DMA_TO_DEVICE);
/* Unmap the data of the skb frags */
......@@ -373,7 +386,7 @@ static void qede_free_failed_tx_pkt(struct qede_dev *edev,
tx_data_bd = (struct eth_tx_bd *)
qed_chain_produce(&txq->tx_pbl);
if (tx_data_bd->nbytes)
dma_unmap_page(&edev->pdev->dev,
dma_unmap_page(txq->dev,
BD_UNMAP_ADDR(tx_data_bd),
BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
}
......@@ -384,12 +397,11 @@ static void qede_free_failed_tx_pkt(struct qede_dev *edev,
/* Free skb */
dev_kfree_skb_any(skb);
txq->sw_tx_ring[idx].skb = NULL;
txq->sw_tx_ring[idx].flags = 0;
txq->sw_tx_ring.skbs[idx].skb = NULL;
txq->sw_tx_ring.skbs[idx].flags = 0;
}
static u32 qede_xmit_type(struct qede_dev *edev,
struct sk_buff *skb, int *ipv6_ext)
static u32 qede_xmit_type(struct sk_buff *skb, int *ipv6_ext)
{
u32 rc = XMIT_L4_CSUM;
__be16 l3_proto;
......@@ -456,18 +468,16 @@ static void qede_set_params_for_ipv6_ext(struct sk_buff *skb,
second_bd->data.bitfields2 = cpu_to_le16(bd2_bits2);
}
static int map_frag_to_bd(struct qede_dev *edev,
static int map_frag_to_bd(struct qede_tx_queue *txq,
skb_frag_t *frag, struct eth_tx_bd *bd)
{
dma_addr_t mapping;
/* Map skb non-linear frag data for DMA */
mapping = skb_frag_dma_map(&edev->pdev->dev, frag, 0,
mapping = skb_frag_dma_map(txq->dev, frag, 0,
skb_frag_size(frag), DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(&edev->pdev->dev, mapping))) {
DP_NOTICE(edev, "Unable to map frag - dropping packet\n");
if (unlikely(dma_mapping_error(txq->dev, mapping)))
return -ENOMEM;
}
/* Setup the data pointer of the frag data */
BD_SET_UNMAP_ADDR_LEN(bd, mapping, skb_frag_size(frag));
......@@ -487,8 +497,7 @@ static u16 qede_get_skb_hlen(struct sk_buff *skb, bool is_encap_pkt)
/* +2 for 1st BD for headers and 2nd BD for headlen (if required) */
#if ((MAX_SKB_FRAGS + 2) > ETH_TX_MAX_BDS_PER_NON_LSO_PACKET)
static bool qede_pkt_req_lin(struct qede_dev *edev, struct sk_buff *skb,
u8 xmit_type)
static bool qede_pkt_req_lin(struct sk_buff *skb, u8 xmit_type)
{
int allowed_frags = ETH_TX_MAX_BDS_PER_NON_LSO_PACKET - 1;
......@@ -524,6 +533,47 @@ static inline void qede_update_tx_producer(struct qede_tx_queue *txq)
mmiowb();
}
static int qede_xdp_xmit(struct qede_dev *edev, struct qede_fastpath *fp,
struct sw_rx_data *metadata, u16 padding, u16 length)
{
struct qede_tx_queue *txq = fp->xdp_tx;
u16 idx = txq->sw_tx_prod & NUM_TX_BDS_MAX;
struct eth_tx_1st_bd *first_bd;
if (!qed_chain_get_elem_left(&txq->tx_pbl)) {
txq->stopped_cnt++;
return -ENOMEM;
}
first_bd = (struct eth_tx_1st_bd *)qed_chain_produce(&txq->tx_pbl);
memset(first_bd, 0, sizeof(*first_bd));
first_bd->data.bd_flags.bitfields =
BIT(ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT);
first_bd->data.bitfields |=
(length & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) <<
ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT;
first_bd->data.nbds = 1;
/* We can safely ignore the offset, as it's 0 for XDP */
BD_SET_UNMAP_ADDR_LEN(first_bd, metadata->mapping + padding, length);
/* Synchronize the buffer back to device, as program [probably]
* has changed it.
*/
dma_sync_single_for_device(&edev->pdev->dev,
metadata->mapping + padding,
length, PCI_DMA_TODEVICE);
txq->sw_tx_ring.pages[idx] = metadata->data;
txq->sw_tx_prod++;
/* Mark the fastpath for future XDP doorbell */
fp->xdp_xmit = 1;
return 0;
}
/* Main transmit function */
static netdev_tx_t qede_start_xmit(struct sk_buff *skb,
struct net_device *ndev)
......@@ -547,15 +597,15 @@ static netdev_tx_t qede_start_xmit(struct sk_buff *skb,
/* Get tx-queue context and netdev index */
txq_index = skb_get_queue_mapping(skb);
WARN_ON(txq_index >= QEDE_TSS_COUNT(edev));
txq = QEDE_TX_QUEUE(edev, txq_index);
txq = edev->fp_array[edev->fp_num_rx + txq_index].txq;
netdev_txq = netdev_get_tx_queue(ndev, txq_index);
WARN_ON(qed_chain_get_elem_left(&txq->tx_pbl) < (MAX_SKB_FRAGS + 1));
xmit_type = qede_xmit_type(edev, skb, &ipv6_ext);
xmit_type = qede_xmit_type(skb, &ipv6_ext);
#if ((MAX_SKB_FRAGS + 2) > ETH_TX_MAX_BDS_PER_NON_LSO_PACKET)
if (qede_pkt_req_lin(edev, skb, xmit_type)) {
if (qede_pkt_req_lin(skb, xmit_type)) {
if (skb_linearize(skb)) {
DP_NOTICE(edev,
"SKB linearization failed - silently dropping this SKB\n");
......@@ -567,7 +617,7 @@ static netdev_tx_t qede_start_xmit(struct sk_buff *skb,
/* Fill the entry in the SW ring and the BDs in the FW ring */
idx = txq->sw_tx_prod & NUM_TX_BDS_MAX;
txq->sw_tx_ring[idx].skb = skb;
txq->sw_tx_ring.skbs[idx].skb = skb;
first_bd = (struct eth_tx_1st_bd *)
qed_chain_produce(&txq->tx_pbl);
memset(first_bd, 0, sizeof(*first_bd));
......@@ -575,11 +625,11 @@ static netdev_tx_t qede_start_xmit(struct sk_buff *skb,
1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT;
/* Map skb linear data for DMA and set in the first BD */
mapping = dma_map_single(&edev->pdev->dev, skb->data,
mapping = dma_map_single(txq->dev, skb->data,
skb_headlen(skb), DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(&edev->pdev->dev, mapping))) {
if (unlikely(dma_mapping_error(txq->dev, mapping))) {
DP_NOTICE(edev, "SKB mapping failed\n");
qede_free_failed_tx_pkt(edev, txq, first_bd, 0, false);
qede_free_failed_tx_pkt(txq, first_bd, 0, false);
qede_update_tx_producer(txq);
return NETDEV_TX_OK;
}
......@@ -687,7 +737,7 @@ static netdev_tx_t qede_start_xmit(struct sk_buff *skb,
/* this marks the BD as one that has no
* individual mapping
*/
txq->sw_tx_ring[idx].flags |= QEDE_TSO_SPLIT_BD;
txq->sw_tx_ring.skbs[idx].flags |= QEDE_TSO_SPLIT_BD;
first_bd->nbytes = cpu_to_le16(hlen);
......@@ -703,12 +753,11 @@ static netdev_tx_t qede_start_xmit(struct sk_buff *skb,
/* Handle fragmented skb */
/* special handle for frags inside 2nd and 3rd bds.. */
while (tx_data_bd && frag_idx < skb_shinfo(skb)->nr_frags) {
rc = map_frag_to_bd(edev,
rc = map_frag_to_bd(txq,
&skb_shinfo(skb)->frags[frag_idx],
tx_data_bd);
if (rc) {
qede_free_failed_tx_pkt(edev, txq, first_bd, nbd,
data_split);
qede_free_failed_tx_pkt(txq, first_bd, nbd, data_split);
qede_update_tx_producer(txq);
return NETDEV_TX_OK;
}
......@@ -728,12 +777,11 @@ static netdev_tx_t qede_start_xmit(struct sk_buff *skb,
memset(tx_data_bd, 0, sizeof(*tx_data_bd));
rc = map_frag_to_bd(edev,
rc = map_frag_to_bd(txq,
&skb_shinfo(skb)->frags[frag_idx],
tx_data_bd);
if (rc) {
qede_free_failed_tx_pkt(edev, txq, first_bd, nbd,
data_split);
qede_free_failed_tx_pkt(txq, first_bd, nbd, data_split);
qede_update_tx_producer(txq);
return NETDEV_TX_OK;
}
......@@ -798,6 +846,27 @@ int qede_txq_has_work(struct qede_tx_queue *txq)
return hw_bd_cons != qed_chain_get_cons_idx(&txq->tx_pbl);
}
static void qede_xdp_tx_int(struct qede_dev *edev, struct qede_tx_queue *txq)
{
struct eth_tx_1st_bd *bd;
u16 hw_bd_cons;
hw_bd_cons = le16_to_cpu(*txq->hw_cons_ptr);
barrier();
while (hw_bd_cons != qed_chain_get_cons_idx(&txq->tx_pbl)) {
bd = (struct eth_tx_1st_bd *)qed_chain_consume(&txq->tx_pbl);
dma_unmap_single(&edev->pdev->dev, BD_UNMAP_ADDR(bd),
PAGE_SIZE, DMA_BIDIRECTIONAL);
__free_page(txq->sw_tx_ring.pages[txq->sw_tx_cons &
NUM_TX_BDS_MAX]);
txq->sw_tx_cons++;
txq->xmit_pkts++;
}
}
static int qede_tx_int(struct qede_dev *edev, struct qede_tx_queue *txq)
{
struct netdev_queue *netdev_txq;
......@@ -881,16 +950,6 @@ bool qede_has_rx_work(struct qede_rx_queue *rxq)
return hw_comp_cons != sw_comp_cons;
}
static bool qede_has_tx_work(struct qede_fastpath *fp)
{
u8 tc;
for (tc = 0; tc < fp->edev->num_tc; tc++)
if (qede_txq_has_work(&fp->txqs[tc]))
return true;
return false;
}
static inline void qede_rx_bd_ring_consume(struct qede_rx_queue *rxq)
{
qed_chain_consume(&rxq->rx_bd_ring);
......@@ -900,8 +959,7 @@ static inline void qede_rx_bd_ring_consume(struct qede_rx_queue *rxq)
/* This function reuses the buffer(from an offset) from
* consumer index to producer index in the bd ring
*/
static inline void qede_reuse_page(struct qede_dev *edev,
struct qede_rx_queue *rxq,
static inline void qede_reuse_page(struct qede_rx_queue *rxq,
struct sw_rx_data *curr_cons)
{
struct eth_rx_bd *rx_bd_prod = qed_chain_produce(&rxq->rx_bd_ring);
......@@ -923,27 +981,62 @@ static inline void qede_reuse_page(struct qede_dev *edev,
/* In case of allocation failures reuse buffers
* from consumer index to produce buffers for firmware
*/
void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq,
struct qede_dev *edev, u8 count)
void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, u8 count)
{
struct sw_rx_data *curr_cons;
for (; count > 0; count--) {
curr_cons = &rxq->sw_rx_ring[rxq->sw_rx_cons & NUM_RX_BDS_MAX];
qede_reuse_page(edev, rxq, curr_cons);
qede_reuse_page(rxq, curr_cons);
qede_rx_bd_ring_consume(rxq);
}
}
static inline int qede_realloc_rx_buffer(struct qede_dev *edev,
struct qede_rx_queue *rxq,
static int qede_alloc_rx_buffer(struct qede_rx_queue *rxq)
{
struct sw_rx_data *sw_rx_data;
struct eth_rx_bd *rx_bd;
dma_addr_t mapping;
struct page *data;
data = alloc_pages(GFP_ATOMIC, 0);
if (unlikely(!data))
return -ENOMEM;
/* Map the entire page as it would be used
* for multiple RX buffer segment size mapping.
*/
mapping = dma_map_page(rxq->dev, data, 0,
PAGE_SIZE, rxq->data_direction);
if (unlikely(dma_mapping_error(rxq->dev, mapping))) {
__free_page(data);
return -ENOMEM;
}
sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_prod & NUM_RX_BDS_MAX];
sw_rx_data->page_offset = 0;
sw_rx_data->data = data;
sw_rx_data->mapping = mapping;
/* Advance PROD and get BD pointer */
rx_bd = (struct eth_rx_bd *)qed_chain_produce(&rxq->rx_bd_ring);
WARN_ON(!rx_bd);
rx_bd->addr.hi = cpu_to_le32(upper_32_bits(mapping));
rx_bd->addr.lo = cpu_to_le32(lower_32_bits(mapping));
rxq->sw_rx_prod++;
return 0;
}
static inline int qede_realloc_rx_buffer(struct qede_rx_queue *rxq,
struct sw_rx_data *curr_cons)
{
/* Move to the next segment in the page */
curr_cons->page_offset += rxq->rx_buf_seg_size;
if (curr_cons->page_offset == PAGE_SIZE) {
if (unlikely(qede_alloc_rx_buffer(edev, rxq))) {
if (unlikely(qede_alloc_rx_buffer(rxq))) {
/* Since we failed to allocate new buffer
* current buffer can be used again.
*/
......@@ -952,15 +1045,15 @@ static inline int qede_realloc_rx_buffer(struct qede_dev *edev,
return -ENOMEM;
}
dma_unmap_page(&edev->pdev->dev, curr_cons->mapping,
PAGE_SIZE, DMA_FROM_DEVICE);
dma_unmap_page(rxq->dev, curr_cons->mapping,
PAGE_SIZE, rxq->data_direction);
} else {
/* Increment refcount of the page as we don't want
* network stack to take the ownership of the page
* which can be recycled multiple times by the driver.
*/
page_ref_inc(curr_cons->data);
qede_reuse_page(edev, rxq, curr_cons);
qede_reuse_page(rxq, curr_cons);
}
return 0;
......@@ -994,22 +1087,20 @@ void qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq)
mmiowb();
}
static u32 qede_get_rxhash(struct qede_dev *edev,
u8 bitfields,
__le32 rss_hash, enum pkt_hash_types *rxhash_type)
static void qede_get_rxhash(struct sk_buff *skb, u8 bitfields, __le32 rss_hash)
{
enum pkt_hash_types hash_type = PKT_HASH_TYPE_NONE;
enum rss_hash_type htype;
u32 hash = 0;
htype = GET_FIELD(bitfields, ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE);
if ((edev->ndev->features & NETIF_F_RXHASH) && htype) {
*rxhash_type = ((htype == RSS_HASH_TYPE_IPV4) ||
(htype == RSS_HASH_TYPE_IPV6)) ?
PKT_HASH_TYPE_L3 : PKT_HASH_TYPE_L4;
return le32_to_cpu(rss_hash);
if (htype) {
hash_type = ((htype == RSS_HASH_TYPE_IPV4) ||
(htype == RSS_HASH_TYPE_IPV6)) ?
PKT_HASH_TYPE_L3 : PKT_HASH_TYPE_L4;
hash = le32_to_cpu(rss_hash);
}
*rxhash_type = PKT_HASH_TYPE_NONE;
return 0;
skb_set_hash(skb, hash, hash_type);
}
static void qede_set_skb_csum(struct sk_buff *skb, u8 csum_flag)
......@@ -1025,12 +1116,14 @@ static void qede_set_skb_csum(struct sk_buff *skb, u8 csum_flag)
static inline void qede_skb_receive(struct qede_dev *edev,
struct qede_fastpath *fp,
struct qede_rx_queue *rxq,
struct sk_buff *skb, u16 vlan_tag)
{
if (vlan_tag)
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
napi_gro_receive(&fp->napi, skb);
fp->rxq->rcv_pkts++;
}
static void qede_set_gro_params(struct qede_dev *edev,
......@@ -1058,7 +1151,7 @@ static int qede_fill_frag_skb(struct qede_dev *edev,
struct qede_agg_info *tpa_info = &rxq->tpa_info[tpa_agg_index];
struct sk_buff *skb = tpa_info->skb;
if (unlikely(tpa_info->agg_state != QEDE_AGG_STATE_START))
if (unlikely(tpa_info->state != QEDE_AGG_STATE_START))
goto out;
/* Add one frag and update the appropriate fields in the skb */
......@@ -1066,7 +1159,7 @@ static int qede_fill_frag_skb(struct qede_dev *edev,
current_bd->data, current_bd->page_offset,
len_on_bd);
if (unlikely(qede_realloc_rx_buffer(edev, rxq, current_bd))) {
if (unlikely(qede_realloc_rx_buffer(rxq, current_bd))) {
/* Incr page ref count to reuse on allocation failure
* so that it doesn't get freed while freeing SKB.
*/
......@@ -1084,8 +1177,9 @@ static int qede_fill_frag_skb(struct qede_dev *edev,
return 0;
out:
tpa_info->agg_state = QEDE_AGG_STATE_ERROR;
qede_recycle_rx_bd_ring(rxq, edev, 1);
tpa_info->state = QEDE_AGG_STATE_ERROR;
qede_recycle_rx_bd_ring(rxq, 1);
return -ENOMEM;
}
......@@ -1096,12 +1190,10 @@ static void qede_tpa_start(struct qede_dev *edev,
struct qede_agg_info *tpa_info = &rxq->tpa_info[cqe->tpa_agg_index];
struct eth_rx_bd *rx_bd_cons = qed_chain_consume(&rxq->rx_bd_ring);
struct eth_rx_bd *rx_bd_prod = qed_chain_produce(&rxq->rx_bd_ring);
struct sw_rx_data *replace_buf = &tpa_info->replace_buf;
dma_addr_t mapping = tpa_info->replace_buf_mapping;
struct sw_rx_data *replace_buf = &tpa_info->buffer;
dma_addr_t mapping = tpa_info->buffer_mapping;
struct sw_rx_data *sw_rx_data_cons;
struct sw_rx_data *sw_rx_data_prod;
enum pkt_hash_types rxhash_type;
u32 rxhash;
sw_rx_data_cons = &rxq->sw_rx_ring[rxq->sw_rx_cons & NUM_RX_BDS_MAX];
sw_rx_data_prod = &rxq->sw_rx_ring[rxq->sw_rx_prod & NUM_RX_BDS_MAX];
......@@ -1122,11 +1214,11 @@ static void qede_tpa_start(struct qede_dev *edev,
/* move partial skb from cons to pool (don't unmap yet)
* save mapping, incase we drop the packet later on.
*/
tpa_info->start_buf = *sw_rx_data_cons;
tpa_info->buffer = *sw_rx_data_cons;
mapping = HILO_U64(le32_to_cpu(rx_bd_cons->addr.hi),
le32_to_cpu(rx_bd_cons->addr.lo));
tpa_info->start_buf_mapping = mapping;
tpa_info->buffer_mapping = mapping;
rxq->sw_rx_cons++;
/* set tpa state to start only if we are able to allocate skb
......@@ -1137,27 +1229,27 @@ static void qede_tpa_start(struct qede_dev *edev,
le16_to_cpu(cqe->len_on_first_bd));
if (unlikely(!tpa_info->skb)) {
DP_NOTICE(edev, "Failed to allocate SKB for gro\n");
tpa_info->agg_state = QEDE_AGG_STATE_ERROR;
tpa_info->state = QEDE_AGG_STATE_ERROR;
goto cons_buf;
}
skb_put(tpa_info->skb, le16_to_cpu(cqe->len_on_first_bd));
memcpy(&tpa_info->start_cqe, cqe, sizeof(tpa_info->start_cqe));
/* Start filling in the aggregation info */
skb_put(tpa_info->skb, le16_to_cpu(cqe->len_on_first_bd));
tpa_info->frag_id = 0;
tpa_info->agg_state = QEDE_AGG_STATE_START;
tpa_info->state = QEDE_AGG_STATE_START;
rxhash = qede_get_rxhash(edev, cqe->bitfields,
cqe->rss_hash, &rxhash_type);
skb_set_hash(tpa_info->skb, rxhash, rxhash_type);
/* Store some information from first CQE */
tpa_info->start_cqe_placement_offset = cqe->placement_offset;
tpa_info->start_cqe_bd_len = le16_to_cpu(cqe->len_on_first_bd);
if ((le16_to_cpu(cqe->pars_flags.flags) >>
PARSING_AND_ERR_FLAGS_TAG8021QEXIST_SHIFT) &
PARSING_AND_ERR_FLAGS_TAG8021QEXIST_MASK)
PARSING_AND_ERR_FLAGS_TAG8021QEXIST_MASK)
tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
else
tpa_info->vlan_tag = 0;
qede_get_rxhash(tpa_info->skb, cqe->bitfields, cqe->rss_hash);
/* This is needed in order to enable forwarding support */
qede_set_gro_params(edev, tpa_info->skb, cqe);
......@@ -1169,7 +1261,7 @@ static void qede_tpa_start(struct qede_dev *edev,
if (unlikely(cqe->ext_bd_len_list[1])) {
DP_ERR(edev,
"Unlikely - got a TPA aggregation with more than one ext_bd_len_list entry in the TPA start\n");
tpa_info->agg_state = QEDE_AGG_STATE_ERROR;
tpa_info->state = QEDE_AGG_STATE_ERROR;
}
}
......@@ -1239,7 +1331,7 @@ static void qede_gro_receive(struct qede_dev *edev,
send_skb:
skb_record_rx_queue(skb, fp->rxq->rxq_id);
qede_skb_receive(edev, fp, skb, vlan_tag);
qede_skb_receive(edev, fp, fp->rxq, skb, vlan_tag);
}
static inline void qede_tpa_cont(struct qede_dev *edev,
......@@ -1276,7 +1368,7 @@ static void qede_tpa_end(struct qede_dev *edev,
DP_ERR(edev,
"Strange - TPA emd with more than a single len_list entry\n");
if (unlikely(tpa_info->agg_state != QEDE_AGG_STATE_START))
if (unlikely(tpa_info->state != QEDE_AGG_STATE_START))
goto err;
/* Sanity */
......@@ -1290,14 +1382,9 @@ static void qede_tpa_end(struct qede_dev *edev,
le16_to_cpu(cqe->total_packet_len), skb->len);
memcpy(skb->data,
page_address(tpa_info->start_buf.data) +
tpa_info->start_cqe.placement_offset +
tpa_info->start_buf.page_offset,
le16_to_cpu(tpa_info->start_cqe.len_on_first_bd));
/* Recycle [mapped] start buffer for the next replacement */
tpa_info->replace_buf = tpa_info->start_buf;
tpa_info->replace_buf_mapping = tpa_info->start_buf_mapping;
page_address(tpa_info->buffer.data) +
tpa_info->start_cqe_placement_offset +
tpa_info->buffer.page_offset, tpa_info->start_cqe_bd_len);
/* Finalize the SKB */
skb->protocol = eth_type_trans(skb, edev->ndev);
......@@ -1310,18 +1397,11 @@ static void qede_tpa_end(struct qede_dev *edev,
qede_gro_receive(edev, fp, skb, tpa_info->vlan_tag);
tpa_info->agg_state = QEDE_AGG_STATE_NONE;
tpa_info->state = QEDE_AGG_STATE_NONE;
return;
err:
/* The BD starting the aggregation is still mapped; Re-use it for
* future aggregations [as replacement buffer]
*/
memcpy(&tpa_info->replace_buf, &tpa_info->start_buf,
sizeof(struct sw_rx_data));
tpa_info->replace_buf_mapping = tpa_info->start_buf_mapping;
tpa_info->start_buf.data = NULL;
tpa_info->agg_state = QEDE_AGG_STATE_NONE;
tpa_info->state = QEDE_AGG_STATE_NONE;
dev_kfree_skb_any(tpa_info->skb);
tpa_info->skb = NULL;
}
......@@ -1403,238 +1483,357 @@ static bool qede_pkt_is_ip_fragmented(struct eth_fast_path_rx_reg_cqe *cqe,
return false;
}
static int qede_rx_int(struct qede_fastpath *fp, int budget)
/* Return true iff packet is to be passed to stack */
static bool qede_rx_xdp(struct qede_dev *edev,
struct qede_fastpath *fp,
struct qede_rx_queue *rxq,
struct bpf_prog *prog,
struct sw_rx_data *bd,
struct eth_fast_path_rx_reg_cqe *cqe)
{
struct qede_dev *edev = fp->edev;
struct qede_rx_queue *rxq = fp->rxq;
u16 len = le16_to_cpu(cqe->len_on_first_bd);
struct xdp_buff xdp;
enum xdp_action act;
u16 hw_comp_cons, sw_comp_cons, sw_rx_index, parse_flag;
int rx_pkt = 0;
u8 csum_flag;
xdp.data = page_address(bd->data) + cqe->placement_offset;
xdp.data_end = xdp.data + len;
act = bpf_prog_run_xdp(prog, &xdp);
hw_comp_cons = le16_to_cpu(*rxq->hw_cons_ptr);
sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring);
if (act == XDP_PASS)
return true;
/* Memory barrier to prevent the CPU from doing speculative reads of CQE
* / BD in the while-loop before reading hw_comp_cons. If the CQE is
* read before it is written by FW, then FW writes CQE and SB, and then
* the CPU reads the hw_comp_cons, it will use an old CQE.
*/
rmb();
/* Count number of packets not to be passed to stack */
rxq->xdp_no_pass++;
/* Loop to complete all indicated BDs */
while (sw_comp_cons != hw_comp_cons) {
struct eth_fast_path_rx_reg_cqe *fp_cqe;
enum pkt_hash_types rxhash_type;
enum eth_rx_cqe_type cqe_type;
struct sw_rx_data *sw_rx_data;
union eth_rx_cqe *cqe;
struct sk_buff *skb;
struct page *data;
__le16 flags;
u16 len, pad;
u32 rx_hash;
/* Get the CQE from the completion ring */
cqe = (union eth_rx_cqe *)
qed_chain_consume(&rxq->rx_comp_ring);
cqe_type = cqe->fast_path_regular.type;
if (unlikely(cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH)) {
edev->ops->eth_cqe_completion(
edev->cdev, fp->id,
(struct eth_slow_path_rx_cqe *)cqe);
goto next_cqe;
switch (act) {
case XDP_TX:
/* We need the replacement buffer before transmit. */
if (qede_alloc_rx_buffer(rxq)) {
qede_recycle_rx_bd_ring(rxq, 1);
return false;
}
if (cqe_type != ETH_RX_CQE_TYPE_REGULAR) {
switch (cqe_type) {
case ETH_RX_CQE_TYPE_TPA_START:
qede_tpa_start(edev, rxq,
&cqe->fast_path_tpa_start);
goto next_cqe;
case ETH_RX_CQE_TYPE_TPA_CONT:
qede_tpa_cont(edev, rxq,
&cqe->fast_path_tpa_cont);
goto next_cqe;
case ETH_RX_CQE_TYPE_TPA_END:
qede_tpa_end(edev, fp,
&cqe->fast_path_tpa_end);
goto next_rx_only;
default:
break;
}
/* Now if there's a transmission problem, we'd still have to
* throw current buffer, as replacement was already allocated.
*/
if (qede_xdp_xmit(edev, fp, bd, cqe->placement_offset, len)) {
dma_unmap_page(rxq->dev, bd->mapping,
PAGE_SIZE, DMA_BIDIRECTIONAL);
__free_page(bd->data);
}
/* Get the data from the SW ring */
sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS_MAX;
sw_rx_data = &rxq->sw_rx_ring[sw_rx_index];
data = sw_rx_data->data;
fp_cqe = &cqe->fast_path_regular;
len = le16_to_cpu(fp_cqe->len_on_first_bd);
pad = fp_cqe->placement_offset;
flags = cqe->fast_path_regular.pars_flags.flags;
/* If this is an error packet then drop it */
parse_flag = le16_to_cpu(flags);
csum_flag = qede_check_csum(parse_flag);
if (unlikely(csum_flag == QEDE_CSUM_ERROR)) {
if (qede_pkt_is_ip_fragmented(&cqe->fast_path_regular,
parse_flag)) {
rxq->rx_ip_frags++;
goto alloc_skb;
}
/* Regardless, we've consumed an Rx BD */
qede_rx_bd_ring_consume(rxq);
return false;
DP_NOTICE(edev,
"CQE in CONS = %u has error, flags = %x, dropping incoming packet\n",
sw_comp_cons, parse_flag);
rxq->rx_hw_errors++;
qede_recycle_rx_bd_ring(rxq, edev, fp_cqe->bd_num);
goto next_cqe;
}
default:
bpf_warn_invalid_xdp_action(act);
case XDP_ABORTED:
case XDP_DROP:
qede_recycle_rx_bd_ring(rxq, cqe->bd_num);
}
alloc_skb:
skb = netdev_alloc_skb(edev->ndev, QEDE_RX_HDR_SIZE);
if (unlikely(!skb)) {
DP_NOTICE(edev,
"skb allocation failed, dropping incoming packet\n");
qede_recycle_rx_bd_ring(rxq, edev, fp_cqe->bd_num);
rxq->rx_alloc_errors++;
goto next_cqe;
return false;
}
static struct sk_buff *qede_rx_allocate_skb(struct qede_dev *edev,
struct qede_rx_queue *rxq,
struct sw_rx_data *bd, u16 len,
u16 pad)
{
unsigned int offset = bd->page_offset;
struct skb_frag_struct *frag;
struct page *page = bd->data;
unsigned int pull_len;
struct sk_buff *skb;
unsigned char *va;
/* Allocate a new SKB with a sufficient large header len */
skb = netdev_alloc_skb(edev->ndev, QEDE_RX_HDR_SIZE);
if (unlikely(!skb))
return NULL;
/* Copy data into SKB - if it's small, we can simply copy it and
* re-use the already allcoated & mapped memory.
*/
if (len + pad <= edev->rx_copybreak) {
memcpy(skb_put(skb, len),
page_address(page) + pad + offset, len);
qede_reuse_page(rxq, bd);
goto out;
}
frag = &skb_shinfo(skb)->frags[0];
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
page, pad + offset, len, rxq->rx_buf_seg_size);
va = skb_frag_address(frag);
pull_len = eth_get_headlen(va, QEDE_RX_HDR_SIZE);
/* Align the pull_len to optimize memcpy */
memcpy(skb->data, va, ALIGN(pull_len, sizeof(long)));
/* Correct the skb & frag sizes offset after the pull */
skb_frag_size_sub(frag, pull_len);
frag->page_offset += pull_len;
skb->data_len -= pull_len;
skb->tail += pull_len;
if (unlikely(qede_realloc_rx_buffer(rxq, bd))) {
/* Incr page ref count to reuse on allocation failure so
* that it doesn't get freed while freeing SKB [as its
* already mapped there].
*/
page_ref_inc(page);
dev_kfree_skb_any(skb);
return NULL;
}
out:
/* We've consumed the first BD and prepared an SKB */
qede_rx_bd_ring_consume(rxq);
return skb;
}
static int qede_rx_build_jumbo(struct qede_dev *edev,
struct qede_rx_queue *rxq,
struct sk_buff *skb,
struct eth_fast_path_rx_reg_cqe *cqe,
u16 first_bd_len)
{
u16 pkt_len = le16_to_cpu(cqe->pkt_len);
struct sw_rx_data *bd;
u16 bd_cons_idx;
u8 num_frags;
pkt_len -= first_bd_len;
/* We've already used one BD for the SKB. Now take care of the rest */
for (num_frags = cqe->bd_num - 1; num_frags > 0; num_frags--) {
u16 cur_size = pkt_len > rxq->rx_buf_size ? rxq->rx_buf_size :
pkt_len;
if (unlikely(!cur_size)) {
DP_ERR(edev,
"Still got %d BDs for mapping jumbo, but length became 0\n",
num_frags);
goto out;
}
/* Copy data into SKB */
if (len + pad <= edev->rx_copybreak) {
memcpy(skb_put(skb, len),
page_address(data) + pad +
sw_rx_data->page_offset, len);
qede_reuse_page(edev, rxq, sw_rx_data);
/* We need a replacement buffer for each BD */
if (unlikely(qede_alloc_rx_buffer(rxq)))
goto out;
/* Now that we've allocated the replacement buffer,
* we can safely consume the next BD and map it to the SKB.
*/
bd_cons_idx = rxq->sw_rx_cons & NUM_RX_BDS_MAX;
bd = &rxq->sw_rx_ring[bd_cons_idx];
qede_rx_bd_ring_consume(rxq);
dma_unmap_page(rxq->dev, bd->mapping,
PAGE_SIZE, DMA_FROM_DEVICE);
skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
bd->data, 0, cur_size);
skb->truesize += PAGE_SIZE;
skb->data_len += cur_size;
skb->len += cur_size;
pkt_len -= cur_size;
}
if (unlikely(pkt_len))
DP_ERR(edev,
"Mapped all BDs of jumbo, but still have %d bytes\n",
pkt_len);
out:
return num_frags;
}
static int qede_rx_process_tpa_cqe(struct qede_dev *edev,
struct qede_fastpath *fp,
struct qede_rx_queue *rxq,
union eth_rx_cqe *cqe,
enum eth_rx_cqe_type type)
{
switch (type) {
case ETH_RX_CQE_TYPE_TPA_START:
qede_tpa_start(edev, rxq, &cqe->fast_path_tpa_start);
return 0;
case ETH_RX_CQE_TYPE_TPA_CONT:
qede_tpa_cont(edev, rxq, &cqe->fast_path_tpa_cont);
return 0;
case ETH_RX_CQE_TYPE_TPA_END:
qede_tpa_end(edev, fp, &cqe->fast_path_tpa_end);
return 1;
default:
return 0;
}
}
static int qede_rx_process_cqe(struct qede_dev *edev,
struct qede_fastpath *fp,
struct qede_rx_queue *rxq)
{
struct bpf_prog *xdp_prog = READ_ONCE(rxq->xdp_prog);
struct eth_fast_path_rx_reg_cqe *fp_cqe;
u16 len, pad, bd_cons_idx, parse_flag;
enum eth_rx_cqe_type cqe_type;
union eth_rx_cqe *cqe;
struct sw_rx_data *bd;
struct sk_buff *skb;
__le16 flags;
u8 csum_flag;
/* Get the CQE from the completion ring */
cqe = (union eth_rx_cqe *)qed_chain_consume(&rxq->rx_comp_ring);
cqe_type = cqe->fast_path_regular.type;
/* Process an unlikely slowpath event */
if (unlikely(cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH)) {
struct eth_slow_path_rx_cqe *sp_cqe;
sp_cqe = (struct eth_slow_path_rx_cqe *)cqe;
edev->ops->eth_cqe_completion(edev->cdev, fp->id, sp_cqe);
return 0;
}
/* Handle TPA cqes */
if (cqe_type != ETH_RX_CQE_TYPE_REGULAR)
return qede_rx_process_tpa_cqe(edev, fp, rxq, cqe, cqe_type);
/* Get the data from the SW ring; Consume it only after it's evident
* we wouldn't recycle it.
*/
bd_cons_idx = rxq->sw_rx_cons & NUM_RX_BDS_MAX;
bd = &rxq->sw_rx_ring[bd_cons_idx];
fp_cqe = &cqe->fast_path_regular;
len = le16_to_cpu(fp_cqe->len_on_first_bd);
pad = fp_cqe->placement_offset;
/* Run eBPF program if one is attached */
if (xdp_prog)
if (!qede_rx_xdp(edev, fp, rxq, xdp_prog, bd, fp_cqe))
return 1;
/* If this is an error packet then drop it */
flags = cqe->fast_path_regular.pars_flags.flags;
parse_flag = le16_to_cpu(flags);
csum_flag = qede_check_csum(parse_flag);
if (unlikely(csum_flag == QEDE_CSUM_ERROR)) {
if (qede_pkt_is_ip_fragmented(fp_cqe, parse_flag)) {
rxq->rx_ip_frags++;
} else {
struct skb_frag_struct *frag;
unsigned int pull_len;
unsigned char *va;
frag = &skb_shinfo(skb)->frags[0];
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, data,
pad + sw_rx_data->page_offset,
len, rxq->rx_buf_seg_size);
va = skb_frag_address(frag);
pull_len = eth_get_headlen(va, QEDE_RX_HDR_SIZE);
/* Align the pull_len to optimize memcpy */
memcpy(skb->data, va, ALIGN(pull_len, sizeof(long)));
skb_frag_size_sub(frag, pull_len);
frag->page_offset += pull_len;
skb->data_len -= pull_len;
skb->tail += pull_len;
if (unlikely(qede_realloc_rx_buffer(edev, rxq,
sw_rx_data))) {
DP_ERR(edev, "Failed to allocate rx buffer\n");
/* Incr page ref count to reuse on allocation
* failure so that it doesn't get freed while
* freeing SKB.
*/
page_ref_inc(sw_rx_data->data);
rxq->rx_alloc_errors++;
qede_recycle_rx_bd_ring(rxq, edev,
fp_cqe->bd_num);
dev_kfree_skb_any(skb);
goto next_cqe;
}
DP_NOTICE(edev,
"CQE has error, flags = %x, dropping incoming packet\n",
parse_flag);
rxq->rx_hw_errors++;
qede_recycle_rx_bd_ring(rxq, fp_cqe->bd_num);
return 0;
}
}
qede_rx_bd_ring_consume(rxq);
/* Basic validation passed; Need to prepare an SKB. This would also
* guarantee to finally consume the first BD upon success.
*/
skb = qede_rx_allocate_skb(edev, rxq, bd, len, pad);
if (!skb) {
rxq->rx_alloc_errors++;
qede_recycle_rx_bd_ring(rxq, fp_cqe->bd_num);
return 0;
}
if (fp_cqe->bd_num != 1) {
u16 pkt_len = le16_to_cpu(fp_cqe->pkt_len);
u8 num_frags;
pkt_len -= len;
for (num_frags = fp_cqe->bd_num - 1; num_frags > 0;
num_frags--) {
u16 cur_size = pkt_len > rxq->rx_buf_size ?
rxq->rx_buf_size : pkt_len;
if (unlikely(!cur_size)) {
DP_ERR(edev,
"Still got %d BDs for mapping jumbo, but length became 0\n",
num_frags);
qede_recycle_rx_bd_ring(rxq, edev,
num_frags);
dev_kfree_skb_any(skb);
goto next_cqe;
}
if (unlikely(qede_alloc_rx_buffer(edev, rxq))) {
qede_recycle_rx_bd_ring(rxq, edev,
num_frags);
dev_kfree_skb_any(skb);
goto next_cqe;
}
sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS_MAX;
sw_rx_data = &rxq->sw_rx_ring[sw_rx_index];
qede_rx_bd_ring_consume(rxq);
dma_unmap_page(&edev->pdev->dev,
sw_rx_data->mapping,
PAGE_SIZE, DMA_FROM_DEVICE);
skb_fill_page_desc(skb,
skb_shinfo(skb)->nr_frags++,
sw_rx_data->data, 0,
cur_size);
skb->truesize += PAGE_SIZE;
skb->data_len += cur_size;
skb->len += cur_size;
pkt_len -= cur_size;
}
/* In case of Jumbo packet, several PAGE_SIZEd buffers will be pointed
* by a single cqe.
*/
if (fp_cqe->bd_num > 1) {
u16 unmapped_frags = qede_rx_build_jumbo(edev, rxq, skb,
fp_cqe, len);
if (unlikely(pkt_len))
DP_ERR(edev,
"Mapped all BDs of jumbo, but still have %d bytes\n",
pkt_len);
if (unlikely(unmapped_frags > 0)) {
qede_recycle_rx_bd_ring(rxq, unmapped_frags);
dev_kfree_skb_any(skb);
return 0;
}
}
skb->protocol = eth_type_trans(skb, edev->ndev);
/* The SKB contains all the data. Now prepare meta-magic */
skb->protocol = eth_type_trans(skb, edev->ndev);
qede_get_rxhash(skb, fp_cqe->bitfields, fp_cqe->rss_hash);
qede_set_skb_csum(skb, csum_flag);
skb_record_rx_queue(skb, rxq->rxq_id);
rx_hash = qede_get_rxhash(edev, fp_cqe->bitfields,
fp_cqe->rss_hash, &rxhash_type);
/* SKB is prepared - pass it to stack */
qede_skb_receive(edev, fp, rxq, skb, le16_to_cpu(fp_cqe->vlan_tag));
skb_set_hash(skb, rx_hash, rxhash_type);
return 1;
}
qede_set_skb_csum(skb, csum_flag);
static int qede_rx_int(struct qede_fastpath *fp, int budget)
{
struct qede_rx_queue *rxq = fp->rxq;
struct qede_dev *edev = fp->edev;
u16 hw_comp_cons, sw_comp_cons;
int work_done = 0;
skb_record_rx_queue(skb, fp->rxq->rxq_id);
hw_comp_cons = le16_to_cpu(*rxq->hw_cons_ptr);
sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring);
qede_skb_receive(edev, fp, skb, le16_to_cpu(fp_cqe->vlan_tag));
next_rx_only:
rx_pkt++;
/* Memory barrier to prevent the CPU from doing speculative reads of CQE
* / BD in the while-loop before reading hw_comp_cons. If the CQE is
* read before it is written by FW, then FW writes CQE and SB, and then
* the CPU reads the hw_comp_cons, it will use an old CQE.
*/
rmb();
next_cqe: /* don't consume bd rx buffer */
/* Loop to complete all indicated BDs */
while ((sw_comp_cons != hw_comp_cons) && (work_done < budget)) {
qede_rx_process_cqe(edev, fp, rxq);
qed_chain_recycle_consumed(&rxq->rx_comp_ring);
sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring);
/* CR TPA - revisit how to handle budget in TPA perhaps
* increase on "end"
*/
if (rx_pkt == budget)
break;
} /* repeat while sw_comp_cons != hw_comp_cons... */
work_done++;
}
/* Update producers */
qede_update_rx_prod(edev, rxq);
rxq->rcv_pkts += rx_pkt;
return work_done;
}
return rx_pkt;
static bool qede_poll_is_more_work(struct qede_fastpath *fp)
{
qed_sb_update_sb_idx(fp->sb_info);
/* *_has_*_work() reads the status block, thus we need to ensure that
* status block indices have been actually read (qed_sb_update_sb_idx)
* prior to this check (*_has_*_work) so that we won't write the
* "newer" value of the status block to HW (if there was a DMA right
* after qede_has_rx_work and if there is no rmb, the memory reading
* (qed_sb_update_sb_idx) may be postponed to right before *_ack_sb).
* In this case there will never be another interrupt until there is
* another update of the status block, while there is still unhandled
* work.
*/
rmb();
if (likely(fp->type & QEDE_FASTPATH_RX))
if (qede_has_rx_work(fp->rxq))
return true;
if (fp->type & QEDE_FASTPATH_XDP)
if (qede_txq_has_work(fp->xdp_tx))
return true;
if (likely(fp->type & QEDE_FASTPATH_TX))
if (qede_txq_has_work(fp->txq))
return true;
return false;
}
static int qede_poll(struct napi_struct *napi, int budget)
......@@ -1643,48 +1842,35 @@ static int qede_poll(struct napi_struct *napi, int budget)
napi);
struct qede_dev *edev = fp->edev;
int rx_work_done = 0;
u8 tc;
for (tc = 0; tc < edev->num_tc; tc++)
if (likely(fp->type & QEDE_FASTPATH_TX) &&
qede_txq_has_work(&fp->txqs[tc]))
qede_tx_int(edev, &fp->txqs[tc]);
if (likely(fp->type & QEDE_FASTPATH_TX) && qede_txq_has_work(fp->txq))
qede_tx_int(edev, fp->txq);
if ((fp->type & QEDE_FASTPATH_XDP) && qede_txq_has_work(fp->xdp_tx))
qede_xdp_tx_int(edev, fp->xdp_tx);
rx_work_done = (likely(fp->type & QEDE_FASTPATH_RX) &&
qede_has_rx_work(fp->rxq)) ?
qede_rx_int(fp, budget) : 0;
if (rx_work_done < budget) {
qed_sb_update_sb_idx(fp->sb_info);
/* *_has_*_work() reads the status block,
* thus we need to ensure that status block indices
* have been actually read (qed_sb_update_sb_idx)
* prior to this check (*_has_*_work) so that
* we won't write the "newer" value of the status block
* to HW (if there was a DMA right after
* qede_has_rx_work and if there is no rmb, the memory
* reading (qed_sb_update_sb_idx) may be postponed
* to right before *_ack_sb). In this case there
* will never be another interrupt until there is
* another update of the status block, while there
* is still unhandled work.
*/
rmb();
/* Fall out from the NAPI loop if needed */
if (!((likely(fp->type & QEDE_FASTPATH_RX) &&
qede_has_rx_work(fp->rxq)) ||
(likely(fp->type & QEDE_FASTPATH_TX) &&
qede_has_tx_work(fp)))) {
if (!qede_poll_is_more_work(fp)) {
napi_complete(napi);
/* Update and reenable interrupts */
qed_sb_ack(fp->sb_info, IGU_INT_ENABLE,
1 /*update*/);
qed_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
} else {
rx_work_done = budget;
}
}
if (fp->xdp_xmit) {
u16 xdp_prod = qed_chain_get_prod_idx(&fp->xdp_tx->tx_pbl);
fp->xdp_xmit = 0;
fp->xdp_tx->tx_db.data.bd_prod = cpu_to_le16(xdp_prod);
qede_update_tx_producer(fp->xdp_tx);
}
return rx_work_done;
}
......@@ -1935,7 +2121,7 @@ static int qede_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
{
struct qede_dev *edev = netdev_priv(dev);
struct qede_vlan *vlan, *tmp;
int rc;
int rc = 0;
DP_VERBOSE(edev, NETIF_MSG_IFUP, "Adding vlan 0x%04x\n", vid);
......@@ -1959,6 +2145,7 @@ static int qede_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
}
/* If interface is down, cache this VLAN ID and return */
__qede_lock(edev);
if (edev->state != QEDE_STATE_OPEN) {
DP_VERBOSE(edev, NETIF_MSG_IFDOWN,
"Interface is down, VLAN %d will be configured when interface is up\n",
......@@ -1966,8 +2153,7 @@ static int qede_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
if (vid != 0)
edev->non_configured_vlans++;
list_add(&vlan->list, &edev->vlan_list);
return 0;
goto out;
}
/* Check for the filter limit.
......@@ -1983,7 +2169,7 @@ static int qede_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
DP_ERR(edev, "Failed to configure VLAN %d\n",
vlan->vid);
kfree(vlan);
return -EINVAL;
goto out;
}
vlan->configured = true;
......@@ -2000,7 +2186,9 @@ static int qede_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
list_add(&vlan->list, &edev->vlan_list);
return 0;
out:
__qede_unlock(edev);
return rc;
}
static void qede_del_vlan_from_list(struct qede_dev *edev,
......@@ -2077,11 +2265,12 @@ static int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
{
struct qede_dev *edev = netdev_priv(dev);
struct qede_vlan *vlan = NULL;
int rc;
int rc = 0;
DP_VERBOSE(edev, NETIF_MSG_IFDOWN, "Removing vlan 0x%04x\n", vid);
/* Find whether entry exists */
__qede_lock(edev);
list_for_each_entry(vlan, &edev->vlan_list, list)
if (vlan->vid == vid)
break;
......@@ -2089,7 +2278,7 @@ static int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
if (!vlan || (vlan->vid != vid)) {
DP_VERBOSE(edev, (NETIF_MSG_IFUP | NETIF_MSG_IFDOWN),
"Vlan isn't configured\n");
return 0;
goto out;
}
if (edev->state != QEDE_STATE_OPEN) {
......@@ -2099,7 +2288,7 @@ static int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
DP_VERBOSE(edev, NETIF_MSG_IFDOWN,
"Interface is down, removing VLAN from list only\n");
qede_del_vlan_from_list(edev, vlan);
return 0;
goto out;
}
/* Remove vlan */
......@@ -2108,7 +2297,7 @@ static int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
vid);
if (rc) {
DP_ERR(edev, "Failed to remove VLAN %d\n", vid);
return -EINVAL;
goto out;
}
}
......@@ -2119,6 +2308,8 @@ static int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
*/
rc = qede_configure_vlan_filters(edev);
out:
__qede_unlock(edev);
return rc;
}
......@@ -2148,7 +2339,13 @@ static void qede_vlan_mark_nonconfigured(struct qede_dev *edev)
edev->accept_any_vlan = false;
}
static int qede_set_features(struct net_device *dev, netdev_features_t features)
static void qede_set_features_reload(struct qede_dev *edev,
struct qede_reload_args *args)
{
edev->ndev->features = args->u.features;
}
int qede_set_features(struct net_device *dev, netdev_features_t features)
{
struct qede_dev *edev = netdev_priv(dev);
netdev_features_t changes = features ^ dev->features;
......@@ -2162,9 +2359,23 @@ static int qede_set_features(struct net_device *dev, netdev_features_t features)
need_reload = edev->gro_disable;
}
if (need_reload && netif_running(edev->ndev)) {
dev->features = features;
qede_reload(edev, NULL, NULL);
if (need_reload) {
struct qede_reload_args args;
args.u.features = features;
args.func = &qede_set_features_reload;
/* Make sure that we definitely need to reload.
* In case of an eBPF attached program, there will be no FW
* aggregations, so no need to actually reload.
*/
__qede_lock(edev);
if (edev->xdp_prog)
args.func(edev, &args);
else
qede_reload(edev, &args, true);
__qede_unlock(edev);
return 1;
}
......@@ -2275,6 +2486,43 @@ static netdev_features_t qede_features_check(struct sk_buff *skb,
return features;
}
static void qede_xdp_reload_func(struct qede_dev *edev,
struct qede_reload_args *args)
{
struct bpf_prog *old;
old = xchg(&edev->xdp_prog, args->u.new_prog);
if (old)
bpf_prog_put(old);
}
static int qede_xdp_set(struct qede_dev *edev, struct bpf_prog *prog)
{
struct qede_reload_args args;
/* If we're called, there was already a bpf reference increment */
args.func = &qede_xdp_reload_func;
args.u.new_prog = prog;
qede_reload(edev, &args, false);
return 0;
}
static int qede_xdp(struct net_device *dev, struct netdev_xdp *xdp)
{
struct qede_dev *edev = netdev_priv(dev);
switch (xdp->command) {
case XDP_SETUP_PROG:
return qede_xdp_set(edev, xdp->prog);
case XDP_QUERY_PROG:
xdp->prog_attached = !!edev->xdp_prog;
return 0;
default:
return -EINVAL;
}
}
static const struct net_device_ops qede_netdev_ops = {
.ndo_open = qede_open,
.ndo_stop = qede_close,
......@@ -2300,6 +2548,7 @@ static const struct net_device_ops qede_netdev_ops = {
.ndo_udp_tunnel_add = qede_udp_tunnel_add,
.ndo_udp_tunnel_del = qede_udp_tunnel_del,
.ndo_features_check = qede_features_check,
.ndo_xdp = qede_xdp,
};
/* -------------------------------------------------------------------------
......@@ -2340,8 +2589,6 @@ static struct qede_dev *qede_alloc_etherdev(struct qed_dev *cdev,
memset(&edev->stats, 0, sizeof(edev->stats));
memcpy(&edev->dev_info, info, sizeof(*info));
edev->num_tc = edev->dev_info.num_tc;
INIT_LIST_HEAD(&edev->vlan_list);
return edev;
......@@ -2439,7 +2686,8 @@ static void qede_free_fp_array(struct qede_dev *edev)
kfree(fp->sb_info);
kfree(fp->rxq);
kfree(fp->txqs);
kfree(fp->xdp_tx);
kfree(fp->txq);
}
kfree(edev->fp_array);
}
......@@ -2472,7 +2720,7 @@ static int qede_alloc_fp_array(struct qede_dev *edev)
for_each_queue(i) {
fp = &edev->fp_array[i];
fp->sb_info = kcalloc(1, sizeof(*fp->sb_info), GFP_KERNEL);
fp->sb_info = kzalloc(sizeof(*fp->sb_info), GFP_KERNEL);
if (!fp->sb_info) {
DP_NOTICE(edev, "sb info struct allocation failed\n");
goto err;
......@@ -2489,21 +2737,22 @@ static int qede_alloc_fp_array(struct qede_dev *edev)
}
if (fp->type & QEDE_FASTPATH_TX) {
fp->txqs = kcalloc(edev->num_tc, sizeof(*fp->txqs),
GFP_KERNEL);
if (!fp->txqs) {
DP_NOTICE(edev,
"TXQ array allocation failed\n");
fp->txq = kzalloc(sizeof(*fp->txq), GFP_KERNEL);
if (!fp->txq)
goto err;
}
}
if (fp->type & QEDE_FASTPATH_RX) {
fp->rxq = kcalloc(1, sizeof(*fp->rxq), GFP_KERNEL);
if (!fp->rxq) {
DP_NOTICE(edev,
"RXQ struct allocation failed\n");
fp->rxq = kzalloc(sizeof(*fp->rxq), GFP_KERNEL);
if (!fp->rxq)
goto err;
if (edev->xdp_prog) {
fp->xdp_tx = kzalloc(sizeof(*fp->xdp_tx),
GFP_KERNEL);
if (!fp->xdp_tx)
goto err;
fp->type |= QEDE_FASTPATH_XDP;
}
}
}
......@@ -2520,12 +2769,11 @@ static void qede_sp_task(struct work_struct *work)
sp_task.work);
struct qed_dev *cdev = edev->cdev;
mutex_lock(&edev->qede_lock);
__qede_lock(edev);
if (edev->state == QEDE_STATE_OPEN) {
if (test_and_clear_bit(QEDE_SP_RX_MODE, &edev->sp_flags))
if (test_and_clear_bit(QEDE_SP_RX_MODE, &edev->sp_flags))
if (edev->state == QEDE_STATE_OPEN)
qede_config_rx_mode(edev->ndev);
}
if (test_and_clear_bit(QEDE_SP_VXLAN_PORT_CONFIG, &edev->sp_flags)) {
struct qed_tunn_params tunn_params;
......@@ -2545,16 +2793,16 @@ static void qede_sp_task(struct work_struct *work)
qed_ops->tunn_config(cdev, &tunn_params);
}
mutex_unlock(&edev->qede_lock);
__qede_unlock(edev);
}
static void qede_update_pf_params(struct qed_dev *cdev)
{
struct qed_pf_params pf_params;
/* 64 rx + 64 tx */
/* 64 rx + 64 tx + 64 XDP */
memset(&pf_params, 0, sizeof(struct qed_pf_params));
pf_params.eth_pf_params.num_cons = 128;
pf_params.eth_pf_params.num_cons = 192;
qed_ops->common->update_pf_params(cdev, &pf_params);
}
......@@ -2703,6 +2951,10 @@ static void __qede_remove(struct pci_dev *pdev, enum qede_remove_mode mode)
pci_set_drvdata(pdev, NULL);
/* Release edev's reference to XDP's bpf if such exist */
if (edev->xdp_prog)
bpf_prog_put(edev->xdp_prog);
free_netdev(ndev);
/* Use global ops since we've freed edev */
......@@ -2807,7 +3059,7 @@ static void qede_free_rx_buffers(struct qede_dev *edev,
data = rx_buf->data;
dma_unmap_page(&edev->pdev->dev,
rx_buf->mapping, PAGE_SIZE, DMA_FROM_DEVICE);
rx_buf->mapping, PAGE_SIZE, rxq->data_direction);
rx_buf->data = NULL;
__free_page(data);
......@@ -2823,7 +3075,7 @@ static void qede_free_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
struct qede_agg_info *tpa_info = &rxq->tpa_info[i];
struct sw_rx_data *replace_buf = &tpa_info->replace_buf;
struct sw_rx_data *replace_buf = &tpa_info->buffer;
if (replace_buf->data) {
dma_unmap_page(&edev->pdev->dev,
......@@ -2849,52 +3101,15 @@ static void qede_free_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
edev->ops->common->chain_free(edev->cdev, &rxq->rx_comp_ring);
}
static int qede_alloc_rx_buffer(struct qede_dev *edev,
struct qede_rx_queue *rxq)
{
struct sw_rx_data *sw_rx_data;
struct eth_rx_bd *rx_bd;
dma_addr_t mapping;
struct page *data;
data = alloc_pages(GFP_ATOMIC, 0);
if (unlikely(!data)) {
DP_NOTICE(edev, "Failed to allocate Rx data [page]\n");
return -ENOMEM;
}
/* Map the entire page as it would be used
* for multiple RX buffer segment size mapping.
*/
mapping = dma_map_page(&edev->pdev->dev, data, 0,
PAGE_SIZE, DMA_FROM_DEVICE);
if (unlikely(dma_mapping_error(&edev->pdev->dev, mapping))) {
__free_page(data);
DP_NOTICE(edev, "Failed to map Rx buffer\n");
return -ENOMEM;
}
sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_prod & NUM_RX_BDS_MAX];
sw_rx_data->page_offset = 0;
sw_rx_data->data = data;
sw_rx_data->mapping = mapping;
/* Advance PROD and get BD pointer */
rx_bd = (struct eth_rx_bd *)qed_chain_produce(&rxq->rx_bd_ring);
WARN_ON(!rx_bd);
rx_bd->addr.hi = cpu_to_le32(upper_32_bits(mapping));
rx_bd->addr.lo = cpu_to_le32(lower_32_bits(mapping));
rxq->sw_rx_prod++;
return 0;
}
static int qede_alloc_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
{
dma_addr_t mapping;
int i;
/* Don't perform FW aggregations in case of XDP */
if (edev->xdp_prog)
edev->gro_disable = 1;
if (edev->gro_disable)
return 0;
......@@ -2905,7 +3120,7 @@ static int qede_alloc_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
struct qede_agg_info *tpa_info = &rxq->tpa_info[i];
struct sw_rx_data *replace_buf = &tpa_info->replace_buf;
struct sw_rx_data *replace_buf = &tpa_info->buffer;
replace_buf->data = alloc_pages(GFP_ATOMIC, 0);
if (unlikely(!replace_buf->data)) {
......@@ -2923,10 +3138,9 @@ static int qede_alloc_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
}
replace_buf->mapping = mapping;
tpa_info->replace_buf.page_offset = 0;
tpa_info->replace_buf_mapping = mapping;
tpa_info->agg_state = QEDE_AGG_STATE_NONE;
tpa_info->buffer.page_offset = 0;
tpa_info->buffer_mapping = mapping;
tpa_info->state = QEDE_AGG_STATE_NONE;
}
return 0;
......@@ -2948,8 +3162,13 @@ static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
if (rxq->rx_buf_size > PAGE_SIZE)
rxq->rx_buf_size = PAGE_SIZE;
/* Segment size to spilt a page in multiple equal parts */
rxq->rx_buf_seg_size = roundup_pow_of_two(rxq->rx_buf_size);
/* Segment size to spilt a page in multiple equal parts,
* unless XDP is used in which case we'd use the entire page.
*/
if (!edev->xdp_prog)
rxq->rx_buf_seg_size = roundup_pow_of_two(rxq->rx_buf_size);
else
rxq->rx_buf_seg_size = PAGE_SIZE;
/* Allocate the parallel driver ring for Rx buffers */
size = sizeof(*rxq->sw_rx_ring) * RX_RING_SIZE;
......@@ -2985,7 +3204,7 @@ static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
/* Allocate buffers for the Rx ring */
for (i = 0; i < rxq->num_rx_buffers; i++) {
rc = qede_alloc_rx_buffer(edev, rxq);
rc = qede_alloc_rx_buffer(rxq);
if (rc) {
DP_ERR(edev,
"Rx buffers allocation failed at index %d\n", i);
......@@ -3001,7 +3220,10 @@ static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
static void qede_free_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
{
/* Free the parallel SW ring */
kfree(txq->sw_tx_ring);
if (txq->is_xdp)
kfree(txq->sw_tx_ring.pages);
else
kfree(txq->sw_tx_ring.skbs);
/* Free the real RQ ring used by FW */
edev->ops->common->chain_free(edev->cdev, &txq->tx_pbl);
......@@ -3010,17 +3232,22 @@ static void qede_free_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
/* This function allocates all memory needed per Tx queue */
static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
{
int size, rc;
union eth_tx_bd_types *p_virt;
int size, rc;
txq->num_tx_buffers = edev->q_num_tx_buffers;
/* Allocate the parallel driver ring for Tx buffers */
size = sizeof(*txq->sw_tx_ring) * TX_RING_SIZE;
txq->sw_tx_ring = kzalloc(size, GFP_KERNEL);
if (!txq->sw_tx_ring) {
DP_NOTICE(edev, "Tx buffers ring allocation failed\n");
goto err;
if (txq->is_xdp) {
size = sizeof(*txq->sw_tx_ring.pages) * TX_RING_SIZE;
txq->sw_tx_ring.pages = kzalloc(size, GFP_KERNEL);
if (!txq->sw_tx_ring.pages)
goto err;
} else {
size = sizeof(*txq->sw_tx_ring.skbs) * TX_RING_SIZE;
txq->sw_tx_ring.skbs = kzalloc(size, GFP_KERNEL);
if (!txq->sw_tx_ring.skbs)
goto err;
}
rc = edev->ops->common->chain_alloc(edev->cdev,
......@@ -3042,16 +3269,13 @@ static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
/* This function frees all memory of a single fp */
static void qede_free_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
{
int tc;
qede_free_mem_sb(edev, fp->sb_info);
if (fp->type & QEDE_FASTPATH_RX)
qede_free_mem_rxq(edev, fp->rxq);
if (fp->type & QEDE_FASTPATH_TX)
for (tc = 0; tc < edev->num_tc; tc++)
qede_free_mem_txq(edev, &fp->txqs[tc]);
qede_free_mem_txq(edev, fp->txq);
}
/* This function allocates all memory needed for a single fp (i.e. an entity
......@@ -3059,28 +3283,31 @@ static void qede_free_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
*/
static int qede_alloc_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
{
int rc, tc;
int rc = 0;
rc = qede_alloc_mem_sb(edev, fp->sb_info, fp->id);
if (rc)
goto err;
goto out;
if (fp->type & QEDE_FASTPATH_RX) {
rc = qede_alloc_mem_rxq(edev, fp->rxq);
if (rc)
goto err;
goto out;
}
if (fp->type & QEDE_FASTPATH_XDP) {
rc = qede_alloc_mem_txq(edev, fp->xdp_tx);
if (rc)
goto out;
}
if (fp->type & QEDE_FASTPATH_TX) {
for (tc = 0; tc < edev->num_tc; tc++) {
rc = qede_alloc_mem_txq(edev, &fp->txqs[tc]);
if (rc)
goto err;
}
rc = qede_alloc_mem_txq(edev, fp->txq);
if (rc)
goto out;
}
return 0;
err:
out:
return rc;
}
......@@ -3119,7 +3346,7 @@ static int qede_alloc_mem_load(struct qede_dev *edev)
/* This function inits fp content and resets the SB, RXQ and TXQ structures */
static void qede_init_fp(struct qede_dev *edev)
{
int queue_id, rxq_index = 0, txq_index = 0, tc;
int queue_id, rxq_index = 0, txq_index = 0;
struct qede_fastpath *fp;
for_each_queue(queue_id) {
......@@ -3128,25 +3355,28 @@ static void qede_init_fp(struct qede_dev *edev)
fp->edev = edev;
fp->id = queue_id;
memset((void *)&fp->napi, 0, sizeof(fp->napi));
memset((void *)fp->sb_info, 0, sizeof(*fp->sb_info));
if (fp->type & QEDE_FASTPATH_XDP) {
fp->xdp_tx->index = QEDE_TXQ_IDX_TO_XDP(edev,
rxq_index);
fp->xdp_tx->is_xdp = 1;
}
if (fp->type & QEDE_FASTPATH_RX) {
memset((void *)fp->rxq, 0, sizeof(*fp->rxq));
fp->rxq->rxq_id = rxq_index++;
/* Determine how to map buffers for this queue */
if (fp->type & QEDE_FASTPATH_XDP)
fp->rxq->data_direction = DMA_BIDIRECTIONAL;
else
fp->rxq->data_direction = DMA_FROM_DEVICE;
fp->rxq->dev = &edev->pdev->dev;
}
if (fp->type & QEDE_FASTPATH_TX) {
memset((void *)fp->txqs, 0,
(edev->num_tc * sizeof(*fp->txqs)));
for (tc = 0; tc < edev->num_tc; tc++) {
fp->txqs[tc].index = txq_index +
tc * QEDE_TSS_COUNT(edev);
if (edev->dev_info.is_legacy)
fp->txqs[tc].is_legacy = true;
}
txq_index++;
fp->txq->index = txq_index++;
if (edev->dev_info.is_legacy)
fp->txq->is_legacy = 1;
fp->txq->dev = &edev->pdev->dev;
}
snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
......@@ -3314,11 +3544,18 @@ static int qede_drain_txq(struct qede_dev *edev,
return 0;
}
static int qede_stop_txq(struct qede_dev *edev,
struct qede_tx_queue *txq, int rss_id)
{
return edev->ops->q_tx_stop(edev->cdev, rss_id, txq->handle);
}
static int qede_stop_queues(struct qede_dev *edev)
{
struct qed_update_vport_params vport_update_params;
struct qed_dev *cdev = edev->cdev;
int rc, tc, i;
struct qede_fastpath *fp;
int rc, i;
/* Disable the vport */
memset(&vport_update_params, 0, sizeof(vport_update_params));
......@@ -3335,53 +3572,49 @@ static int qede_stop_queues(struct qede_dev *edev)
/* Flush Tx queues. If needed, request drain from MCP */
for_each_queue(i) {
struct qede_fastpath *fp = &edev->fp_array[i];
fp = &edev->fp_array[i];
if (fp->type & QEDE_FASTPATH_TX) {
for (tc = 0; tc < edev->num_tc; tc++) {
struct qede_tx_queue *txq = &fp->txqs[tc];
rc = qede_drain_txq(edev, fp->txq, true);
if (rc)
return rc;
}
rc = qede_drain_txq(edev, txq, true);
if (rc)
return rc;
}
if (fp->type & QEDE_FASTPATH_XDP) {
rc = qede_drain_txq(edev, fp->xdp_tx, true);
if (rc)
return rc;
}
}
/* Stop all Queues in reverse order */
for (i = QEDE_QUEUE_CNT(edev) - 1; i >= 0; i--) {
struct qed_stop_rxq_params rx_params;
fp = &edev->fp_array[i];
/* Stop the Tx Queue(s) */
if (edev->fp_array[i].type & QEDE_FASTPATH_TX) {
for (tc = 0; tc < edev->num_tc; tc++) {
struct qed_stop_txq_params tx_params;
u8 val;
tx_params.rss_id = i;
val = edev->fp_array[i].txqs[tc].index;
tx_params.tx_queue_id = val;
rc = edev->ops->q_tx_stop(cdev, &tx_params);
if (rc) {
DP_ERR(edev, "Failed to stop TXQ #%d\n",
tx_params.tx_queue_id);
return rc;
}
}
if (fp->type & QEDE_FASTPATH_TX) {
rc = qede_stop_txq(edev, fp->txq, i);
if (rc)
return rc;
}
/* Stop the Rx Queue */
if (edev->fp_array[i].type & QEDE_FASTPATH_RX) {
memset(&rx_params, 0, sizeof(rx_params));
rx_params.rss_id = i;
rx_params.rx_queue_id = edev->fp_array[i].rxq->rxq_id;
rc = edev->ops->q_rx_stop(cdev, &rx_params);
if (fp->type & QEDE_FASTPATH_RX) {
rc = edev->ops->q_rx_stop(cdev, i, fp->rxq->handle);
if (rc) {
DP_ERR(edev, "Failed to stop RXQ #%d\n", i);
return rc;
}
}
/* Stop the XDP forwarding queue */
if (fp->type & QEDE_FASTPATH_XDP) {
rc = qede_stop_txq(edev, fp->xdp_tx, i);
if (rc)
return rc;
bpf_prog_put(fp->rxq->xdp_prog);
}
}
/* Stop the vport */
......@@ -3392,9 +3625,55 @@ static int qede_stop_queues(struct qede_dev *edev)
return rc;
}
static int qede_start_txq(struct qede_dev *edev,
struct qede_fastpath *fp,
struct qede_tx_queue *txq, u8 rss_id, u16 sb_idx)
{
dma_addr_t phys_table = qed_chain_get_pbl_phys(&txq->tx_pbl);
u32 page_cnt = qed_chain_get_page_cnt(&txq->tx_pbl);
struct qed_queue_start_common_params params;
struct qed_txq_start_ret_params ret_params;
int rc;
memset(&params, 0, sizeof(params));
memset(&ret_params, 0, sizeof(ret_params));
/* Let the XDP queue share the queue-zone with one of the regular txq.
* We don't really care about its coalescing.
*/
if (txq->is_xdp)
params.queue_id = QEDE_TXQ_XDP_TO_IDX(edev, txq);
else
params.queue_id = txq->index;
params.sb = fp->sb_info->igu_sb_id;
params.sb_idx = sb_idx;
rc = edev->ops->q_tx_start(edev->cdev, rss_id, &params, phys_table,
page_cnt, &ret_params);
if (rc) {
DP_ERR(edev, "Start TXQ #%d failed %d\n", txq->index, rc);
return rc;
}
txq->doorbell_addr = ret_params.p_doorbell;
txq->handle = ret_params.p_handle;
/* Determine the FW consumer address associated */
txq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[sb_idx];
/* Prepare the doorbell parameters */
SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_DEST, DB_DEST_XCM);
SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, DB_AGG_CMD_SET);
SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_VAL_SEL,
DQ_XCM_ETH_TX_BD_PROD_CMD);
txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
return rc;
}
static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
{
int rc, tc, i;
int vlan_removal_en = 1;
struct qed_dev *cdev = edev->cdev;
struct qed_update_vport_params vport_update_params;
......@@ -3402,6 +3681,7 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
struct qed_dev_info *qed_info = &edev->dev_info.common;
struct qed_start_vport_params start = {0};
bool reset_rss_indir = false;
int rc, i;
if (!edev->num_queues) {
DP_ERR(edev,
......@@ -3433,11 +3713,12 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
u32 page_cnt;
if (fp->type & QEDE_FASTPATH_RX) {
struct qed_rxq_start_ret_params ret_params;
struct qede_rx_queue *rxq = fp->rxq;
__le16 *val;
memset(&ret_params, 0, sizeof(ret_params));
memset(&q_params, 0, sizeof(q_params));
q_params.rss_id = i;
q_params.queue_id = rxq->rxq_id;
q_params.vport_id = 0;
q_params.sb = fp->sb_info->igu_sb_id;
......@@ -3447,60 +3728,44 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
qed_chain_get_pbl_phys(&rxq->rx_comp_ring);
page_cnt = qed_chain_get_page_cnt(&rxq->rx_comp_ring);
rc = edev->ops->q_rx_start(cdev, &q_params,
rc = edev->ops->q_rx_start(cdev, i, &q_params,
rxq->rx_buf_size,
rxq->rx_bd_ring.p_phys_addr,
p_phys_table,
page_cnt,
&rxq->hw_rxq_prod_addr);
page_cnt, &ret_params);
if (rc) {
DP_ERR(edev, "Start RXQ #%d failed %d\n", i,
rc);
return rc;
}
/* Use the return parameters */
rxq->hw_rxq_prod_addr = ret_params.p_prod;
rxq->handle = ret_params.p_handle;
val = &fp->sb_info->sb_virt->pi_array[RX_PI];
rxq->hw_cons_ptr = val;
qede_update_rx_prod(edev, rxq);
}
if (!(fp->type & QEDE_FASTPATH_TX))
continue;
for (tc = 0; tc < edev->num_tc; tc++) {
struct qede_tx_queue *txq = &fp->txqs[tc];
p_phys_table = qed_chain_get_pbl_phys(&txq->tx_pbl);
page_cnt = qed_chain_get_page_cnt(&txq->tx_pbl);
memset(&q_params, 0, sizeof(q_params));
q_params.rss_id = i;
q_params.queue_id = txq->index;
q_params.vport_id = 0;
q_params.sb = fp->sb_info->igu_sb_id;
q_params.sb_idx = TX_PI(tc);
if (fp->type & QEDE_FASTPATH_XDP) {
rc = qede_start_txq(edev, fp, fp->xdp_tx, i, XDP_PI);
if (rc)
return rc;
rc = edev->ops->q_tx_start(cdev, &q_params,
p_phys_table, page_cnt,
&txq->doorbell_addr);
if (rc) {
DP_ERR(edev, "Start TXQ #%d failed %d\n",
txq->index, rc);
fp->rxq->xdp_prog = bpf_prog_add(edev->xdp_prog, 1);
if (IS_ERR(fp->rxq->xdp_prog)) {
rc = PTR_ERR(fp->rxq->xdp_prog);
fp->rxq->xdp_prog = NULL;
return rc;
}
}
txq->hw_cons_ptr =
&fp->sb_info->sb_virt->pi_array[TX_PI(tc)];
SET_FIELD(txq->tx_db.data.params,
ETH_DB_DATA_DEST, DB_DEST_XCM);
SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD,
DB_AGG_CMD_SET);
SET_FIELD(txq->tx_db.data.params,
ETH_DB_DATA_AGG_VAL_SEL,
DQ_XCM_ETH_TX_BD_PROD_CMD);
txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
if (fp->type & QEDE_FASTPATH_TX) {
rc = qede_start_txq(edev, fp, fp->txq, i, TX_PI(0));
if (rc)
return rc;
}
}
......@@ -3595,15 +3860,18 @@ enum qede_unload_mode {
QEDE_UNLOAD_NORMAL,
};
static void qede_unload(struct qede_dev *edev, enum qede_unload_mode mode)
static void qede_unload(struct qede_dev *edev, enum qede_unload_mode mode,
bool is_locked)
{
struct qed_link_params link_params;
int rc;
DP_INFO(edev, "Starting qede unload\n");
if (!is_locked)
__qede_lock(edev);
qede_roce_dev_event_close(edev);
mutex_lock(&edev->qede_lock);
edev->state = QEDE_STATE_CLOSED;
/* Close OS Tx */
......@@ -3635,7 +3903,8 @@ static void qede_unload(struct qede_dev *edev, enum qede_unload_mode mode)
qede_free_fp_array(edev);
out:
mutex_unlock(&edev->qede_lock);
if (!is_locked)
__qede_unlock(edev);
DP_INFO(edev, "Ending qede unload\n");
}
......@@ -3644,7 +3913,8 @@ enum qede_load_mode {
QEDE_LOAD_RELOAD,
};
static int qede_load(struct qede_dev *edev, enum qede_load_mode mode)
static int qede_load(struct qede_dev *edev, enum qede_load_mode mode,
bool is_locked)
{
struct qed_link_params link_params;
struct qed_link_output link_output;
......@@ -3652,21 +3922,24 @@ static int qede_load(struct qede_dev *edev, enum qede_load_mode mode)
DP_INFO(edev, "Starting qede load\n");
if (!is_locked)
__qede_lock(edev);
rc = qede_set_num_queues(edev);
if (rc)
goto err0;
goto out;
rc = qede_alloc_fp_array(edev);
if (rc)
goto err0;
goto out;
qede_init_fp(edev);
rc = qede_alloc_mem_load(edev);
if (rc)
goto err1;
DP_INFO(edev, "Allocated %d RSS queues on %d TC/s\n",
QEDE_QUEUE_CNT(edev), edev->num_tc);
DP_INFO(edev, "Allocated %d Rx, %d Tx queues\n",
QEDE_RSS_COUNT(edev), QEDE_TSS_COUNT(edev));
rc = qede_set_real_num_queues(edev);
if (rc)
......@@ -3688,10 +3961,6 @@ static int qede_load(struct qede_dev *edev, enum qede_load_mode mode)
/* Add primary mac and set Rx filters */
ether_addr_copy(edev->primary_mac, edev->ndev->dev_addr);
mutex_lock(&edev->qede_lock);
edev->state = QEDE_STATE_OPEN;
mutex_unlock(&edev->qede_lock);
/* Program un-configured VLANs */
qede_configure_vlan_filters(edev);
......@@ -3706,10 +3975,12 @@ static int qede_load(struct qede_dev *edev, enum qede_load_mode mode)
qede_roce_dev_event_open(edev);
qede_link_update(edev, &link_output);
edev->state = QEDE_STATE_OPEN;
DP_INFO(edev, "Ending successfully qede load\n");
return 0;
goto out;
err4:
qede_sync_free_irqs(edev);
memset(&edev->int_info.msix_cnt, 0, sizeof(struct qed_int_info));
......@@ -3723,26 +3994,40 @@ static int qede_load(struct qede_dev *edev, enum qede_load_mode mode)
edev->num_queues = 0;
edev->fp_num_tx = 0;
edev->fp_num_rx = 0;
err0:
out:
if (!is_locked)
__qede_unlock(edev);
return rc;
}
/* 'func' should be able to run between unload and reload assuming interface
* is actually running, or afterwards in case it's currently DOWN.
*/
void qede_reload(struct qede_dev *edev,
void (*func)(struct qede_dev *, union qede_reload_args *),
union qede_reload_args *args)
struct qede_reload_args *args, bool is_locked)
{
qede_unload(edev, QEDE_UNLOAD_NORMAL);
/* Call function handler to update parameters
* needed for function load.
*/
if (func)
func(edev, args);
if (!is_locked)
__qede_lock(edev);
qede_load(edev, QEDE_LOAD_RELOAD);
/* Since qede_lock is held, internal state wouldn't change even
* if netdev state would start transitioning. Check whether current
* internal configuration indicates device is up, then reload.
*/
if (edev->state == QEDE_STATE_OPEN) {
qede_unload(edev, QEDE_UNLOAD_NORMAL, true);
if (args)
args->func(edev, args);
qede_load(edev, QEDE_LOAD_RELOAD, true);
/* Since no one is going to do it for us, re-configure */
qede_config_rx_mode(edev->ndev);
} else if (args) {
args->func(edev, args);
}
mutex_lock(&edev->qede_lock);
qede_config_rx_mode(edev->ndev);
mutex_unlock(&edev->qede_lock);
if (!is_locked)
__qede_unlock(edev);
}
/* called with rtnl_lock */
......@@ -3755,8 +4040,7 @@ static int qede_open(struct net_device *ndev)
edev->ops->common->set_power_state(edev->cdev, PCI_D0);
rc = qede_load(edev, QEDE_LOAD_NORMAL);
rc = qede_load(edev, QEDE_LOAD_NORMAL, false);
if (rc)
return rc;
......@@ -3771,7 +4055,7 @@ static int qede_close(struct net_device *ndev)
{
struct qede_dev *edev = netdev_priv(ndev);
qede_unload(edev, QEDE_UNLOAD_NORMAL);
qede_unload(edev, QEDE_UNLOAD_NORMAL, false);
edev->ops->common->update_drv_state(edev->cdev, false);
......@@ -3903,15 +4187,8 @@ static void qede_set_rx_mode(struct net_device *ndev)
{
struct qede_dev *edev = netdev_priv(ndev);
DP_INFO(edev, "qede_set_rx_mode called\n");
if (edev->state != QEDE_STATE_OPEN) {
DP_INFO(edev,
"qede_set_rx_mode called while interface is down\n");
} else {
set_bit(QEDE_SP_RX_MODE, &edev->sp_flags);
schedule_delayed_work(&edev->sp_task, 0);
}
set_bit(QEDE_SP_RX_MODE, &edev->sp_flags);
schedule_delayed_work(&edev->sp_task, 0);
}
/* Must be called with qede_lock held */
......
include/linux/qed/qed_chain.h
View file @ 348bfec2
......@@ -56,23 +56,6 @@ struct qed_chain_pbl_u32 {
u32 cons_page_idx;
};
struct qed_chain_pbl {
/* Base address of a pre-allocated buffer for pbl */
dma_addr_t p_phys_table;
void *p_virt_table;
/* Table for keeping the virtual addresses of the chain pages,
* respectively to the physical addresses in the pbl table.
*/
void **pp_virt_addr_tbl;
/* Index to current used page by producer/consumer */
union {
struct qed_chain_pbl_u16 pbl16;
struct qed_chain_pbl_u32 pbl32;
} u;
};
struct qed_chain_u16 {
/* Cyclic index of next element to produce/consme */
u16 prod_idx;
......@@ -86,46 +69,78 @@ struct qed_chain_u32 {
};
struct qed_chain {
void *p_virt_addr;
dma_addr_t p_phys_addr;
void *p_prod_elem;
void *p_cons_elem;
/* fastpath portion of the chain - required for commands such
* as produce / consume.
*/
/* Point to next element to produce/consume */
void *p_prod_elem;
void *p_cons_elem;
/* Fastpath portions of the PBL [if exists] */
struct {
/* Table for keeping the virtual addresses of the chain pages,
* respectively to the physical addresses in the pbl table.
*/
void **pp_virt_addr_tbl;
enum qed_chain_mode mode;
enum qed_chain_use_mode intended_use; /* used to produce/consume */
enum qed_chain_cnt_type cnt_type;
union {
struct qed_chain_pbl_u16 u16;
struct qed_chain_pbl_u32 u32;
} c;
} pbl;
union {
struct qed_chain_u16 chain16;
struct qed_chain_u32 chain32;
} u;
/* Capacity counts only usable elements */
u32 capacity;
u32 page_cnt;
/* Number of elements - capacity is for usable elements only,
* while size will contain total number of elements [for entire chain].
enum qed_chain_mode mode;
/* Elements information for fast calculations */
u16 elem_per_page;
u16 elem_per_page_mask;
u16 elem_size;
u16 next_page_mask;
u16 usable_per_page;
u8 elem_unusable;
u8 cnt_type;
/* Slowpath of the chain - required for initialization and destruction,
* but isn't involved in regular functionality.
*/
u32 capacity;
/* Base address of a pre-allocated buffer for pbl */
struct {
dma_addr_t p_phys_table;
void *p_virt_table;
} pbl_sp;
/* Address of first page of the chain - the address is required
* for fastpath operation [consume/produce] but only for the the SINGLE
* flavour which isn't considered fastpath [== SPQ].
*/
void *p_virt_addr;
dma_addr_t p_phys_addr;
/* Total number of elements [for entire chain] */
u32 size;
/* Elements information for fast calculations */
u16 elem_per_page;
u16 elem_per_page_mask;
u16 elem_unusable;
u16 usable_per_page;
u16 elem_size;
u16 next_page_mask;
struct qed_chain_pbl pbl;
u8 intended_use;
};
#define QED_CHAIN_PBL_ENTRY_SIZE (8)
#define QED_CHAIN_PAGE_SIZE (0x1000)
#define ELEMS_PER_PAGE(elem_size) (QED_CHAIN_PAGE_SIZE / (elem_size))
#define UNUSABLE_ELEMS_PER_PAGE(elem_size, mode) \
((mode == QED_CHAIN_MODE_NEXT_PTR) ? \
(1 + ((sizeof(struct qed_chain_next) - 1) / \
(elem_size))) : 0)
#define UNUSABLE_ELEMS_PER_PAGE(elem_size, mode) \
(((mode) == QED_CHAIN_MODE_NEXT_PTR) ? \
(u8)(1 + ((sizeof(struct qed_chain_next) - 1) / \
(elem_size))) : 0)
#define USABLE_ELEMS_PER_PAGE(elem_size, mode) \
((u32)(ELEMS_PER_PAGE(elem_size) - \
......@@ -186,7 +201,7 @@ static inline u16 qed_chain_get_usable_per_page(struct qed_chain *p_chain)
return p_chain->usable_per_page;
}
static inline u16 qed_chain_get_unusable_per_page(struct qed_chain *p_chain)
static inline u8 qed_chain_get_unusable_per_page(struct qed_chain *p_chain)
{
return p_chain->elem_unusable;
}
......@@ -198,7 +213,7 @@ static inline u32 qed_chain_get_page_cnt(struct qed_chain *p_chain)
static inline dma_addr_t qed_chain_get_pbl_phys(struct qed_chain *p_chain)
{
return p_chain->pbl.p_phys_table;
return p_chain->pbl_sp.p_phys_table;
}
/**
......@@ -214,10 +229,10 @@ static inline dma_addr_t qed_chain_get_pbl_phys(struct qed_chain *p_chain)
static inline void
qed_chain_advance_page(struct qed_chain *p_chain,
void **p_next_elem, void *idx_to_inc, void *page_to_inc)
{
struct qed_chain_next *p_next = NULL;
u32 page_index = 0;
switch (p_chain->mode) {
case QED_CHAIN_MODE_NEXT_PTR:
p_next = *p_next_elem;
......@@ -305,7 +320,7 @@ static inline void *qed_chain_produce(struct qed_chain *p_chain)
if ((p_chain->u.chain16.prod_idx &
p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
p_prod_idx = &p_chain->u.chain16.prod_idx;
p_prod_page_idx = &p_chain->pbl.u.pbl16.prod_page_idx;
p_prod_page_idx = &p_chain->pbl.c.u16.prod_page_idx;
qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
p_prod_idx, p_prod_page_idx);
}
......@@ -314,7 +329,7 @@ static inline void *qed_chain_produce(struct qed_chain *p_chain)
if ((p_chain->u.chain32.prod_idx &
p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
p_prod_idx = &p_chain->u.chain32.prod_idx;
p_prod_page_idx = &p_chain->pbl.u.pbl32.prod_page_idx;
p_prod_page_idx = &p_chain->pbl.c.u32.prod_page_idx;
qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
p_prod_idx, p_prod_page_idx);
}
......@@ -378,7 +393,7 @@ static inline void *qed_chain_consume(struct qed_chain *p_chain)
if ((p_chain->u.chain16.cons_idx &
p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
p_cons_idx = &p_chain->u.chain16.cons_idx;
p_cons_page_idx = &p_chain->pbl.u.pbl16.cons_page_idx;
p_cons_page_idx = &p_chain->pbl.c.u16.cons_page_idx;
qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
p_cons_idx, p_cons_page_idx);
}
......@@ -387,8 +402,8 @@ static inline void *qed_chain_consume(struct qed_chain *p_chain)
if ((p_chain->u.chain32.cons_idx &
p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
p_cons_idx = &p_chain->u.chain32.cons_idx;
p_cons_page_idx = &p_chain->pbl.u.pbl32.cons_page_idx;
qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
p_cons_page_idx = &p_chain->pbl.c.u32.cons_page_idx;
qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
p_cons_idx, p_cons_page_idx);
}
p_chain->u.chain32.cons_idx++;
......@@ -429,25 +444,26 @@ static inline void qed_chain_reset(struct qed_chain *p_chain)
u32 reset_val = p_chain->page_cnt - 1;
if (is_chain_u16(p_chain)) {
p_chain->pbl.u.pbl16.prod_page_idx = (u16)reset_val;
p_chain->pbl.u.pbl16.cons_page_idx = (u16)reset_val;
p_chain->pbl.c.u16.prod_page_idx = (u16)reset_val;
p_chain->pbl.c.u16.cons_page_idx = (u16)reset_val;
} else {
p_chain->pbl.u.pbl32.prod_page_idx = reset_val;
p_chain->pbl.u.pbl32.cons_page_idx = reset_val;
p_chain->pbl.c.u32.prod_page_idx = reset_val;
p_chain->pbl.c.u32.cons_page_idx = reset_val;
}
}
switch (p_chain->intended_use) {
case QED_CHAIN_USE_TO_CONSUME_PRODUCE:
case QED_CHAIN_USE_TO_PRODUCE:
/* Do nothing */
break;
case QED_CHAIN_USE_TO_CONSUME:
/* produce empty elements */
for (i = 0; i < p_chain->capacity; i++)
qed_chain_recycle_consumed(p_chain);
break;
case QED_CHAIN_USE_TO_CONSUME_PRODUCE:
case QED_CHAIN_USE_TO_PRODUCE:
default:
/* Do nothing */
break;
}
}
......@@ -473,13 +489,13 @@ static inline void qed_chain_init_params(struct qed_chain *p_chain,
p_chain->p_virt_addr = NULL;
p_chain->p_phys_addr = 0;
p_chain->elem_size = elem_size;
p_chain->intended_use = intended_use;
p_chain->intended_use = (u8)intended_use;
p_chain->mode = mode;
p_chain->cnt_type = cnt_type;
p_chain->cnt_type = (u8)cnt_type;
p_chain->elem_per_page = ELEMS_PER_PAGE(elem_size);
p_chain->elem_per_page = ELEMS_PER_PAGE(elem_size);
p_chain->usable_per_page = USABLE_ELEMS_PER_PAGE(elem_size, mode);
p_chain->elem_per_page_mask = p_chain->elem_per_page - 1;
p_chain->elem_per_page_mask = p_chain->elem_per_page - 1;
p_chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(elem_size, mode);
p_chain->next_page_mask = (p_chain->usable_per_page &
p_chain->elem_per_page_mask);
......@@ -488,8 +504,8 @@ static inline void qed_chain_init_params(struct qed_chain *p_chain,
p_chain->capacity = p_chain->usable_per_page * page_cnt;
p_chain->size = p_chain->elem_per_page * page_cnt;
p_chain->pbl.p_phys_table = 0;
p_chain->pbl.p_virt_table = NULL;
p_chain->pbl_sp.p_phys_table = 0;
p_chain->pbl_sp.p_virt_table = NULL;
p_chain->pbl.pp_virt_addr_tbl = NULL;
}
......@@ -530,8 +546,8 @@ static inline void qed_chain_init_pbl_mem(struct qed_chain *p_chain,
dma_addr_t p_phys_pbl,
void **pp_virt_addr_tbl)
{
p_chain->pbl.p_phys_table = p_phys_pbl;
p_chain->pbl.p_virt_table = p_virt_pbl;
p_chain->pbl_sp.p_phys_table = p_phys_pbl;
p_chain->pbl_sp.p_virt_table = p_virt_pbl;
p_chain->pbl.pp_virt_addr_tbl = pp_virt_addr_tbl;
}
......
include/linux/qed/qed_eth_if.h
View file @ 348bfec2
......@@ -15,6 +15,29 @@
#include <linux/qed/qed_if.h>
#include <linux/qed/qed_iov_if.h>
struct qed_queue_start_common_params {
/* Should always be relative to entity sending this. */
u8 vport_id;
u16 queue_id;
/* Relative, but relevant only for PFs */
u8 stats_id;
/* These are always absolute */
u16 sb;
u8 sb_idx;
};
struct qed_rxq_start_ret_params {
void __iomem *p_prod;
void *p_handle;
};
struct qed_txq_start_ret_params {
void __iomem *p_doorbell;
void *p_handle;
};
struct qed_dev_eth_info {
struct qed_dev_info common;
......@@ -56,18 +79,6 @@ struct qed_start_vport_params {
bool clear_stats;
};
struct qed_stop_rxq_params {
u8 rss_id;
u8 rx_queue_id;
u8 vport_id;
bool eq_completion_only;
};
struct qed_stop_txq_params {
u8 rss_id;
u8 tx_queue_id;
};
enum qed_filter_rx_mode_type {
QED_FILTER_RX_MODE_TYPE_REGULAR,
QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC,
......@@ -112,15 +123,6 @@ struct qed_filter_params {
union qed_filter_type_params filter;
};
struct qed_queue_start_common_params {
u8 rss_id;
u8 queue_id;
u8 vport_id;
u16 sb;
u16 sb_idx;
u16 vf_qid;
};
struct qed_tunn_params {
u16 vxlan_port;
u8 update_vxlan_port;
......@@ -220,24 +222,24 @@ struct qed_eth_ops {
struct qed_update_vport_params *params);
int (*q_rx_start)(struct qed_dev *cdev,
u8 rss_num,
struct qed_queue_start_common_params *params,
u16 bd_max_bytes,
dma_addr_t bd_chain_phys_addr,
dma_addr_t cqe_pbl_addr,
u16 cqe_pbl_size,
void __iomem **pp_prod);
struct qed_rxq_start_ret_params *ret_params);
int (*q_rx_stop)(struct qed_dev *cdev,
struct qed_stop_rxq_params *params);
int (*q_rx_stop)(struct qed_dev *cdev, u8 rss_id, void *handle);
int (*q_tx_start)(struct qed_dev *cdev,
u8 rss_num,
struct qed_queue_start_common_params *params,
dma_addr_t pbl_addr,
u16 pbl_size,
void __iomem **pp_doorbell);
struct qed_txq_start_ret_params *ret_params);
int (*q_tx_stop)(struct qed_dev *cdev,
struct qed_stop_txq_params *params);
int (*q_tx_stop)(struct qed_dev *cdev, u8 rss_id, void *handle);
int (*filter_config)(struct qed_dev *cdev,
struct qed_filter_params *params);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7