Commit 5adf4c47 authored by Tariq Toukan's avatar Tariq Toukan Committed by Saeed Mahameed

net/mlx5e: RX, Re-work initializaiton of RX function pointers

Instead of exposing the RQ datapath handlers (from en_rx.c) so that
they are set in the control path (in en_main.c), wrap this logic
in a single function in en_rx.c and expose it alone.

Every profile will now have a pointer to the new mlx5e_rx_handlers
structure, instead of directly pointing to the previously-exposed
RQ handlers.

This significantly improves locality and modularity of the driver,
and allows many functions in en_rx.c to become static.
Signed-off-by: default avatarTariq Toukan <tariqt@mellanox.com>
Reviewed-by: default avatarMaxim Mikityanskiy <maximmi@mellanox.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
parent 123f0f53
...@@ -530,6 +530,8 @@ typedef struct sk_buff * ...@@ -530,6 +530,8 @@ typedef struct sk_buff *
typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq); typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);
typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16); typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk);
enum mlx5e_rq_flag { enum mlx5e_rq_flag {
MLX5E_RQ_FLAG_XDP_XMIT, MLX5E_RQ_FLAG_XDP_XMIT,
MLX5E_RQ_FLAG_XDP_REDIRECT, MLX5E_RQ_FLAG_XDP_REDIRECT,
...@@ -812,6 +814,13 @@ struct mlx5e_priv { ...@@ -812,6 +814,13 @@ struct mlx5e_priv {
struct mlx5e_scratchpad scratchpad; struct mlx5e_scratchpad scratchpad;
}; };
struct mlx5e_rx_handlers {
mlx5e_fp_handle_rx_cqe handle_rx_cqe;
mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe;
};
extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic;
struct mlx5e_profile { struct mlx5e_profile {
int (*init)(struct mlx5_core_dev *mdev, int (*init)(struct mlx5_core_dev *mdev,
struct net_device *netdev, struct net_device *netdev,
...@@ -828,10 +837,7 @@ struct mlx5e_profile { ...@@ -828,10 +837,7 @@ struct mlx5e_profile {
void (*update_carrier)(struct mlx5e_priv *priv); void (*update_carrier)(struct mlx5e_priv *priv);
unsigned int (*stats_grps_num)(struct mlx5e_priv *priv); unsigned int (*stats_grps_num)(struct mlx5e_priv *priv);
mlx5e_stats_grp_t *stats_grps; mlx5e_stats_grp_t *stats_grps;
struct { const struct mlx5e_rx_handlers *rx_handlers;
mlx5e_fp_handle_rx_cqe handle_rx_cqe;
mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe;
} rx_handlers;
int max_tc; int max_tc;
u8 rq_groups; u8 rq_groups;
}; };
...@@ -860,26 +866,9 @@ void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info); ...@@ -860,26 +866,9 @@ void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info);
void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
struct mlx5e_dma_info *dma_info, struct mlx5e_dma_info *dma_info,
bool recycle); bool recycle);
void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq); bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq);
int mlx5e_poll_ico_cq(struct mlx5e_cq *cq); int mlx5e_poll_ico_cq(struct mlx5e_cq *cq);
bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq); bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);
void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx);
struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx);
struct sk_buff *
mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt);
struct sk_buff *
mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt);
void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s); void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s);
......
...@@ -47,7 +47,6 @@ ...@@ -47,7 +47,6 @@
struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev, struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev,
struct sk_buff *skb, u32 *cqe_bcnt); struct sk_buff *skb, u32 *cqe_bcnt);
void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
void mlx5e_ipsec_inverse_table_init(void); void mlx5e_ipsec_inverse_table_init(void);
bool mlx5e_ipsec_feature_check(struct sk_buff *skb, struct net_device *netdev, bool mlx5e_ipsec_feature_check(struct sk_buff *skb, struct net_device *netdev,
......
...@@ -45,7 +45,6 @@ ...@@ -45,7 +45,6 @@
#include "en_tc.h" #include "en_tc.h"
#include "en_rep.h" #include "en_rep.h"
#include "en_accel/ipsec.h" #include "en_accel/ipsec.h"
#include "en_accel/ipsec_rxtx.h"
#include "en_accel/en_accel.h" #include "en_accel/en_accel.h"
#include "en_accel/tls.h" #include "en_accel/tls.h"
#include "accel/ipsec.h" #include "accel/ipsec.h"
...@@ -65,7 +64,6 @@ ...@@ -65,7 +64,6 @@
#include "en/hv_vhca_stats.h" #include "en/hv_vhca_stats.h"
#include "en/devlink.h" #include "en/devlink.h"
#include "lib/mlx5.h" #include "lib/mlx5.h"
#include "fpga/ipsec.h"
bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
{ {
...@@ -428,29 +426,6 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, ...@@ -428,29 +426,6 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
pool_size = MLX5_MPWRQ_PAGES_PER_WQE << pool_size = MLX5_MPWRQ_PAGES_PER_WQE <<
mlx5e_mpwqe_get_log_rq_size(params, xsk); mlx5e_mpwqe_get_log_rq_size(params, xsk);
rq->post_wqes = mlx5e_post_rx_mpwqes;
rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe;
#ifdef CONFIG_MLX5_EN_IPSEC
if (MLX5_IPSEC_DEV(mdev)) {
err = -EINVAL;
netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n");
goto err_rq_wq_destroy;
}
#endif
if (!rq->handle_rx_cqe) {
err = -EINVAL;
netdev_err(c->netdev, "RX handler of MPWQE RQ is not set, err %d\n", err);
goto err_rq_wq_destroy;
}
rq->mpwqe.skb_from_cqe_mpwrq = xsk ?
mlx5e_xsk_skb_from_cqe_mpwrq_linear :
mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ?
mlx5e_skb_from_cqe_mpwrq_linear :
mlx5e_skb_from_cqe_mpwrq_nonlinear;
rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
rq->mpwqe.num_strides = rq->mpwqe.num_strides =
BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
...@@ -492,30 +467,13 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, ...@@ -492,30 +467,13 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
if (err) if (err)
goto err_free; goto err_free;
rq->post_wqes = mlx5e_post_rx_wqes;
rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
#ifdef CONFIG_MLX5_EN_IPSEC
if ((mlx5_fpga_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) &&
c->priv->ipsec)
rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
else
#endif
rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe;
if (!rq->handle_rx_cqe) {
err = -EINVAL;
netdev_err(c->netdev, "RX handler of RQ is not set, err %d\n", err);
goto err_free;
}
rq->wqe.skb_from_cqe = xsk ?
mlx5e_xsk_skb_from_cqe_linear :
mlx5e_rx_is_linear_skb(params, NULL) ?
mlx5e_skb_from_cqe_linear :
mlx5e_skb_from_cqe_nonlinear;
rq->mkey_be = c->mkey_be; rq->mkey_be = c->mkey_be;
} }
err = mlx5e_rq_set_handlers(rq, params, xsk);
if (err)
goto err_free;
if (xsk) { if (xsk) {
err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
MEM_TYPE_XSK_BUFF_POOL, NULL); MEM_TYPE_XSK_BUFF_POOL, NULL);
...@@ -5288,8 +5246,7 @@ static const struct mlx5e_profile mlx5e_nic_profile = { ...@@ -5288,8 +5246,7 @@ static const struct mlx5e_profile mlx5e_nic_profile = {
.update_rx = mlx5e_update_nic_rx, .update_rx = mlx5e_update_nic_rx,
.update_stats = mlx5e_update_ndo_stats, .update_stats = mlx5e_update_ndo_stats,
.update_carrier = mlx5e_update_carrier, .update_carrier = mlx5e_update_carrier,
.rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe, .rx_handlers = &mlx5e_rx_handlers_nic,
.rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
.max_tc = MLX5E_MAX_NUM_TC, .max_tc = MLX5E_MAX_NUM_TC,
.rq_groups = MLX5E_NUM_RQ_GROUPS(XSK), .rq_groups = MLX5E_NUM_RQ_GROUPS(XSK),
.stats_grps = mlx5e_nic_stats_grps, .stats_grps = mlx5e_nic_stats_grps,
......
...@@ -1143,8 +1143,7 @@ static const struct mlx5e_profile mlx5e_rep_profile = { ...@@ -1143,8 +1143,7 @@ static const struct mlx5e_profile mlx5e_rep_profile = {
.enable = mlx5e_rep_enable, .enable = mlx5e_rep_enable,
.update_rx = mlx5e_update_rep_rx, .update_rx = mlx5e_update_rep_rx,
.update_stats = mlx5e_update_ndo_stats, .update_stats = mlx5e_update_ndo_stats,
.rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe_rep, .rx_handlers = &mlx5e_rx_handlers_rep,
.rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq_rep,
.max_tc = 1, .max_tc = 1,
.rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR),
.stats_grps = mlx5e_rep_stats_grps, .stats_grps = mlx5e_rep_stats_grps,
...@@ -1163,8 +1162,7 @@ static const struct mlx5e_profile mlx5e_uplink_rep_profile = { ...@@ -1163,8 +1162,7 @@ static const struct mlx5e_profile mlx5e_uplink_rep_profile = {
.update_rx = mlx5e_update_rep_rx, .update_rx = mlx5e_update_rep_rx,
.update_stats = mlx5e_update_ndo_stats, .update_stats = mlx5e_update_ndo_stats,
.update_carrier = mlx5e_update_carrier, .update_carrier = mlx5e_update_carrier,
.rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe_rep, .rx_handlers = &mlx5e_rx_handlers_rep,
.rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq_rep,
.max_tc = MLX5E_MAX_NUM_TC, .max_tc = MLX5E_MAX_NUM_TC,
.rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR),
.stats_grps = mlx5e_ul_rep_stats_grps, .stats_grps = mlx5e_ul_rep_stats_grps,
......
...@@ -41,6 +41,8 @@ ...@@ -41,6 +41,8 @@
#include "lib/port_tun.h" #include "lib/port_tun.h"
#ifdef CONFIG_MLX5_ESWITCH #ifdef CONFIG_MLX5_ESWITCH
extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep;
struct mlx5e_neigh_update_table { struct mlx5e_neigh_update_table {
struct rhashtable neigh_ht; struct rhashtable neigh_ht;
/* Save the neigh hash entries in a list in addition to the hash table /* Save the neigh hash entries in a list in addition to the hash table
...@@ -223,10 +225,6 @@ bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv); ...@@ -223,10 +225,6 @@ bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv);
int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv); int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv);
void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv); void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv);
void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq,
struct mlx5_cqe64 *cqe);
void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv); void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv);
bool mlx5e_eswitch_vf_rep(struct net_device *netdev); bool mlx5e_eswitch_vf_rep(struct net_device *netdev);
......
...@@ -44,12 +44,29 @@ ...@@ -44,12 +44,29 @@
#include "en_rep.h" #include "en_rep.h"
#include "en/rep/tc.h" #include "en/rep/tc.h"
#include "ipoib/ipoib.h" #include "ipoib/ipoib.h"
#include "accel/ipsec.h"
#include "fpga/ipsec.h"
#include "en_accel/ipsec_rxtx.h" #include "en_accel/ipsec_rxtx.h"
#include "en_accel/tls_rxtx.h" #include "en_accel/tls_rxtx.h"
#include "lib/clock.h" #include "lib/clock.h"
#include "en/xdp.h" #include "en/xdp.h"
#include "en/xsk/rx.h" #include "en/xsk/rx.h"
#include "en/health.h" #include "en/health.h"
#include "en/params.h"
static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx);
static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx);
static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic = {
.handle_rx_cqe = mlx5e_handle_rx_cqe,
.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
};
static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config) static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
{ {
...@@ -370,7 +387,7 @@ static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq, ...@@ -370,7 +387,7 @@ static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
mlx5e_put_rx_frag(rq, wi, recycle); mlx5e_put_rx_frag(rq, wi, recycle);
} }
void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix) static void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
{ {
struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix); struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
...@@ -537,7 +554,7 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) ...@@ -537,7 +554,7 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
return err; return err;
} }
void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
{ {
struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix]; struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
/* Don't recycle, this function is called on rq/netdev close */ /* Don't recycle, this function is called on rq/netdev close */
...@@ -1106,7 +1123,7 @@ static void mlx5e_fill_xdp_buff(struct mlx5e_rq *rq, void *va, u16 headroom, ...@@ -1106,7 +1123,7 @@ static void mlx5e_fill_xdp_buff(struct mlx5e_rq *rq, void *va, u16 headroom,
xdp->frame_sz = rq->buff.frame0_sz; xdp->frame_sz = rq->buff.frame0_sz;
} }
struct sk_buff * static struct sk_buff *
mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
{ {
...@@ -1146,7 +1163,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, ...@@ -1146,7 +1163,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
return skb; return skb;
} }
struct sk_buff * static struct sk_buff *
mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
{ {
...@@ -1201,7 +1218,7 @@ static void trigger_report(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) ...@@ -1201,7 +1218,7 @@ static void trigger_report(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
} }
} }
void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ {
struct mlx5_wq_cyc *wq = &rq->wqe.wq; struct mlx5_wq_cyc *wq = &rq->wqe.wq;
struct mlx5e_wqe_frag_info *wi; struct mlx5e_wqe_frag_info *wi;
...@@ -1244,7 +1261,7 @@ void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) ...@@ -1244,7 +1261,7 @@ void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
} }
#ifdef CONFIG_MLX5_ESWITCH #ifdef CONFIG_MLX5_ESWITCH
void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ {
struct net_device *netdev = rq->netdev; struct net_device *netdev = rq->netdev;
struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5e_priv *priv = netdev_priv(netdev);
...@@ -1299,8 +1316,7 @@ void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) ...@@ -1299,8 +1316,7 @@ void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
mlx5_wq_cyc_pop(wq); mlx5_wq_cyc_pop(wq);
} }
void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
struct mlx5_cqe64 *cqe)
{ {
u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
u16 wqe_id = be16_to_cpu(cqe->wqe_id); u16 wqe_id = be16_to_cpu(cqe->wqe_id);
...@@ -1358,9 +1374,14 @@ void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, ...@@ -1358,9 +1374,14 @@ void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq,
mlx5e_free_rx_mpwqe(rq, wi, true); mlx5e_free_rx_mpwqe(rq, wi, true);
mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index); mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
} }
const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = {
.handle_rx_cqe = mlx5e_handle_rx_cqe_rep,
.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq_rep,
};
#endif #endif
struct sk_buff * static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx) u16 cqe_bcnt, u32 head_offset, u32 page_idx)
{ {
...@@ -1406,7 +1427,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w ...@@ -1406,7 +1427,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
return skb; return skb;
} }
struct sk_buff * static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx) u16 cqe_bcnt, u32 head_offset, u32 page_idx)
{ {
...@@ -1456,7 +1477,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, ...@@ -1456,7 +1477,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
return skb; return skb;
} }
void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ {
u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
u16 wqe_id = be16_to_cpu(cqe->wqe_id); u16 wqe_id = be16_to_cpu(cqe->wqe_id);
...@@ -1652,7 +1673,7 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq, ...@@ -1652,7 +1673,7 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
stats->bytes += cqe_bcnt; stats->bytes += cqe_bcnt;
} }
void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) static void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ {
struct mlx5_wq_cyc *wq = &rq->wqe.wq; struct mlx5_wq_cyc *wq = &rq->wqe.wq;
struct mlx5e_wqe_frag_info *wi; struct mlx5e_wqe_frag_info *wi;
...@@ -1688,11 +1709,15 @@ void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) ...@@ -1688,11 +1709,15 @@ void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
mlx5_wq_cyc_pop(wq); mlx5_wq_cyc_pop(wq);
} }
const struct mlx5e_rx_handlers mlx5i_rx_handlers = {
.handle_rx_cqe = mlx5i_handle_rx_cqe,
.handle_rx_cqe_mpwqe = NULL, /* Not supported */
};
#endif /* CONFIG_MLX5_CORE_IPOIB */ #endif /* CONFIG_MLX5_CORE_IPOIB */
#ifdef CONFIG_MLX5_EN_IPSEC #ifdef CONFIG_MLX5_EN_IPSEC
void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) static void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ {
struct mlx5_wq_cyc *wq = &rq->wqe.wq; struct mlx5_wq_cyc *wq = &rq->wqe.wq;
struct mlx5e_wqe_frag_info *wi; struct mlx5e_wqe_frag_info *wi;
...@@ -1729,3 +1754,55 @@ void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) ...@@ -1729,3 +1754,55 @@ void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
} }
#endif /* CONFIG_MLX5_EN_IPSEC */ #endif /* CONFIG_MLX5_EN_IPSEC */
int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk)
{
struct mlx5_core_dev *mdev = rq->mdev;
struct mlx5e_channel *c = rq->channel;
switch (rq->wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
rq->mpwqe.skb_from_cqe_mpwrq = xsk ?
mlx5e_xsk_skb_from_cqe_mpwrq_linear :
mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ?
mlx5e_skb_from_cqe_mpwrq_linear :
mlx5e_skb_from_cqe_mpwrq_nonlinear;
rq->post_wqes = mlx5e_post_rx_mpwqes;
rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
rq->handle_rx_cqe = c->priv->profile->rx_handlers->handle_rx_cqe_mpwqe;
#ifdef CONFIG_MLX5_EN_IPSEC
if (MLX5_IPSEC_DEV(mdev)) {
netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n");
return -EINVAL;
}
#endif
if (!rq->handle_rx_cqe) {
netdev_err(c->netdev, "RX handler of MPWQE RQ is not set\n");
return -EINVAL;
}
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
rq->wqe.skb_from_cqe = xsk ?
mlx5e_xsk_skb_from_cqe_linear :
mlx5e_rx_is_linear_skb(params, NULL) ?
mlx5e_skb_from_cqe_linear :
mlx5e_skb_from_cqe_nonlinear;
rq->post_wqes = mlx5e_post_rx_wqes;
rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
#ifdef CONFIG_MLX5_EN_IPSEC
if ((mlx5_fpga_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) &&
c->priv->ipsec)
rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
else
#endif
rq->handle_rx_cqe = c->priv->profile->rx_handlers->handle_rx_cqe;
if (!rq->handle_rx_cqe) {
netdev_err(c->netdev, "RX handler of RQ is not set\n");
return -EINVAL;
}
}
return 0;
}
...@@ -464,8 +464,7 @@ static const struct mlx5e_profile mlx5i_nic_profile = { ...@@ -464,8 +464,7 @@ static const struct mlx5e_profile mlx5i_nic_profile = {
.update_rx = mlx5i_update_nic_rx, .update_rx = mlx5i_update_nic_rx,
.update_stats = NULL, /* mlx5i_update_stats */ .update_stats = NULL, /* mlx5i_update_stats */
.update_carrier = NULL, /* no HW update in IB link */ .update_carrier = NULL, /* no HW update in IB link */
.rx_handlers.handle_rx_cqe = mlx5i_handle_rx_cqe, .rx_handlers = &mlx5i_rx_handlers,
.rx_handlers.handle_rx_cqe_mpwqe = NULL, /* Not supported */
.max_tc = MLX5I_MAX_NUM_TC, .max_tc = MLX5I_MAX_NUM_TC,
.rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR),
.stats_grps = mlx5i_stats_grps, .stats_grps = mlx5i_stats_grps,
......
...@@ -42,6 +42,7 @@ ...@@ -42,6 +42,7 @@
extern const struct ethtool_ops mlx5i_ethtool_ops; extern const struct ethtool_ops mlx5i_ethtool_ops;
extern const struct ethtool_ops mlx5i_pkey_ethtool_ops; extern const struct ethtool_ops mlx5i_pkey_ethtool_ops;
extern const struct mlx5e_rx_handlers mlx5i_rx_handlers;
#define MLX5_IB_GRH_BYTES 40 #define MLX5_IB_GRH_BYTES 40
#define MLX5_IPOIB_ENCAP_LEN 4 #define MLX5_IPOIB_ENCAP_LEN 4
...@@ -117,7 +118,6 @@ struct mlx5i_tx_wqe { ...@@ -117,7 +118,6 @@ struct mlx5i_tx_wqe {
void mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, void mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
struct mlx5_av *av, u32 dqpn, u32 dqkey, bool xmit_more); struct mlx5_av *av, u32 dqpn, u32 dqkey, bool xmit_more);
void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
void mlx5i_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); void mlx5i_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
#endif /* CONFIG_MLX5_CORE_IPOIB */ #endif /* CONFIG_MLX5_CORE_IPOIB */
......
...@@ -349,8 +349,7 @@ static const struct mlx5e_profile mlx5i_pkey_nic_profile = { ...@@ -349,8 +349,7 @@ static const struct mlx5e_profile mlx5i_pkey_nic_profile = {
.disable = NULL, .disable = NULL,
.update_rx = mlx5i_update_nic_rx, .update_rx = mlx5i_update_nic_rx,
.update_stats = NULL, .update_stats = NULL,
.rx_handlers.handle_rx_cqe = mlx5i_handle_rx_cqe, .rx_handlers = &mlx5i_rx_handlers,
.rx_handlers.handle_rx_cqe_mpwqe = NULL, /* Not supported */
.max_tc = MLX5I_MAX_NUM_TC, .max_tc = MLX5I_MAX_NUM_TC,
.rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR),
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment